diff --git a/src-tauri/crates/berd-voice/PROTOCOL.md b/src-tauri/crates/berd-voice/PROTOCOL.md index 786a3efcf..8306ff893 100644 --- a/src-tauri/crates/berd-voice/PROTOCOL.md +++ b/src-tauri/crates/berd-voice/PROTOCOL.md @@ -3,8 +3,10 @@ `berd-voice session` is a development, full-authority voice session. The child owns speech recognition, finalized-input order, confirmation, speak admission, synthesis, source-frame delivery, playback lifecycle, and barge-in. The parent -owns capture and playback devices: it writes normalized microphone PCM on stdin -and consumes synthesized PCM from a dedicated inherited pipe. The child writes +owns capture and conversational playback devices: it writes normalized microphone PCM on +stdin and consumes synthesized PCM from a dedicated inherited pipe. Status cues are +the narrow exception: the child plays them on the output-device name supplied by the +parent, or the system default when none is supplied. The child writes flushed JSONL events to stdout. Diagnostics go only to stderr. ## Startup @@ -44,17 +46,19 @@ remain terminal speech events. has no device-owning or stdout-multiplexed fallback. The first request must be `hello`. `input_during_tts` is the host's resolved -initial policy; a host-specific `auto` mode must be resolved before the request: +initial policy; a host-specific `auto` mode must be resolved before the request. +`status_sound_output_device` is the host's selected playback-device name; omitting it uses the system default: ```json -{"type":"hello","id":1,"input_during_tts":"allow_barge_in"} +{"type":"hello","id":1,"input_during_tts":"allow_barge_in","status_sound_output_device":"MacBook Pro Speakers"} ``` -The response uses `protocol:4` as the exact session message-set version. The -parent must reject a version it does not support: +The response uses `protocol:5` as the exact session message-set version. The +parent must reject a version it does not support; the JSON version is independent +of the fixed binary framing marker described below: ```json -{"type":"ready","id":1,"protocol":4,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} +{"type":"ready","id":1,"protocol":5,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} ``` The `session.tts` object is the authoritative, sanitized TTS configuration. @@ -64,7 +68,9 @@ and `rate`. Credentials, endpoints, and bundle paths never appear on stdout. Detailed backend errors are diagnostics on stderr only; protocol rejection and fatal messages are sanitized at the stdout boundary. `session.input_during_tts` is the authoritative effective assistant-input -policy and has its own revision. +policy and has its own revision. Status-sound settings are per-update parameters, +not session snapshot configuration. The runtime owns cue cadence and playback; it +does not persist preferences. ## Stdin framing @@ -179,9 +185,10 @@ it never admits a replacement while old host audio may still be active. ## Parent requests ```text -{"type":"hello","id":u64,"input_during_tts":"allow_barge_in"|"suppress_input"} +{"type":"hello","id":u64,"input_during_tts":"allow_barge_in"|"suppress_input","status_sound_output_device":string?} {"type":"set_paused","active":bool} {"type":"set_input_muted","id":u64,"active":bool} +{"type":"set_conversation_status","id":u64,"status":"working"|"waiting","settings":StatusSoundSettings} {"type":"set_tts_settings","id":u64,"expected_revision":u64,"settings":TtsSettings} {"type":"set_input_during_tts","id":u64,"expected_revision":u64,"policy":"allow_barge_in"|"suppress_input"} {"type":"reset_input","id":u64} @@ -198,6 +205,12 @@ Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. The parent cannot author speaking state or finalized input; those are derived only from PCM by the child runtime. +`StatusSoundSettings` has a `mode` of `off`, `working`, or `working-and-waiting` and an optional per-session `volume` from 0 to 1 that defaults to `0.8`. No cue is emitted until the first `set_conversation_status` request. Cues play immediately on status transitions and repeat every five seconds; repeated identical updates preserve the cadence. After conversation audio ends, the current cue resumes immediately. `off` disables both cues; `working` repeats only the working cue and stays silent while waiting; `working-and-waiting` repeats the current working or waiting cue. Active user speech or assistant output suppresses playback without changing the working/waiting state. Pending recognition alone does not suppress playback. On macOS, working uses the system Pop sound and waiting uses Purr through the selected output device. The applied request is acknowledged with: + +```text +{"type":"conversation_status_applied","id":u64,"status":"working"|"waiting","settings":StatusSoundSettings} +``` + `set_tts_settings` accepts the same tagged public object projected by `ready`, without `revision`. It changes settings only for the already-active backend: diff --git a/src-tauri/crates/berd-voice/README.md b/src-tauri/crates/berd-voice/README.md index 3e14b5b3c..0035f54d1 100644 --- a/src-tauri/crates/berd-voice/README.md +++ b/src-tauri/crates/berd-voice/README.md @@ -18,8 +18,12 @@ Siri bridge emits normalized 48 kHz mono Float32 PCM without opening an audio device; the existing Berd Siri player and the CLI use the same decoder. `berd-voice session` exposes the development voice-session protocol documented -in [PROTOCOL.md](PROTOCOL.md). Siri TTS and macOS speech recognition are the -defaults: +in [PROTOCOL.md](PROTOCOL.md). The host supplies persisted status-sound settings +and semantic `working` / `waiting` updates through that protocol; the runtime +owns the five-second cadence, speech suppression, and macOS Pop/Purr playback. +The default mode is `working`; `working-and-waiting` also plays idle cues, and `off` disables both. Status sounds default to a gain of `0.8`. Cues play immediately on a status transition and repeat every five seconds. When conversation audio ends, the current cue resumes immediately. Repeated identical status updates preserve the cadence. The host determines when work begins; the chained client signals working on its first live tool call and stays working until the run ends. + +Siri TTS and macOS speech recognition are the defaults: ```text berd-voice session --voice Aaron --language en-US --rate 1.0 diff --git a/src-tauri/crates/berd-voice/native/siri_tts_bridge.h b/src-tauri/crates/berd-voice/native/siri_tts_bridge.h index 153c51815..7db985d66 100644 --- a/src-tauri/crates/berd-voice/native/siri_tts_bridge.h +++ b/src-tauri/crates/berd-voice/native/siri_tts_bridge.h @@ -101,6 +101,16 @@ bool berd_siri_tts_speak( char **error_out ); +/// Decodes an audio file through AVFoundation into malloc-owned mono Float32 PCM. +/// The caller releases successful samples with `berd_audio_free_samples`. +float *berd_audio_file_load_mono_pcm( + const char *path, + uint32_t *sample_rate_out, + uint32_t *frame_count_out, + char **error_out +); +void berd_audio_free_samples(float *samples); + /// Opaque Pocket PCM player backed by AVAudioUnitTimePitch. Samples are /// mono, noninterleaved float PCM. Device ID 0 uses the system default. void *berd_pocket_audio_player_create( diff --git a/src-tauri/crates/berd-voice/native/siri_tts_bridge.m b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m index ec80eb370..74404c1f6 100644 --- a/src-tauri/crates/berd-voice/native/siri_tts_bridge.m +++ b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m @@ -1604,6 +1604,99 @@ bool berd_siri_tts_speak( } } +float *berd_audio_file_load_mono_pcm( + const char *pathValue, + uint32_t *sampleRateOut, + uint32_t *frameCountOut, + char **errorOut +) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + if (!pathValue || !sampleRateOut || !frameCountOut) { + BerdSetError(errorOut, BerdError(38, @"An audio path and output pointers are required.")); + return NULL; + } + NSURL *url = [NSURL fileURLWithPath:[NSString stringWithUTF8String:pathValue]]; + ExtAudioFileRef file = NULL; + OSStatus status = ExtAudioFileOpenURL((__bridge CFURLRef)url, &file); + if (status != noErr || !file) { + BerdSetError(errorOut, BerdError(39, @"Could not open the audio file.")); + return NULL; + } + AudioStreamBasicDescription sourceFormat = {0}; + UInt32 propertySize = sizeof(sourceFormat); + status = ExtAudioFileGetProperty( + file, kExtAudioFileProperty_FileDataFormat, &propertySize, &sourceFormat); + SInt64 sourceFrames = 0; + propertySize = sizeof(sourceFrames); + if (status == noErr) { + status = ExtAudioFileGetProperty( + file, kExtAudioFileProperty_FileLengthFrames, &propertySize, &sourceFrames); + } + AudioStreamBasicDescription clientFormat = {0}; + clientFormat.mSampleRate = sourceFormat.mSampleRate; + clientFormat.mFormatID = kAudioFormatLinearPCM; + clientFormat.mFormatFlags = kAudioFormatFlagsNativeFloatPacked; + uint32_t channelCount = sourceFormat.mChannelsPerFrame; + clientFormat.mBytesPerPacket = sizeof(float) * channelCount; + clientFormat.mFramesPerPacket = 1; + clientFormat.mBytesPerFrame = sizeof(float) * channelCount; + clientFormat.mChannelsPerFrame = channelCount; + clientFormat.mBitsPerChannel = 8 * sizeof(float); + if (status == noErr) { + status = ExtAudioFileSetProperty( + file, kExtAudioFileProperty_ClientDataFormat, + sizeof(clientFormat), &clientFormat); + } + if (status != noErr || sourceFrames <= 0 || sourceFrames > UINT32_MAX) { + ExtAudioFileDispose(file); + BerdSetError(errorOut, BerdError(40, @"Could not prepare the audio file for decoding.")); + return NULL; + } + uint32_t capacity = (uint32_t)sourceFrames; + float *samples = malloc((size_t)capacity * channelCount * sizeof(float)); + if (!samples) { + ExtAudioFileDispose(file); + BerdSetError(errorOut, BerdError(41, @"Could not allocate decoded audio samples.")); + return NULL; + } + uint32_t frameCount = 0; + while (frameCount < capacity) { + UInt32 requestedFrames = capacity - frameCount; + AudioBufferList buffers = {0}; + buffers.mNumberBuffers = 1; + buffers.mBuffers[0].mNumberChannels = channelCount; + buffers.mBuffers[0].mDataByteSize = requestedFrames * channelCount * sizeof(float); + buffers.mBuffers[0].mData = samples + ((size_t)frameCount * channelCount); + status = ExtAudioFileRead(file, &requestedFrames, &buffers); + if (status != noErr || requestedFrames == 0) break; + frameCount += requestedFrames; + } + ExtAudioFileDispose(file); + if (status != noErr || frameCount != capacity) { + free(samples); + BerdSetError(errorOut, BerdError(42, @"Could not decode the audio file.")); + return NULL; + } + if (channelCount > 1) { + for (uint32_t frame = 0; frame < frameCount; frame++) { + float mixed = 0; + for (uint32_t channel = 0; channel < channelCount; channel++) { + mixed += samples[frame * channelCount + channel]; + } + samples[frame] = mixed / channelCount; + } + } + *sampleRateOut = (uint32_t)clientFormat.mSampleRate; + *frameCountOut = frameCount; + return samples; + } +} + +void berd_audio_free_samples(float *samples) { + free(samples); +} + void *berd_pocket_audio_player_create( uint32_t sampleRate, float rate, diff --git a/src-tauri/crates/berd-voice/src/lib.rs b/src-tauri/crates/berd-voice/src/lib.rs index f5a792564..ba7ddd06d 100644 --- a/src-tauri/crates/berd-voice/src/lib.rs +++ b/src-tauri/crates/berd-voice/src/lib.rs @@ -32,6 +32,7 @@ pub mod realtime_pipe; pub mod session; pub mod siri; pub mod spokesperson_voice_update; +mod status_sounds; mod synthesis; mod tts; @@ -54,6 +55,11 @@ pub use pocket::{ }; #[cfg(target_os = "macos")] pub use siri::SiriTts; +pub use status_sounds::ConversationStatus; +pub use status_sounds::ManagedStatusSoundRuntime; +pub use status_sounds::StatusSoundMode; +pub use status_sounds::StatusSoundRuntime; +pub use status_sounds::StatusSoundSettings; pub use synthesis::{synthesize_pcm16_wav, WavSynthesis, WavSynthesisError, WavSynthesisErrorKind}; pub use tts::{ OpenAiTts, PocketTtsBackend, StreamingTextChunk, StreamingTextChunks, StreamingTtsText, diff --git a/src-tauri/crates/berd-voice/src/macos_audio_output.rs b/src-tauri/crates/berd-voice/src/macos_audio_output.rs index f48a6eeaa..57a2bf73a 100644 --- a/src-tauri/crates/berd-voice/src/macos_audio_output.rs +++ b/src-tauri/crates/berd-voice/src/macos_audio_output.rs @@ -1,10 +1,17 @@ -//! Safe ownership wrapper for the shared macOS AVAudioUnitTimePitch PCM player. +//! Safe wrappers for the shared macOS audio FFI boundary. use std::ffi::{c_char, c_void, CStr}; use crate::PcmAudioOutput; unsafe extern "C" { + fn berd_audio_file_load_mono_pcm( + path: *const c_char, + sample_rate_out: *mut u32, + frame_count_out: *mut u32, + error_out: *mut *mut c_char, + ) -> *mut f32; + fn berd_audio_free_samples(samples: *mut f32); fn berd_pocket_audio_player_create( sample_rate: u32, rate: f32, @@ -25,6 +32,30 @@ unsafe extern "C" { fn berd_siri_tts_free_string(value: *mut c_char); } +pub(crate) fn load_mono_audio_file(path: &str) -> Result<(u32, Vec), String> { + let path = std::ffi::CString::new(path).map_err(|_| "audio path contains NUL".to_string())?; + let mut sample_rate = 0; + let mut frame_count = 0; + let mut error = std::ptr::null_mut(); + // SAFETY: The bridge copies the path and returns an owned allocation with + // the reported frame count, released below by its paired free function. + let raw = unsafe { + berd_audio_file_load_mono_pcm( + path.as_ptr(), + &mut sample_rate, + &mut frame_count, + &mut error, + ) + }; + if raw.is_null() { + return Err(take_error(error, "Could not decode audio file")); + } + // SAFETY: A successful bridge call returns exactly `frame_count` initialized samples. + let samples = unsafe { std::slice::from_raw_parts(raw, frame_count as usize) }.to_vec(); + unsafe { berd_audio_free_samples(raw) }; + Ok((sample_rate, samples)) +} + pub struct PocketAudioPlayer { raw: *mut c_void, delivery_safety_frames: u64, diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index 481c8806c..59d17cdba 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -46,6 +46,7 @@ use berd_voice::spokesperson_voice_update::{ validate_voice_update_settings, VoiceBarrierAction, VoiceUpdateAction, VoiceUpdatePurpose, VoiceUpdateQueue, VoiceUpdateRequest, VoiceUpdateTransaction, }; +use berd_voice::StatusSoundRuntime; use berd_voice::{ estimated_spoken_through_utf8, local_assets::{ @@ -65,7 +66,7 @@ use session_audio::{ AUDIO_CANCELLED, }; -const SESSION_PROTOCOL_VERSION: u32 = 4; +const SESSION_PROTOCOL_VERSION: u32 = 5; const INPUT_FRAME_MARKER: u8 = 3; const MAX_LINE_BYTES: usize = 1024 * 1024; const FRAME_MAGIC: [u8; 2] = *b"BV"; @@ -1235,6 +1236,20 @@ fn run_management_command(command: ManagementCommand) -> Result<(), ManagementFa } } +fn standard_session_status_cues_suppressed( + core: &SessionCore, + assistant_output_active: bool, +) -> bool { + core.user_speaking() || assistant_output_active +} + +fn expert_session_status_cues_suppressed( + turn_gate: &ExpertTurnGate, + assistant_output_active: bool, +) -> bool { + turn_gate.lifecycle.user_speaking() || assistant_output_active +} + fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String> { let (control_tx, control_rx) = mpsc::channel(); let (pcm_tx, pcm_rx) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY); @@ -1259,6 +1274,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let mut processed_pcm = 0_u64; let mut held: Option = None; let mut active: Option = None; + let mut status_sound_runtime = StatusSoundRuntime::default(); loop { if let Some(events) = input_events.as_mut() { @@ -1328,6 +1344,11 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String }, )?; } + let conversation_active = standard_session_status_cues_suppressed(&core, active.is_some()); + let status_sound_result = status_sound_runtime.poll(conversation_active); + if let Err(message) = status_sound_result { + eprintln!("status sound playback disabled: {message}"); + } let Some(input) = receive_session_input( &control_rx, @@ -1424,6 +1445,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String Input::Request(SessionRequest::Hello { id, input_during_tts, + status_sound_output_device, }) => { if initialized { write_message( @@ -1474,6 +1496,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String input_runtime = Some(runtime); input_events = Some(events); initialized = true; + status_sound_runtime.set_output_device(status_sound_output_device); let input_policy = InputDuringTtsSlot::new(input_during_tts); let session = VoiceSessionSnapshot { tts: slot.snapshot()?, @@ -1500,6 +1523,21 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String )?; return Ok(()); } + Input::Request(SessionRequest::SetConversationStatus { + id, + status, + settings, + }) => { + status_sound_runtime.update(status, settings); + write_message( + &mut writer, + &SessionMessage::ConversationStatusApplied { + id, + status, + settings, + }, + )?; + } Input::Request(SessionRequest::SetInputMuted { id, active: muted }) => { handle_input_muted( id, @@ -2445,6 +2483,7 @@ fn run_expert_spokesperson_session( let mut input_muted = false; let mut queued_tts_settings = VoiceUpdateQueue::::default(); let mut pending_voice_update: Option = None; + let mut status_sound_runtime = StatusSoundRuntime::default(); loop { if initialized { @@ -3300,6 +3339,12 @@ fn run_expert_spokesperson_session( )?; } } + let conversation_active = + expert_session_status_cues_suppressed(&turn_gate, active.is_some()); + let status_sound_result = status_sound_runtime.poll(conversation_active); + if let Err(message) = status_sound_result { + eprintln!("status sound playback disabled: {message}"); + } let Some(input) = receive_session_input( &control_rx, @@ -3393,6 +3438,7 @@ fn run_expert_spokesperson_session( Input::Request(SessionRequest::Hello { id, input_during_tts, + status_sound_output_device, }) => { if initialized { write_protocol_fatal( @@ -3443,6 +3489,7 @@ fn run_expert_spokesperson_session( session_tts = Some(tts); input_during_tts_slot = Some(input_policy); initialized = true; + status_sound_runtime.set_output_device(status_sound_output_device); turn_gate.lifecycle.session_started(Instant::now()); write_message( &mut writer, @@ -3821,6 +3868,21 @@ fn run_expert_spokesperson_session( cancel_live_playback(&mut active); } } + Input::Request(SessionRequest::SetConversationStatus { + id, + status, + settings, + }) => { + status_sound_runtime.update(status, settings); + write_message( + &mut writer, + &SessionMessage::ConversationStatusApplied { + id, + status, + settings, + }, + )?; + } Input::Request(SessionRequest::SetInputMuted { id, active: muted }) => { if pending_voice_update.is_some() { rollback_spokesperson_voice_update( @@ -6495,6 +6557,7 @@ fn validate_request(request: SessionRequest) -> Result { let id = match &request { SessionRequest::Hello { id, .. } | SessionRequest::SetInputMuted { id, .. } + | SessionRequest::SetConversationStatus { id, .. } | SessionRequest::SetTtsSettings { id, .. } | SessionRequest::SetInputDuringTts { id, .. } | SessionRequest::ResetInput { id } @@ -6521,6 +6584,9 @@ fn validate_request(request: SessionRequest) -> Result { return Err("request id must be positive".into()); } match &request { + SessionRequest::SetConversationStatus { settings, .. } => { + settings.validate().map_err(str::to_string)?; + } SessionRequest::PrepareSpeak { text, .. } if text.len() > MAX_SPEAK_TEXT_BYTES => { return Err("speak text exceeds 16 KiB".into()) } @@ -6752,6 +6818,25 @@ mod tests { use std::os::unix::net::UnixStream; use std::sync::Mutex; + #[test] + fn status_cues_ignore_pending_recognition_but_suppress_actual_audio() { + let mut core = SessionCore::default(); + core.set_recognition_pending(true); + assert!(!standard_session_status_cues_suppressed(&core, false)); + assert!(standard_session_status_cues_suppressed(&core, true)); + core.set_user_speaking(true); + assert!(standard_session_status_cues_suppressed(&core, false)); + core.set_user_speaking(false); + assert!(!standard_session_status_cues_suppressed(&core, false)); + + let mut gate = ExpertTurnGate::default(); + gate.begin_user_speaking("pending-transcript".into()); + assert!(expert_session_status_cues_suppressed(&gate, false)); + gate.finish_user_speaking(); + assert!(gate.input_blocks_output()); + assert!(!expert_session_status_cues_suppressed(&gate, false)); + assert!(expert_session_status_cues_suppressed(&gate, true)); + } fn synthesis_config(tts: SynthesisTtsConfig, output: PathBuf) -> SynthesisConfig { SynthesisConfig { tts, @@ -9511,6 +9596,35 @@ mod tests { assert!(control_receiver.try_recv().is_err()); } + #[test] + fn status_sound_requests_validate_volume() { + for volume in [-1.0, 2.0, f32::NAN, f32::INFINITY] { + let request = SessionRequest::SetConversationStatus { + id: 1, + status: berd_voice::ConversationStatus::Working, + settings: berd_voice::StatusSoundSettings { + volume, + ..Default::default() + }, + }; + assert_eq!( + validate_request(request).unwrap_err(), + "status sound volume must be finite and between 0 and 1" + ); + } + for volume in [0.0, 0.8, 1.0] { + assert!(validate_request(SessionRequest::SetConversationStatus { + id: 1, + status: berd_voice::ConversationStatus::Working, + settings: berd_voice::StatusSoundSettings { + volume, + ..Default::default() + }, + }) + .is_ok()); + } + } + #[test] fn input_policy_update_requires_a_positive_expected_revision() { let request = SessionRequest::SetInputDuringTts { diff --git a/src-tauri/crates/berd-voice/src/protocol.rs b/src-tauri/crates/berd-voice/src/protocol.rs index 125f8363d..c2103889c 100644 --- a/src-tauri/crates/berd-voice/src/protocol.rs +++ b/src-tauri/crates/berd-voice/src/protocol.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; use crate::{ input::{InputDuringTtsPolicy, InputDuringTtsSnapshot}, openai_realtime_protocol::RealtimeExpertDeliveryEvent, - TtsConfigurationSnapshot, TtsSettings, + ConversationStatus, StatusSoundSettings, TtsConfigurationSnapshot, TtsSettings, }; #[derive(Clone, Debug, Deserialize, PartialEq)] @@ -12,6 +12,13 @@ pub enum SessionRequest { Hello { id: u64, input_during_tts: InputDuringTtsPolicy, + #[serde(default)] + status_sound_output_device: Option, + }, + SetConversationStatus { + id: u64, + status: ConversationStatus, + settings: StatusSoundSettings, }, SetPaused { active: bool, @@ -192,6 +199,11 @@ pub enum SessionMessage { protocol: u32, session: VoiceSessionSnapshot, }, + ConversationStatusApplied { + id: u64, + status: ConversationStatus, + settings: StatusSoundSettings, + }, TtsSettingsResult { id: u64, outcome: TtsSettingsOutcome, @@ -341,7 +353,7 @@ mod tests { assert_eq!( serde_json::to_string(&SessionMessage::Ready { id: 4, - protocol: 4, + protocol: 5, session: VoiceSessionSnapshot { tts: TtsConfigurationSnapshot { revision: 1, @@ -358,7 +370,7 @@ mod tests { }, }) .unwrap(), - r#"{"type":"ready","id":4,"protocol":4,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"# + r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"# ); assert_eq!( serde_json::from_str::( diff --git a/src-tauri/crates/berd-voice/src/realtime_host_lifecycle.rs b/src-tauri/crates/berd-voice/src/realtime_host_lifecycle.rs index 3085d3c85..49e25ed47 100644 --- a/src-tauri/crates/berd-voice/src/realtime_host_lifecycle.rs +++ b/src-tauri/crates/berd-voice/src/realtime_host_lifecycle.rs @@ -86,6 +86,10 @@ impl RealtimeHostLifecycle { self.activity.input_blocks_output() } + pub fn user_speaking(&self) -> bool { + self.activity.user_speaking + } + pub fn is_busy(&self, work: RealtimeHostWork) -> bool { self.activity.is_busy(work) } diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs new file mode 100644 index 000000000..935236172 --- /dev/null +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -0,0 +1,675 @@ +use std::{ + sync::{ + mpsc::{self, RecvTimeoutError, Sender}, + Arc, Mutex, + }, + thread, + time::{Duration, Instant}, +}; + +use serde::{Deserialize, Serialize}; + +pub const DEFAULT_STATUS_SOUND_VOLUME: f32 = 0.8; +pub const STATUS_SOUND_INTERVAL: Duration = Duration::from_secs(5); + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum StatusSoundMode { + Off, + WorkingAndWaiting, + #[default] + Working, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ConversationStatus { + Working, + Waiting, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct StatusSoundSettings { + pub mode: StatusSoundMode, + #[serde(default = "default_status_sound_volume")] + pub volume: f32, +} + +impl Default for StatusSoundSettings { + fn default() -> Self { + Self { + mode: StatusSoundMode::default(), + volume: DEFAULT_STATUS_SOUND_VOLUME, + } + } +} + +impl StatusSoundSettings { + pub fn validate(self) -> Result { + if !self.volume.is_finite() || !(0.0..=1.0).contains(&self.volume) { + return Err("status sound volume must be finite and between 0 and 1"); + } + Ok(self) + } +} + +const fn default_status_sound_volume() -> f32 { + DEFAULT_STATUS_SOUND_VOLUME +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct StatusSoundCue { + pub status: ConversationStatus, + pub volume: f32, +} + +/// Pure policy for deciding which cue, if any, a fixed-cadence runtime tick plays. +/// The host owns the timer and supplies whether conversation activity should suppress cues. +#[derive(Debug, Default)] +pub struct StatusSoundStateMachine { + current: Option<(ConversationStatus, StatusSoundSettings)>, +} + +impl StatusSoundStateMachine { + pub fn update(&mut self, status: ConversationStatus, settings: StatusSoundSettings) -> bool { + let next = (status, settings); + let changed = self.current != Some(next); + self.current = Some(next); + changed + } + + pub fn tick(&mut self, conversation_active: bool) -> Option { + let (status, settings) = self.current?; + let working_only = settings.mode == StatusSoundMode::Working; + let waiting = status == ConversationStatus::Waiting; + if settings.mode == StatusSoundMode::Off || conversation_active || (working_only && waiting) + { + return None; + } + Some(StatusSoundCue { + status, + volume: settings.volume, + }) + } +} + +/// Owns status cadence and cue playback for one running voice session. +/// Producers report semantic state; they never choose or play a sound. +pub struct StatusSoundRuntime { + machine: StatusSoundStateMachine, + next_tick: Option, + player: StatusSoundPlayer, + output_device: Option, + conversation_active: bool, + input_controls: Option, +} + +impl Default for StatusSoundRuntime { + fn default() -> Self { + Self { + machine: StatusSoundStateMachine::default(), + next_tick: None, + player: StatusSoundPlayer::new(), + output_device: None, + conversation_active: false, + input_controls: None, + } + } +} + +impl StatusSoundRuntime { + pub fn set_input_controls(&mut self, controls: Option) { + self.input_controls = controls; + } + + pub fn set_output_device(&mut self, output_device: Option) { + if self.output_device != output_device { + self.player.stop(); + self.output_device = output_device; + } + } + + pub fn update(&mut self, status: ConversationStatus, settings: StatusSoundSettings) { + let changed = self.machine.update(status, settings); + if changed { + self.player.stop(); + self.next_tick = Some(Instant::now()); + } + } + + pub fn stop(&mut self) { + self.player.stop(); + } + + pub fn poll(&mut self, conversation_active: bool) -> Result { + if conversation_active { + self.stop(); + } else if self.conversation_active { + self.next_tick = Some(Instant::now()); + } + self.conversation_active = conversation_active; + if conversation_active { + self.player.reap(); + return Ok(false); + } + self.player.reap(); + let now = Instant::now(); + if self.next_tick.is_some_and(|deadline| now >= deadline) { + self.next_tick = Some(now + STATUS_SOUND_INTERVAL); + if let Some(cue) = self.machine.tick(conversation_active) { + self.player.play( + cue, + self.output_device.as_deref(), + self.input_controls.as_ref(), + )?; + } + } + Ok(self.player.is_active()) + } +} + +enum StatusSoundCommand { + Update(ConversationStatus, StatusSoundSettings), + ConversationActive(bool), + OutputDevice(Option), + Shutdown, +} + +/// Thread-safe status-sound service for hosts that do not own a polling loop. +#[derive(Clone)] +pub struct ManagedStatusSoundRuntime { + inner: Arc, +} + +struct StatusSoundWorker { + commands: Sender, + worker: Mutex>>, +} + +impl ManagedStatusSoundRuntime { + pub fn spawn(output_device: Option) -> Result { + Self::spawn_with_input_controls(output_device, None) + } + + pub fn spawn_with_input_controls( + output_device: Option, + input_controls: Option, + ) -> Result { + let (commands, receiver) = mpsc::channel(); + let worker = thread::Builder::new() + .name("berd-status-sounds".into()) + .spawn(move || { + let mut runtime = StatusSoundRuntime::default(); + let mut conversation_active = false; + runtime.set_output_device(output_device); + runtime.set_input_controls(input_controls); + loop { + match receiver.recv_timeout(Duration::from_millis(10)) { + Ok(StatusSoundCommand::Update(status, settings)) => { + runtime.update(status, settings); + } + Ok(StatusSoundCommand::ConversationActive(active)) => { + conversation_active = active; + } + Ok(StatusSoundCommand::OutputDevice(device)) => { + runtime.set_output_device(device); + } + Ok(StatusSoundCommand::Shutdown) | Err(RecvTimeoutError::Disconnected) => { + runtime.stop(); + break; + } + Err(RecvTimeoutError::Timeout) => {} + } + if let Err(message) = runtime.poll(conversation_active) { + eprintln!("status sound playback disabled: {message}"); + } + } + }) + .map_err(|error| format!("Could not start status sound runtime: {error}"))?; + Ok(Self { + inner: Arc::new(StatusSoundWorker { + commands, + worker: Mutex::new(Some(worker)), + }), + }) + } + + pub fn update( + &self, + status: ConversationStatus, + settings: StatusSoundSettings, + ) -> Result<(), String> { + settings.validate().map_err(str::to_string)?; + self.send(StatusSoundCommand::Update(status, settings)) + } + + pub fn set_conversation_active(&self, active: bool) -> Result<(), String> { + self.send(StatusSoundCommand::ConversationActive(active)) + } + + pub fn set_output_device(&self, output_device: Option) -> Result<(), String> { + self.send(StatusSoundCommand::OutputDevice(output_device)) + } + + fn send(&self, command: StatusSoundCommand) -> Result<(), String> { + self.inner + .commands + .send(command) + .map_err(|_| "Status sound runtime is unavailable".to_string()) + } + + pub fn finish(&self) -> Result<(), String> { + let _ = self.inner.commands.send(StatusSoundCommand::Shutdown); + let worker = self + .inner + .worker + .lock() + .map_err(|_| "Status sound worker join state is unavailable")? + .take(); + if let Some(worker) = worker { + let deadline = Instant::now() + Duration::from_secs(2); + while !worker.is_finished() { + if Instant::now() >= deadline { + reap_status_sound_worker(worker); + return Err("Status sound worker shutdown timed out".into()); + } + thread::sleep(Duration::from_millis(10)); + } + worker + .join() + .map_err(|_| "Status sound worker panicked".to_string())?; + } + Ok(()) + } +} + +impl Drop for StatusSoundWorker { + fn drop(&mut self) { + let _ = self.commands.send(StatusSoundCommand::Shutdown); + if let Ok(worker) = self.worker.get_mut() { + if let Some(worker) = worker.take() { + reap_status_sound_worker(worker); + } + } + } +} + +fn reap_status_sound_worker(worker: thread::JoinHandle<()>) { + let _ = thread::Builder::new() + .name("berd-status-sound-reaper".into()) + .spawn(move || { + let _ = worker.join(); + }); +} + +#[cfg(not(target_os = "macos"))] +struct StatusSoundPlayer; + +#[cfg(not(target_os = "macos"))] +impl StatusSoundPlayer { + fn new() -> Self { + Self + } + + fn play( + &mut self, + _cue: StatusSoundCue, + _output_device: Option<&str>, + _input_controls: Option<&crate::input::VoiceInputControls>, + ) -> Result<(), String> { + Err("status sound playback is only available on macOS".into()) + } + + fn reap(&mut self) {} + + fn is_active(&self) -> bool { + false + } + + fn stop(&mut self) {} +} + +#[cfg(target_os = "macos")] +const STATUS_SOUND_OUTPUT_TAIL: Duration = Duration::from_millis(100); + +#[cfg(target_os = "macos")] +struct ActiveStatusSound { + player: crate::macos_audio_output::PocketAudioPlayer, + output_tail_deadline: Option, + _input_activity: Option, +} + +#[cfg(target_os = "macos")] +struct StatusSoundPlayer { + working: Result, + waiting: Result, + active: Vec, +} + +#[cfg(target_os = "macos")] +impl StatusSoundPlayer { + fn new() -> Self { + Self { + working: load_system_sound("Pop"), + waiting: load_system_sound("Purr"), + active: Vec::new(), + } + } + + fn play( + &mut self, + cue: StatusSoundCue, + output_device: Option<&str>, + input_controls: Option<&crate::input::VoiceInputControls>, + ) -> Result<(), String> { + let asset = match cue.status { + ConversationStatus::Working => &self.working, + ConversationStatus::Waiting => &self.waiting, + } + .as_ref() + .map_err(Clone::clone)?; + let player = crate::macos_audio_output::PocketAudioPlayer::new( + asset.sample_rate, + 1.0, + output_device, + )?; + let samples = asset + .samples + .iter() + .map(|sample| sample * cue.volume) + .collect::>(); + let input_activity = input_controls + .map(|controls| { + controls.begin_assistant_activity( + 0.65, + crate::input::InputDuringTtsPolicy::SuppressInput, + ) + }) + .transpose()?; + player.enqueue(&samples)?; + self.active.push(ActiveStatusSound { + player, + output_tail_deadline: None, + _input_activity: input_activity, + }); + Ok(()) + } + + fn reap(&mut self) { + let now = Instant::now(); + self.active.retain_mut(|sound| { + if !sound.player.is_empty() { + sound.output_tail_deadline = None; + return true; + } + let deadline = sound + .output_tail_deadline + .get_or_insert(now + STATUS_SOUND_OUTPUT_TAIL); + now < *deadline + }); + } + + fn is_active(&self) -> bool { + !self.active.is_empty() + } + + fn stop(&mut self) { + for sound in self.active.drain(..) { + sound.player.stop(); + } + } +} + +#[cfg(target_os = "macos")] +struct StatusSoundAsset { + sample_rate: u32, + samples: Vec, +} + +#[cfg(target_os = "macos")] +fn load_system_sound(name: &str) -> Result { + let source = format!("/System/Library/Sounds/{name}.aiff"); + let (sample_rate, samples) = crate::macos_audio_output::load_mono_audio_file(&source) + .map_err(|error| format!("could not decode {name} status sound: {error}"))?; + Ok(StatusSoundAsset { + sample_rate, + samples, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn settings(mode: StatusSoundMode) -> StatusSoundSettings { + StatusSoundSettings { + mode, + ..StatusSoundSettings::default() + } + } + + fn assert_status(machine: &mut StatusSoundStateMachine, expected: ConversationStatus) { + let actual = machine.tick(false).map(|cue| cue.status); + assert_eq!(actual, Some(expected)); + } + + #[test] + fn managed_worker_finish_joins_and_is_idempotent_across_clones() { + let runtime = ManagedStatusSoundRuntime::spawn(None).unwrap(); + let other = runtime.clone(); + drop(runtime); + other.set_conversation_active(true).unwrap(); + other.finish().unwrap(); + assert!(other.inner.worker.lock().unwrap().is_none()); + assert!(other.set_conversation_active(false).is_err()); + other.finish().unwrap(); + } + + #[test] + fn last_owner_drop_stops_and_reaps_the_worker() { + let (commands, receiver) = mpsc::channel(); + let (stopped, completed) = mpsc::channel(); + let worker = thread::spawn(move || { + assert!(matches!(receiver.recv(), Ok(StatusSoundCommand::Shutdown))); + stopped.send(()).unwrap(); + }); + let runtime = ManagedStatusSoundRuntime { + inner: Arc::new(StatusSoundWorker { + commands, + worker: Mutex::new(Some(worker)), + }), + }; + drop(runtime); + completed.recv_timeout(Duration::from_secs(2)).unwrap(); + } + + #[test] + fn off_suppresses_both_statuses_and_round_trips_on_the_wire() { + let mode: StatusSoundMode = serde_json::from_str("\"off\"").unwrap(); + assert_eq!(mode, StatusSoundMode::Off); + assert_eq!(serde_json::to_string(&mode).unwrap(), "\"off\""); + let mut machine = StatusSoundStateMachine::default(); + for status in [ConversationStatus::Working, ConversationStatus::Waiting] { + machine.update(status, settings(mode)); + assert_eq!(machine.tick(false), None); + assert_eq!(machine.tick(true), None); + } + } + + #[test] + fn first_status_update_schedules_an_immediate_cue() { + let mut runtime = StatusSoundRuntime::default(); + let started_at = Instant::now(); + runtime.update( + ConversationStatus::Working, + settings(StatusSoundMode::Working), + ); + assert!(runtime.next_tick.unwrap() >= started_at); + assert!(runtime.next_tick.unwrap() <= Instant::now()); + runtime.update( + ConversationStatus::Waiting, + settings(StatusSoundMode::Working), + ); + assert!(!runtime.poll(false).unwrap()); + assert_eq!(runtime.machine.tick(false), None); + } + + #[test] + fn defaults_to_working_only() { + assert_eq!( + StatusSoundSettings::default(), + settings(StatusSoundMode::Working) + ); + } + + #[test] + fn working_and_waiting_repeats_both_statuses() { + let mut machine = StatusSoundStateMachine::default(); + let settings = settings(StatusSoundMode::WorkingAndWaiting); + machine.update(ConversationStatus::Working, settings); + assert_status(&mut machine, ConversationStatus::Working); + assert_status(&mut machine, ConversationStatus::Working); + machine.update(ConversationStatus::Waiting, settings); + assert_status(&mut machine, ConversationStatus::Waiting); + assert_status(&mut machine, ConversationStatus::Waiting); + } + + #[test] + fn working_only_repeats_working_and_stays_silent_while_waiting() { + let mut machine = StatusSoundStateMachine::default(); + let settings = settings(StatusSoundMode::Working); + machine.update(ConversationStatus::Working, settings); + assert_status(&mut machine, ConversationStatus::Working); + assert_status(&mut machine, ConversationStatus::Working); + machine.update(ConversationStatus::Waiting, settings); + assert_eq!(machine.tick(false), None); + assert_eq!(machine.tick(false), None); + } + + #[test] + fn conversation_audio_suppresses_without_changing_the_status() { + let mut machine = StatusSoundStateMachine::default(); + machine.update( + ConversationStatus::Working, + settings(StatusSoundMode::Working), + ); + assert_eq!(machine.tick(true), None); + assert!(machine.tick(false).is_some()); + } + + #[test] + fn no_status_event_means_no_startup_cue() { + assert_eq!(StatusSoundStateMachine::default().tick(false), None); + } + + #[test] + fn resuming_after_conversation_audio_schedules_cue_immediately() { + let mut runtime = StatusSoundRuntime::default(); + runtime.update( + ConversationStatus::Working, + settings(StatusSoundMode::Working), + ); + assert!(!runtime.poll(true).unwrap()); + runtime.next_tick = Some(Instant::now() + Duration::from_secs(60)); + let _ = runtime.poll(false); + assert!(runtime.next_tick.unwrap() < Instant::now() + STATUS_SOUND_INTERVAL); + } + + #[test] + fn session_volume_defaults_to_point_eight_and_accepts_an_override() { + assert_eq!(StatusSoundSettings::default().volume, 0.8); + let explicit = StatusSoundSettings { + mode: StatusSoundMode::Working, + volume: 0.25, + }; + assert_eq!(explicit.validate().unwrap().volume, 0.25); + } + + #[test] + fn rejects_invalid_session_volume() { + for volume in [f32::NAN, f32::INFINITY, -0.1, 1.1] { + assert!(StatusSoundSettings { + mode: StatusSoundMode::Working, + volume, + } + .validate() + .is_err()); + } + } + + #[test] + fn duplicate_updates_preserve_the_existing_cadence() { + let mut runtime = StatusSoundRuntime::default(); + let settings = settings(StatusSoundMode::WorkingAndWaiting); + runtime.update(ConversationStatus::Working, settings); + let deadline = runtime.next_tick; + runtime.update(ConversationStatus::Working, settings); + assert_eq!(runtime.next_tick, deadline); + } + + #[test] + fn changed_updates_restart_the_cadence() { + let mut runtime = StatusSoundRuntime::default(); + runtime.update( + ConversationStatus::Working, + settings(StatusSoundMode::WorkingAndWaiting), + ); + runtime.next_tick = Some(Instant::now() + Duration::from_secs(60)); + runtime.update( + ConversationStatus::Waiting, + settings(StatusSoundMode::WorkingAndWaiting), + ); + assert!(runtime.next_tick.unwrap() <= Instant::now()); + } + + #[cfg(target_os = "macos")] + #[test] + #[ignore = "opens the default CoreAudio output and plays the macOS Pop and Purr cues"] + fn macos_player_decodes_and_queues_both_status_cues() { + let mut player = StatusSoundPlayer::new(); + let controls = crate::input::VoiceInputControls::default(); + for status in [ConversationStatus::Working, ConversationStatus::Waiting] { + player + .play( + StatusSoundCue { + status, + volume: DEFAULT_STATUS_SOUND_VOLUME, + }, + None, + Some(&controls), + ) + .unwrap(); + } + assert_eq!(player.active.len(), 2); + assert!(controls.is_muted()); + player.stop(); + assert!(!controls.is_muted()); + } + + #[cfg(target_os = "macos")] + #[test] + #[ignore = "opens CoreAudio output and plays a status cue"] + fn macos_cue_suppresses_input_until_audio_and_tail_drain() { + let controls = crate::input::VoiceInputControls::default(); + let mut player = StatusSoundPlayer::new(); + player + .play( + StatusSoundCue { + status: ConversationStatus::Working, + volume: DEFAULT_STATUS_SOUND_VOLUME, + }, + None, + Some(&controls), + ) + .unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while player.is_active() { + assert!(controls.is_muted()); + assert!(Instant::now() < deadline, "status cue did not drain"); + player.reap(); + thread::sleep(Duration::from_millis(10)); + } + assert!(!controls.is_muted()); + } +} diff --git a/src-tauri/crates/berd-voice/tests/session_protocol.rs b/src-tauri/crates/berd-voice/tests/session_protocol.rs index 3b1f66c21..5bf1b8fb8 100644 --- a/src-tauri/crates/berd-voice/tests/session_protocol.rs +++ b/src-tauri/crates/berd-voice/tests/session_protocol.rs @@ -2510,7 +2510,7 @@ fn siri_session_reaches_ready_without_openai_credentials() { stdin.flush().unwrap(); let ready = receive(); assert_eq!(ready["type"], "ready"); - assert_eq!(ready["protocol"], 4); + assert_eq!(ready["protocol"], 5); assert_eq!(ready["session"]["tts"]["backend"], "siri"); assert_eq!(ready["session"]["tts"]["voice"], voice); assert_eq!(ready["session"]["tts"]["language"], language); @@ -2519,6 +2519,25 @@ fn siri_session_reaches_ready_without_openai_credentials() { ready["session"]["input_during_tts"], json!({"revision":1,"policy":"allow_barge_in"}) ); + write_session_json( + &mut stdin, + &json!({ + "type":"set_conversation_status", + "id":19, + "status":"working", + "settings":{"mode":"working"} + }), + ); + stdin.flush().unwrap(); + assert_eq!( + receive(), + json!({ + "type":"conversation_status_applied", + "id":19, + "status":"working", + "settings":{"mode":"working"} + }) + ); write_session_json( &mut stdin, &json!({ diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index b20ef1d9f..90ed055e1 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -77,6 +77,17 @@ pub struct MicrophoneMuteRequest { renderer_epoch: u64, } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StatusSoundUpdateRequest { + session_id: String, + expected_revision: u64, + status: berd_voice::ConversationStatus, + settings: berd_voice::StatusSoundSettings, + renderer_id: String, + renderer_epoch: u64, +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct AssistantSpeakingRequest { @@ -242,6 +253,9 @@ struct Runtime { controls_visibility_generation: u64, controls_window_revision: Option, native_microphone_mute_control: bool, + status_sounds: Option, + status_sound_user_speaking: bool, + status_sound_playbacks: usize, admission: Option>, voice_input_quarantined: bool, } @@ -529,6 +543,7 @@ type StopSnapshot = ( Option, u64, Option, + Option, Option<(RuntimeOwner, String)>, ); @@ -554,6 +569,32 @@ pub struct NativeVoiceState { #[must_use = "assistant speech policy ends when the guard is dropped"] pub(crate) struct AssistantSpeechGuard { _activity: Option, + runtime: Arc>, +} + +impl Runtime { + fn status_sound_suppressed(&self) -> bool { + self.status_sound_user_speaking || self.status_sound_playbacks > 0 + } + + fn publish_status_sound_suppression(&self) { + if let Some(status_sounds) = self.status_sounds.as_ref() { + if let Err(error) = + status_sounds.set_conversation_active(self.status_sound_suppressed()) + { + log::warn!("Could not update status sound activity: {error}"); + } + } + } +} + +impl Drop for AssistantSpeechGuard { + fn drop(&mut self) { + if let Ok(mut runtime) = self.runtime.lock() { + runtime.status_sound_playbacks -= 1; + runtime.publish_status_sound_suppression(); + } + } } impl NativeVoiceState { @@ -778,8 +819,13 @@ impl NativeVoiceState { .input_controls .begin_assistant_activity(sensitivity.vad_threshold(), input_during_tts) .ok(); + if let Ok(mut runtime) = self.runtime.lock() { + runtime.status_sound_playbacks += 1; + runtime.publish_status_sound_suppression(); + } AssistantSpeechGuard { _activity: activity, + runtime: Arc::clone(&self.runtime), } } @@ -997,32 +1043,6 @@ impl NativeVoiceState { Ok(()) } - fn assistant_activity_target( - &self, - caller_window_label: &str, - session_id: &str, - expected_revision: u64, - ) -> Result, String> { - let runtime = self - .runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())?; - if runtime.session_id.as_deref() != Some(session_id) - || runtime.revision != expected_revision - { - return Ok(None); - } - let owner_window_label = runtime - .owner - .as_ref() - .map(|owner| owner.window_label.clone()) - .ok_or_else(|| "The native voice conversation has no owning window.".to_string())?; - if owner_window_label != caller_window_label { - return Err("Only the voice conversation owner can report assistant activity.".into()); - } - Ok(Some((owner_window_label, runtime.revision))) - } - fn set_assistant_speaking( &self, app: &AppHandle, @@ -1031,10 +1051,27 @@ impl NativeVoiceState { expected_revision: u64, speaking: bool, ) -> Result<(), String> { - let Some((owner_window_label, revision)) = - self.assistant_activity_target(caller_window_label, session_id, expected_revision)? - else { - return Ok(()); + let (owner_window_label, revision) = { + let runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(()); + } + let owner_window_label = runtime + .owner + .as_ref() + .map(|owner| owner.window_label.clone()) + .ok_or_else(|| "The native voice conversation has no owning window.".to_string())?; + if owner_window_label != caller_window_label { + return Err( + "Only the voice conversation owner can report assistant activity.".into(), + ); + } + (owner_window_label, runtime.revision) }; let event = NativeVoiceEvent::Activity { session_id: session_id.to_string(), @@ -1052,6 +1089,22 @@ impl NativeVoiceState { Ok(()) } + fn set_status_sound_input_activity( + &self, + session_id: &str, + revision: u64, + user_speaking: bool, + ) { + let Ok(mut runtime) = self.runtime.lock() else { + return; + }; + if runtime.session_id.as_deref() != Some(session_id) || runtime.revision != revision { + return; + } + runtime.status_sound_user_speaking = user_speaking; + runtime.publish_status_sound_suppression(); + } + fn take_stop_snapshot( &self, expected_lifecycle: Option<(&str, u64)>, @@ -1075,6 +1128,7 @@ impl NativeVoiceState { session_id, runtime.revision, runtime.pipeline.take(), + runtime.status_sounds.take(), owner.zip(owner_id), ))) } @@ -1591,6 +1645,20 @@ pub async fn start_native_voice_conversation( window_label: window_label.clone(), }); runtime.pipeline = pipeline.take(); + let output_device = super::pocket_voice::selected_output_device(); + let effective_output_device = + super::pocket_voice::effective_output_device_name(output_device.as_deref()); + let cue_input_controls = + super::pocket_voice::output_device_uses_speakers(effective_output_device.as_deref()) + .then(|| state.input_controls.clone()); + runtime.status_sounds = berd_voice::ManagedStatusSoundRuntime::spawn_with_input_controls( + output_device, + cue_input_controls, + ) + .map_err(|error| log::warn!("Status sounds unavailable: {error}")) + .ok(); + runtime.status_sound_user_speaking = false; + runtime.publish_status_sound_suppression(); runtime.admission = Some(Arc::new(BerdAdmissionCoordinator::default())); runtime.controls_ready = false; // Voice always starts from its owning session, where the in-session @@ -1707,6 +1775,7 @@ pub async fn start_native_voice_conversation( } berd_voice::input::VoiceInputEvent::SpeakingChanged(speaking) => { admission.set_user_speaking(speaking); + event_state.set_status_sound_input_activity(&session_id, revision, speaking); let event = NativeVoiceEvent::Activity { session_id: session_id.clone(), activity: if speaking { @@ -1790,6 +1859,11 @@ pub async fn start_native_voice_conversation( if let Some(admission) = current.admission.take() { admission.close(); } + if let Some(status_sounds) = current.status_sounds.take() { + if let Err(error) = status_sounds.finish() { + log::warn!("Status sound shutdown failed: {error}"); + } + } current.session_id = None; current.lifecycle_id = None; current.owner = None; @@ -1863,6 +1937,44 @@ pub async fn set_native_voice_microphone_muted( Ok(status(&app, &state).await) } +#[tauri::command] +pub fn update_native_voice_status_sounds( + state: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, + webview_window: WebviewWindow, + request: StatusSoundUpdateRequest, +) -> Result<(), String> { + capture.with_active_renderer( + webview_window.label(), + &request.renderer_id, + request.renderer_epoch, + || { + let runtime = state + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(request.session_id.as_str()) + || runtime.revision != request.expected_revision + { + return Ok(()); + } + if runtime + .owner + .as_ref() + .map(|owner| owner.window_label.as_str()) + != Some(webview_window.label()) + { + return Err("Only the voice conversation owner can update status sounds.".into()); + } + runtime + .status_sounds + .as_ref() + .ok_or_else(|| "Status sound runtime is unavailable".to_string())? + .update(request.status, request.settings) + }, + ) +} + #[tauri::command] pub fn set_native_voice_assistant_speaking( app: AppHandle, @@ -2191,11 +2303,16 @@ impl NativeVoiceState { &self, expected_lifecycle: Option<(&str, u64)>, ) -> Result, String> { - let Some((session_id, revision, pipeline, owner)) = + let Some((session_id, revision, pipeline, status_sounds, owner)) = self.take_stop_snapshot(expected_lifecycle)? else { return Ok(None); }; + if let Some(status_sounds) = status_sounds { + if let Err(error) = status_sounds.finish() { + log::warn!("Status sound shutdown failed: {error}"); + } + } // Keep the lifecycle current through the bounded shutdown window so a // cooperative worker can flush its final utterance durably. A worker // that misses the deadline is quarantined; its revision-bound late @@ -3255,6 +3372,40 @@ mod tests { assert!(!caller_owns_target("voice-buddy", None, true,)); } + #[test] + fn status_sound_suppression_ends_between_speech_segments() { + let state = NativeVoiceState::default(); + for _ in 0..2 { + let speech = state.begin_assistant_speech( + InterruptionSensitivity::Balanced, + berd_voice::input::InputDuringTtsPolicy::AllowBargeIn, + ); + assert!(state.runtime.lock().unwrap().status_sound_suppressed()); + drop(speech); + assert!(!state.runtime.lock().unwrap().status_sound_suppressed()); + } + } + + #[test] + fn status_sounds_wait_for_all_audio_to_stop() { + let state = NativeVoiceState::default(); + let first = state.begin_assistant_speech( + InterruptionSensitivity::Balanced, + berd_voice::input::InputDuringTtsPolicy::AllowBargeIn, + ); + let second = state.begin_assistant_speech( + InterruptionSensitivity::Balanced, + berd_voice::input::InputDuringTtsPolicy::AllowBargeIn, + ); + drop(first); + assert!(state.runtime.lock().unwrap().status_sound_suppressed()); + state.runtime.lock().unwrap().status_sound_user_speaking = true; + drop(second); + assert!(state.runtime.lock().unwrap().status_sound_suppressed()); + state.runtime.lock().unwrap().status_sound_user_speaking = false; + assert!(!state.runtime.lock().unwrap().status_sound_suppressed()); + } + #[test] fn assistant_suppression_uses_the_shared_input_controls() { let state = NativeVoiceState::default(); @@ -3297,43 +3448,6 @@ mod tests { assert!(!state.input_controls.is_muted()); } - #[test] - fn assistant_activity_is_bound_to_the_exact_voice_lifecycle() { - let state = NativeVoiceState::default(); - { - let mut runtime = state.runtime.lock().expect("lock native runtime"); - runtime.session_id = Some("session-1".to_string()); - runtime.revision = 7; - runtime.owner = Some(RuntimeOwner { - window_label: "main".to_string(), - }); - } - - assert_eq!( - state - .assistant_activity_target("main", "session-1", 7) - .expect("current activity target"), - Some(("main".to_string(), 7)), - ); - assert_eq!( - state - .assistant_activity_target("main", "session-1", 6) - .expect("stale activity is ignored"), - None, - ); - assert!(state - .assistant_activity_target("session:other", "session-1", 7) - .is_err()); - - state.runtime.lock().expect("lock native runtime").revision = 8; - assert_eq!( - state - .assistant_activity_target("main", "session-1", 7) - .expect("prior lifecycle activity is ignored after restart"), - None, - ); - } - #[test] fn stale_controls_watchdog_cannot_take_a_restarted_voice_lifecycle() { let state = NativeVoiceState::default(); diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index f179adf3d..294e5aae3 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -33,6 +33,7 @@ pub struct OpenAiRealtimeRuntimeState { struct NativeRealtimeRuntime { owner_window: String, runtime: Arc, + status_sounds: Option, protocol: RealtimeExpertSpokespersonSession, semantic_revision: Arc, } @@ -128,11 +129,24 @@ pub fn start_openai_realtime_spokesperson_runtime( let semantic_revision = Arc::new(AtomicU64::new(0)); let event_window = webview_window.clone(); let event_session_id = session_id.clone(); + let status_sounds = + berd_voice::ManagedStatusSoundRuntime::spawn(super::pocket_voice::selected_output_device()) + .map_err(|error| log::warn!("Status sounds unavailable: {error}")) + .ok(); + let runtime_status_sounds = status_sounds.clone(); + let mut status_sound_activity = RealtimeStatusSoundActivity::default(); let runtime = Arc::new(ManagedRealtimeHost::spawn( config, Arc::clone(&semantic_revision), create_native_realtime_output, - move |event| emit_runtime_provider_event(&event_window, &event_session_id, event), + move |event| { + update_realtime_status_sound_activity( + runtime_status_sounds.as_ref(), + &event, + &mut status_sound_activity, + ); + emit_runtime_provider_event(&event_window, &event_session_id, event) + }, )?); log::info!( "Starting Expert-Spokesperson session {session_id} with execution_path=berd_voice_in_process transport=websocket playback=native_pcm" @@ -142,6 +156,7 @@ pub fn start_openai_realtime_spokesperson_runtime( NativeRealtimeRuntime { owner_window: webview_window.label().into(), runtime, + status_sounds, protocol: RealtimeExpertSpokespersonSession::new(initial_cursor, call_id), semantic_revision, }, @@ -159,6 +174,29 @@ fn ensure_native_realtime_playback_supported() -> Result<(), String> { Err("Native OpenAI Realtime playback is not supported on this platform".into()) } +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RealtimeStatusSoundUpdate { + status: berd_voice::ConversationStatus, + settings: berd_voice::StatusSoundSettings, +} + +#[tauri::command] +pub fn update_openai_realtime_status_sounds( + state: State<'_, OpenAiRealtimeRuntimeState>, + webview_window: WebviewWindow, + session_id: String, + update: RealtimeStatusSoundUpdate, +) -> Result<(), String> { + with_runtime_entry(state, session_id, webview_window.label(), |entry| { + entry + .status_sounds + .as_ref() + .ok_or_else(|| "Status sound runtime is unavailable".to_string())? + .update(update.status, update.settings) + }) +} + #[tauri::command] pub fn send_openai_realtime_spokesperson_runtime_event( state: State<'_, OpenAiRealtimeRuntimeState>, @@ -208,6 +246,11 @@ pub async fn stop_openai_realtime_spokesperson_runtime( .get(&session_id) .map(|entry| { ensure_runtime_owner(&entry.owner_window, webview_window.label())?; + if let Some(status_sounds) = entry.status_sounds.as_ref() { + if let Err(error) = status_sounds.finish() { + log::warn!("Status sound shutdown failed: {error}"); + } + } Ok::<_, String>(Arc::clone(&entry.runtime)) }) .transpose()?; @@ -237,6 +280,11 @@ pub async fn release_openai_realtime_spokesperson_runtime( sessions.remove(&session_id) }; if let Some(entry) = entry { + if let Some(status_sounds) = entry.status_sounds.as_ref() { + if let Err(error) = status_sounds.finish() { + log::warn!("Status sound shutdown failed: {error}"); + } + } tauri::async_runtime::spawn_blocking(move || entry.runtime.finish()) .await .map_err(|error| format!("OpenAI Realtime runtime release task failed: {error}"))??; @@ -256,7 +304,14 @@ pub fn handle_owner_window_destroyed(app: &AppHandle, window_label: &str) { owned_session_ids .into_iter() .filter_map(|session_id| sessions.remove(&session_id)) - .map(|entry| entry.runtime) + .map(|entry| { + if let Some(status_sounds) = entry.status_sounds.as_ref() { + if let Err(error) = status_sounds.finish() { + log::warn!("Status sound shutdown failed: {error}"); + } + } + entry.runtime + }) .collect::>() } Err(_) => { @@ -275,6 +330,24 @@ pub fn handle_owner_window_destroyed(app: &AppHandle, window_label: &str) { } } +fn with_runtime_entry( + state: State<'_, OpenAiRealtimeRuntimeState>, + session_id: String, + owner_window: &str, + operation: impl FnOnce(&NativeRealtimeRuntime) -> Result, +) -> Result { + let session_id = non_empty_session_id(session_id)?; + let sessions = state + .sessions + .lock() + .map_err(|_| "OpenAI Realtime runtime state is unavailable".to_string())?; + let entry = sessions + .get(&session_id) + .ok_or("OpenAI Realtime runtime session is not active")?; + ensure_runtime_owner(&entry.owner_window, owner_window)?; + operation(entry) +} + fn with_runtime( state: State<'_, OpenAiRealtimeRuntimeState>, session_id: String, @@ -344,6 +417,76 @@ pub async fn update_openai_realtime_spokesperson_settings( .map_err(|error| format!("Spokesperson settings task failed: {error}"))? } +#[derive(Default)] +struct RealtimeStatusSoundActivity { + speaking_item_ids: std::collections::HashSet, + transcription_item_ids: std::collections::HashSet, + playback_active: bool, +} + +impl RealtimeStatusSoundActivity { + fn update(&mut self, event: &serde_json::Value) -> Option { + let item_id = || { + event + .get("item_id") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + }; + match event.get("type").and_then(serde_json::Value::as_str) { + Some("input_audio_buffer.speech_started") => { + if let Some(item_id) = item_id() { + self.speaking_item_ids.insert(item_id.clone()); + self.transcription_item_ids.insert(item_id); + } + } + Some("input_audio_buffer.speech_stopped") => { + if let Some(item_id) = item_id() { + self.speaking_item_ids.remove(&item_id); + } + } + Some( + "conversation.item.input_audio_transcription.completed" + | "conversation.item.input_audio_transcription.failed", + ) => { + if let Some(item_id) = item_id() { + self.speaking_item_ids.remove(&item_id); + self.transcription_item_ids.remove(&item_id); + } + } + Some("input_audio_buffer.cleared") => { + self.speaking_item_ids.clear(); + self.transcription_item_ids.clear(); + } + Some("output_audio_buffer.started") => self.playback_active = true, + Some("output_audio_buffer.stopped" | "output_audio_buffer.cleared") => { + self.playback_active = false; + } + _ => return None, + } + Some( + !self.speaking_item_ids.is_empty() + || !self.transcription_item_ids.is_empty() + || self.playback_active, + ) + } +} + +fn update_realtime_status_sound_activity( + status_sounds: Option<&berd_voice::ManagedStatusSoundRuntime>, + event: &serde_json::Value, + activity: &mut RealtimeStatusSoundActivity, +) { + let Some(conversation_active) = activity.update(event) else { + return; + }; + let Some(status_sounds) = status_sounds else { + return; + }; + if let Err(error) = status_sounds.set_conversation_active(conversation_active) { + log::warn!("Could not update Realtime status sound activity: {error}"); + } +} + fn emit_runtime_provider_event( window: &WebviewWindow, session_id: &str, @@ -714,6 +857,7 @@ fn client_secret_value(value: &serde_json::Value) -> Option<&str> { mod tests { use super::{ ensure_runtime_owner, parse_client_secret, realtime_transcription_client_secret_request, + RealtimeStatusSoundActivity, }; use serde_json::json; @@ -728,6 +872,67 @@ mod tests { } } + #[test] + fn realtime_status_sound_activity_aggregates_input_and_output() { + let mut activity = RealtimeStatusSoundActivity::default(); + + assert_eq!( + activity.update( + &json!({ "type": "input_audio_buffer.speech_started", "item_id": "user-1" }) + ), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "output_audio_buffer.started" })), + Some(true) + ); + assert_eq!( + activity.update( + &json!({ "type": "input_audio_buffer.speech_stopped", "item_id": "user-1" }) + ), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "output_audio_buffer.stopped" })), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "conversation.item.input_audio_transcription.completed", "item_id": "user-1" })), + Some(false) + ); + assert_eq!(activity.update(&json!({ "type": "response.done" })), None); + } + + #[test] + fn realtime_status_sound_activity_releases_abandoned_input_when_cleared() { + let mut activity = RealtimeStatusSoundActivity::default(); + + assert_eq!( + activity.update( + &json!({ "type": "input_audio_buffer.speech_started", "item_id": "user-1" }) + ), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "input_audio_buffer.cleared" })), + Some(false) + ); + } + + #[test] + fn realtime_status_sound_activity_treats_cleared_output_as_inactive() { + let mut activity = RealtimeStatusSoundActivity::default(); + + assert_eq!( + activity.update(&json!({ "type": "output_audio_buffer.started" })), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "output_audio_buffer.cleared" })), + Some(false) + ); + } + #[test] fn parses_supported_client_secret_shapes() { assert_eq!( diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index b9e1fbab9..7ac82dc90 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -422,7 +422,7 @@ pub(crate) fn effective_output_device_name(configured: Option<&str>) -> Option) -> Option { +pub(crate) fn effective_output_device_name(configured: Option<&str>) -> Option { configured.map(ToOwned::to_owned) } diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 9187d5239..79cdcf904 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -612,6 +612,7 @@ pub fn run() { commands::openai_realtime::stop_openai_realtime_spokesperson_runtime, commands::openai_realtime::release_openai_realtime_spokesperson_runtime, commands::openai_realtime::update_openai_realtime_spokesperson_settings, + commands::openai_realtime::update_openai_realtime_status_sounds, commands::openai_realtime::create_openai_realtime_expert_instructions, commands::openai_realtime::create_openai_realtime_transcript_seed, commands::openai_realtime::deliver_openai_realtime_expert_message, @@ -713,6 +714,7 @@ pub fn run() { commands::native_voice::cancel_native_voice_assistant_speech, commands::native_voice::set_native_voice_microphone_muted, commands::native_voice::set_native_voice_assistant_speaking, + commands::native_voice::update_native_voice_status_sounds, commands::native_voice::drain_native_voice_conversation_transcripts, commands::native_voice::acknowledge_native_voice_conversation_transcript, commands::native_voice::reject_native_voice_conversation_transcript, diff --git a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts index 1ce646783..b5de9856a 100644 --- a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts +++ b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts @@ -173,6 +173,21 @@ describe("acpNotificationHandler", () => { ).not.toHaveBeenCalled(); }); + it("does not mark replayed tool calls as live run activity", async () => { + markSessionReplayLoading(); + await handleSessionNotification({ + sessionId: "acp-session", + update: { + sessionUpdate: "tool_call", + toolCallId: "old-tool", + title: "shell", + }, + } as never); + expect( + useChatStore.getState().getSessionRuntime("acp-session").hasToolCallInRun, + ).toBe(false); + }); + it("keeps tool calls that arrive before the first text chunk on the pending assistant message", async () => { registerPreparedSession("acp-session", "goose", "/Users/aharvard"); setActiveMessageId("acp-session", "assistant-1"); @@ -186,6 +201,10 @@ describe("acpNotificationHandler", () => { }, } as never); + expect( + useChatStore.getState().getSessionRuntime("acp-session").hasToolCallInRun, + ).toBe(true); + await handleSessionNotification({ sessionId: "acp-session", update: { diff --git a/src/features/chat/acp/acpNotificationHandler.ts b/src/features/chat/acp/acpNotificationHandler.ts index e2da1f650..b860a4c02 100644 --- a/src/features/chat/acp/acpNotificationHandler.ts +++ b/src/features/chat/acp/acpNotificationHandler.ts @@ -641,6 +641,7 @@ function handleLive(sessionId: string, update: SessionUpdate): void { case "tool_call": { flushBufferedStreamingUpdatesForSession(sessionId); + store.markToolCallInRun(sessionId); const messageId = ensureLiveAssistantMessage(sessionId); const identity = getToolCallIdentity(update); const chainSummary = getToolChainSummary(update); diff --git a/src/features/chat/stores/__tests__/chatStore.test.ts b/src/features/chat/stores/__tests__/chatStore.test.ts index 796c0a23f..dacdd1b85 100644 --- a/src/features/chat/stores/__tests__/chatStore.test.ts +++ b/src/features/chat/stores/__tests__/chatStore.test.ts @@ -120,6 +120,26 @@ describe("chatStore", () => { expect(messagesBySession.s11).toHaveLength(1); }); + it("latches live tool activity once and clears it between runs", () => { + const store = useChatStore.getState(); + store.setChatState("tools", "thinking"); + store.setActiveRunId("tools", "run-1"); + expect(store.getSessionRuntime("tools").hasToolCallInRun).toBe(false); + store.markToolCallInRun("tools"); + const working = store.getSessionRuntime("tools"); + store.markToolCallInRun("tools"); + expect(store.getSessionRuntime("tools")).toBe(working); + store.setChatState("tools", "streaming"); + expect(store.getSessionRuntime("tools").hasToolCallInRun).toBe(true); + store.setChatState("tools", "idle"); + store.settleActiveRun("tools"); + store.setActiveRunId("tools", "run-2"); + expect(store.getSessionRuntime("tools").hasToolCallInRun).toBe(false); + store.markToolCallInRun("tools"); + store.setActiveRunId("tools", "run-3"); + expect(store.getSessionRuntime("tools").hasToolCallInRun).toBe(false); + }); + it("does not evict inactive messages for a running session", () => { useChatStore.getState().setActiveSession("running"); useChatStore diff --git a/src/features/chat/stores/chatStore.ts b/src/features/chat/stores/chatStore.ts index 9e6f2a588..33ffb53c8 100644 --- a/src/features/chat/stores/chatStore.ts +++ b/src/features/chat/stores/chatStore.ts @@ -455,6 +455,7 @@ interface ChatStoreActions { clearSettledStreamingMessage: (sessionId: string) => boolean; settleActiveRun: (sessionId: string) => void; setActiveRunId: (sessionId: string, runId: string | null) => void; + markToolCallInRun: (sessionId: string) => void; setRunCancellationPending: (sessionId: string, pending: boolean) => void; setPendingInterventionBoundary: ( sessionId: string, @@ -924,6 +925,7 @@ const createChatStore: StateCreator< const shouldClearStreamTracking = !isSessionRunning(current.chatState); if ( current.activeRunId === null && + !current.hasToolCallInRun && !current.isRunCancellationPending && (!shouldClearStreamTracking || (current.streamingMessageId === null && @@ -942,6 +944,7 @@ const createChatStore: StateCreator< ...(shouldClearStreamTracking ? { streamingMessageId: null, + hasToolCallInRun: false, pendingInterventionBoundary: null, } : {}), @@ -950,17 +953,41 @@ const createChatStore: StateCreator< }; }), + markToolCallInRun: (sessionId) => + set((state) => { + const current = + state.sessionStateById[sessionId] ?? createInitialSessionRuntime(); + if (current.hasToolCallInRun) return state; + return { + sessionStateById: { + ...state.sessionStateById, + [sessionId]: { ...current, hasToolCallInRun: true }, + }, + }; + }), + setActiveRunId: (sessionId, activeRunId) => - set((state) => ({ - sessionStateById: { - ...state.sessionStateById, - [sessionId]: { - ...(state.sessionStateById[sessionId] ?? - createInitialSessionRuntime()), - activeRunId, + set((state) => { + const current = + state.sessionStateById[sessionId] ?? createInitialSessionRuntime(); + const replacedRun = + current.activeRunId !== null && + activeRunId !== null && + current.activeRunId !== activeRunId; + const settled = + activeRunId === null && !isSessionRunning(current.chatState); + return { + sessionStateById: { + ...state.sessionStateById, + [sessionId]: { + ...current, + activeRunId, + hasToolCallInRun: + replacedRun || settled ? false : current.hasToolCallInRun, + }, }, - }, - })), + }; + }), setRunCancellationPending: (sessionId, isRunCancellationPending) => set((state) => ({ @@ -1302,12 +1329,17 @@ const createChatStore: StateCreator< const current = state.sessionStateById[sessionId] ?? createInitialSessionRuntime(); const isIdle = chatState === "idle"; + const hasToolCallInRun = + !isSessionRunning(chatState) && current.activeRunId === null + ? false + : current.hasToolCallInRun; const streamingMessageId = isIdle ? null : current.streamingMessageId; const pendingInterventionBoundary = isIdle ? null : current.pendingInterventionBoundary; if ( current.chatState === chatState && + current.hasToolCallInRun === hasToolCallInRun && current.streamingMessageId === streamingMessageId && current.pendingInterventionBoundary === pendingInterventionBoundary ) { @@ -1320,6 +1352,7 @@ const createChatStore: StateCreator< [sessionId]: { ...current, chatState, + hasToolCallInRun, streamingMessageId, pendingInterventionBoundary, }, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index c06c825c1..a80850283 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -589,6 +589,29 @@ export function rejectVoiceConversationTranscript( ); } +export interface VoiceStatusSoundSettings { + mode: "off" | "working" | "working-and-waiting"; + volume?: number; +} + +export async function updateVoiceConversationStatusSounds( + status: VoiceConversationStatus, + conversationStatus: "working" | "waiting", + settings: VoiceStatusSoundSettings, +): Promise { + const { rendererId, rendererEpoch } = await getRendererInstance(); + return invoke("update_native_voice_status_sounds", { + request: { + sessionId: status.sessionId, + expectedRevision: status.revision, + status: conversationStatus, + settings, + rendererId, + rendererEpoch, + }, + }); +} + export async function startVoiceConversation( sessionId: string, inputBackend: "parakeet" | "macos" | "openai" = "parakeet", diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts index 67d7e348d..a64cfb065 100644 --- a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; import type { OpenAiRealtimeProtocolEvent } from "@/shared/api/openaiRealtime"; +import { setStatusSoundPreference } from "../lib/statusSoundPreference"; import { collectRealtimeTranscriptSeedTurns, requestOpenAiRealtimeConversationStart, @@ -200,6 +201,7 @@ const mocks = vi.hoisted(() => ({ setControlsSuppressed: vi.fn(), startControls: vi.fn(), startRuntime: vi.fn(), + updateStatusSounds: vi.fn<() => Promise>(), stopControls: vi.fn(), stopRuntime: vi.fn(), updateRuntimeSettings: vi.fn(), @@ -358,6 +360,7 @@ vi.mock("@/shared/api/openaiRealtime", () => ({ stopOpenAiRealtimeVoiceControls: mocks.stopControls, stopOpenAiRealtimeSpokespersonRuntime: mocks.stopRuntime, updateOpenAiRealtimeSpokespersonSettings: mocks.updateRuntimeSettings, + updateOpenAiRealtimeStatusSounds: mocks.updateStatusSounds, unknownOpenAiRealtimeHandoffIds: mocks.unknownHandoffIds, })); @@ -510,6 +513,7 @@ describe("collectRealtimeTranscriptSeedTurns", () => { }); beforeEach(() => { + window.localStorage.clear(); vi.clearAllMocks(); mocks.activeEmissary = null; mocks.createResponse = true; @@ -538,6 +542,8 @@ beforeEach(() => { value: { getUserMedia: vi.fn().mockResolvedValue(stream) }, }); mocks.appendSessionSystemPrompt.mockResolvedValue(undefined); + mocks.updateStatusSounds.mockReset(); + mocks.updateStatusSounds.mockResolvedValue(undefined); mocks.claimMicrophone.mockResolvedValue(undefined); mocks.createHandoffToolOutput.mockReturnValue({ type: "conversation.item.create", @@ -910,6 +916,17 @@ describe("useOpenAiRealtimeConversation lifecycle", () => { await act(async () => owner.result.current.onToggle()); await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(mocks.updateStatusSounds).toHaveBeenCalledWith( + "session-a", + "waiting", + { mode: "working" }, + ); + act(() => setStatusSoundPreference({ mode: "working-and-waiting" })); + expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( + "session-a", + "waiting", + { mode: "working-and-waiting" }, + ); act(() => { mocks.preferenceListener?.({ voice: "cedar", speed: 1.5 }); }); @@ -924,6 +941,60 @@ describe("useOpenAiRealtimeConversation lifecycle", () => { ); }); + it("keeps working status until the admitted master run settles", async () => { + const onSend = vi.fn().mockImplementation(async () => { + useChatStore.getState().setChatState("session-a", "thinking"); + useChatStore.getState().setActiveRunId("session-a", "run-1"); + return true; + }); + const owner = renderConversation("session-a", onSend); + + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + mocks.updateStatusSounds.mockClear(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(owner.result.current.state).toBe("agent-working"); + expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( + "session-a", + "working", + { mode: "working" }, + ); + + act(() => { + useChatStore.getState().setActiveRunId("session-a", null); + useChatStore.getState().setChatState("session-a", "idle"); + }); + + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( + "session-a", + "waiting", + { mode: "working" }, + ); + + act(() => { + useChatStore.getState().setChatState("session-a", "thinking"); + useChatStore.getState().setActiveRunId("session-a", "run-2"); + }); + await waitFor(() => + expect(owner.result.current.state).toBe("agent-working"), + ); + expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( + "session-a", + "working", + { mode: "working" }, + ); + }); + it("serializes rapid voice settings changes against each applied revision", async () => { let resolveFirst!: (snapshot: { revision: number; diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts index 1d7a06a0a..736d86460 100644 --- a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts @@ -33,6 +33,7 @@ import { stopOpenAiRealtimeSpokespersonRuntime, releaseOpenAiRealtimeSpokespersonRuntime, updateOpenAiRealtimeSpokespersonSettings, + updateOpenAiRealtimeStatusSounds, type OpenAiRealtimeTranscriptSeedTurn, type OpenAiRealtimeExpertDeliveryEvent, } from "@/shared/api/openaiRealtime"; @@ -57,6 +58,15 @@ import { type MasterMessageMode, sendRealtimeEvents, } from "../lib/realtimeEmissaryProtocol"; +import { + getStatusSoundPreference, + subscribeToStatusSoundPreference, + type StatusSoundPreference, +} from "../lib/statusSoundPreference"; +import { + getRealtimeVoicePreference, + subscribeToRealtimeVoicePreference, +} from "../lib/realtimeVoicePreference"; import { requestVoiceConversationEnd, trackVoiceAssistantResponse, @@ -64,10 +74,6 @@ import { trackVoiceConversationStarted, trackVoiceUserUtterance, } from "../lib/voiceTelemetry"; -import { - getRealtimeVoicePreference, - subscribeToRealtimeVoicePreference, -} from "../lib/realtimeVoicePreference"; import { beginVoiceControlsVisibilityLease, observeVoiceConversationControlVisibility, @@ -439,12 +445,50 @@ const OFF_SNAPSHOT: Snapshot = { ownerWindowLabel: null, }; +function publishRealtimeStatus( + sessionId: string, + status: "working" | "waiting", + settings: StatusSoundPreference = getStatusSoundPreference(), +): void { + void updateOpenAiRealtimeStatusSounds(sessionId, status, settings).catch( + (error) => + console.warn(`Could not publish Realtime voice ${status} status`, error), + ); +} + +function resetRealtimeStatusWhenRunSettles( + runtime: OpenAiRealtimeConversationRuntime, + sessionId: string, +): void { + let sawRun = false; + let publishedStatus: "working" | "waiting" = "working"; + const check = () => { + if (runtime.getSnapshot().boundSessionId !== sessionId) { + unsubscribe(); + return; + } + const master = useChatStore.getState().getSessionRuntime(sessionId); + const working = + master.activeRunId !== null || isSessionRunning(master.chatState); + if (working) sawRun = true; + if (!sawRun) return; + const nextStatus = working ? "working" : "waiting"; + if (nextStatus === publishedStatus) return; + publishedStatus = nextStatus; + if (working) runtime.markWorking(sessionId); + else runtime.markWaiting(sessionId); + }; + const unsubscribe = useChatStore.subscribe(check); + queueMicrotask(check); +} + class OpenAiRealtimeConversationRuntime { private snapshot: Snapshot = OFF_SNAPSHOT; private readonly listeners = new Set<() => void>(); private nativeMicrophone: NativeMicrophone | null = null; private releaseRuntimeListener: (() => void) | null = null; private releaseVoicePreferenceListener: (() => void) | null = null; + private releaseStatusSoundPreferenceListener: (() => void) | null = null; private realtimeSettingsRevision = 1; private realtimeSettingsQueue = Promise.resolve(); private realtimeRuntimeSessionId: string | null = null; @@ -884,6 +928,17 @@ class OpenAiRealtimeConversationRuntime { this.bridgeCallScope.id, runtimeOptions, ); + publishRealtimeStatus(sessionId, "waiting"); + this.releaseStatusSoundPreferenceListener = + subscribeToStatusSoundPreference((settings) => { + if (isStale() || this.realtimeRuntimeSessionId !== sessionId) + return; + publishRealtimeStatus( + sessionId, + this.snapshot.state === "agent-working" ? "working" : "waiting", + settings, + ); + }); } catch (error) { if (this.realtimeRuntimeSessionId === sessionId) { this.realtimeRuntimeSessionId = null; @@ -1176,6 +1231,24 @@ class OpenAiRealtimeConversationRuntime { this.setSnapshot(OFF_SNAPSHOT); } + markWorking(sessionId: string): void { + if (this.snapshot.boundSessionId !== sessionId) return; + this.setSnapshot({ ...this.snapshot, state: "agent-working" }); + const runtimeSessionId = this.realtimeRuntimeSessionId; + if (runtimeSessionId) { + publishRealtimeStatus(runtimeSessionId, "working"); + } + } + + markWaiting(sessionId: string): void { + if (this.snapshot.boundSessionId !== sessionId) return; + this.setSnapshot({ ...this.snapshot, state: "listening" }); + const runtimeSessionId = this.realtimeRuntimeSessionId; + if (runtimeSessionId) { + publishRealtimeStatus(runtimeSessionId, "waiting"); + } + } + private deliverToMaster( sessionId: string, text: string, @@ -1243,6 +1316,10 @@ class OpenAiRealtimeConversationRuntime { }; if (!continueAfterStop) { this.setSnapshot({ ...this.snapshot, state: "agent-working" }); + const runtimeSessionId = this.realtimeRuntimeSessionId; + if (runtimeSessionId) { + publishRealtimeStatus(runtimeSessionId, "working"); + } } for (;;) { const opportunity = await waitForMasterDeliveryOpportunity( @@ -1281,8 +1358,9 @@ class OpenAiRealtimeConversationRuntime { } } onDelivered?.(); - if (this.snapshot.boundSessionId === sessionId) - this.setSnapshot({ ...this.snapshot, state: "listening" }); + if (this.snapshot.boundSessionId === sessionId) { + resetRealtimeStatusWhenRunSettles(this, sessionId); + } }) .catch((error) => { if (isAbortError(error)) return; @@ -1336,6 +1414,7 @@ class OpenAiRealtimeConversationRuntime { this.nativeMicrophone?.stop(); this.releaseRuntimeListener?.(); this.releaseVoicePreferenceListener?.(); + this.releaseStatusSoundPreferenceListener?.(); const realtimeRuntimeSessionId = this.realtimeRuntimeSessionId; this.realtimeRuntimeSessionId = null; this.releaseControlsListener?.(); @@ -1350,6 +1429,7 @@ class OpenAiRealtimeConversationRuntime { this.nativeMicrophone = null; this.releaseRuntimeListener = null; this.releaseVoicePreferenceListener = null; + this.releaseStatusSoundPreferenceListener = null; this.realtimeSettingsQueue = Promise.resolve(); if (controlsRevision > 0) { await stopOpenAiRealtimeVoiceControls( diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 1bbd56015..6752807b8 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -3,6 +3,7 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { useVoiceConversationStore } from "../stores/voiceConversationStore"; import { VoiceMicrophoneCaptureError } from "../api/voiceConversation"; +import { setStatusSoundPreference } from "../lib/statusSoundPreference"; const nativeAssistantSpeechMocks = vi.hoisted(() => ({ capture: vi.fn(() => []), @@ -13,6 +14,7 @@ const nativeAssistantSpeechMocks = vi.hoisted(() => ({ const tauriWindowMocks = vi.hoisted(() => ({ label: "main" })); const voiceApiMocks = vi.hoisted(() => ({ confirmForegroundSession: vi.fn<() => Promise>(), + updateStatusSounds: vi.fn<() => Promise>().mockResolvedValue(undefined), })); const microphonePermissionMocks = vi.hoisted(() => ({ getStatus: vi.fn<() => Promise<"authorized" | "denied">>(), @@ -38,6 +40,7 @@ vi.mock("../api/voiceConversation", async (importOriginal) => ({ ...(await importOriginal()), confirmVoiceConversationForegroundSession: voiceApiMocks.confirmForegroundSession, + updateVoiceConversationStatusSounds: voiceApiMocks.updateStatusSounds, })); vi.mock("../api/microphonePermission", () => ({ @@ -64,6 +67,7 @@ import { createVoiceTranscriptDeliveryQueue, hasDeliveredVoiceTranscript, observeVoiceConversationControlVisibility, + observeChainedVoiceStatus, replaceActiveVoiceConversation, resetVoiceUiWhenRunSettles, resolveActiveVoiceButtonAction, @@ -197,6 +201,43 @@ describe("voice transcript delivery coordination", () => { ).toBe(false); }); + it("silences waiting during a tool-free run without changing the preference", () => { + setStatusSoundPreference({ mode: "working-and-waiting" }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 3, + }, + }); + const stopObserving = observeChainedVoiceStatus(); + const store = useChatStore.getState(); + store.setChatState("session-1", "thinking"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenLastCalledWith( + expect.anything(), + "waiting", + { mode: "off" }, + ); + store.setChatState("session-1", "idle"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenLastCalledWith( + expect.anything(), + "waiting", + { mode: "working-and-waiting" }, + ); + store.setActiveRunId("session-1", "run-1"); + store.markToolCallInRun("session-1"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenLastCalledWith( + expect.anything(), + "working", + { mode: "working-and-waiting" }, + ); + stopObserving(); + }); + it("keeps working state until an admitted run actually settles", async () => { useVoiceConversationStore.setState({ status: { @@ -212,17 +253,57 @@ describe("voice transcript delivery coordination", () => { activityFallbackState: "agent-working", }); + const stopObserving = observeChainedVoiceStatus(); resetVoiceUiWhenRunSettles("session-1", 3); await Promise.resolve(); expect(useVoiceConversationStore.getState().uiState).toBe("agent-working"); useChatStore.getState().setActiveRunId("session-1", "run-1"); + useChatStore.getState().markToolCallInRun("session-1"); + voiceApiMocks.updateStatusSounds.mockClear(); + useVoiceConversationStore.getState().setUiState("agent-speaking"); + useVoiceConversationStore.getState().setUiState("listening"); + expect(voiceApiMocks.updateStatusSounds).not.toHaveBeenCalled(); useChatStore.getState().setActiveRunId("session-1", null); expect(useVoiceConversationStore.getState().uiState).toBe("listening"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1", revision: 3 }), + "waiting", + { mode: "working" }, + ); + + voiceApiMocks.updateStatusSounds.mockClear(); + useChatStore.getState().setActiveRunId("session-1", "run-2"); + expect(useVoiceConversationStore.getState().uiState).toBe("agent-working"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1" }), + "waiting", + { mode: "off" }, + ); + useChatStore.getState().markToolCallInRun("session-1"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1", revision: 3 }), + "working", + { mode: "working" }, + ); + + voiceApiMocks.updateStatusSounds.mockClear(); + useChatStore.getState().markToolCallInRun("session-1"); + expect(voiceApiMocks.updateStatusSounds).not.toHaveBeenCalled(); + useChatStore.getState().setActiveRunId("session-1", null); + useChatStore.getState().setError("session-1", "run failed"); + expect(useVoiceConversationStore.getState().uiState).toBe("listening"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1", revision: 3 }), + "waiting", + { mode: "working" }, + ); + stopObserving(); }); beforeEach(() => { + window.localStorage.clear(); tauriWindowMocks.label = "main"; nativeAssistantSpeechMocks.capture.mockClear(); nativeAssistantSpeechMocks.start.mockClear(); @@ -231,13 +312,62 @@ describe("voice transcript delivery coordination", () => { nativeAssistantSpeechMocks.takeNotices.mockReturnValue(null); voiceApiMocks.confirmForegroundSession.mockReset(); voiceApiMocks.confirmForegroundSession.mockResolvedValue(1); + voiceApiMocks.updateStatusSounds.mockReset(); + voiceApiMocks.updateStatusSounds.mockResolvedValue(undefined); microphonePermissionMocks.getStatus.mockReset(); microphonePermissionMocks.getStatus.mockResolvedValue("authorized"); useChatStore.setState({ messagesBySession: {}, sessionStateById: {} }); }); + it("applies settings changes to an active chained runtime", async () => { + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 3, + }, + uiState: "agent-speaking", + activityFallbackState: "agent-working", + hydrated: true, + init: vi.fn().mockResolvedValue(undefined), + }); + const { unmount } = renderHook(() => + useVoiceConversationController({ + sessionId: "session-1", + onSend: vi.fn(), + enabled: true, + isGooseSession: true, + pocketReady: true, + onPocketSetupRequired: vi.fn(), + }), + ); + + act(() => { + useChatStore.getState().setActiveRunId("session-1", "run-1"); + useChatStore.getState().markToolCallInRun("session-1"); + useVoiceConversationStore.getState().setUiState("listening"); + setStatusSoundPreference({ mode: "working-and-waiting" }); + }); + + await waitFor(() => + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1", revision: 3 }), + "working", + { mode: "working-and-waiting" }, + ), + ); + unmount(); + }); + it("delivers a queued transcript after its chat becomes temporarily ineligible", async () => { - const onSend = vi.fn().mockResolvedValue(true); + const onSend = vi.fn().mockImplementation(async () => { + useChatStore.getState().setActiveRunId("session-1", "run-1"); + return true; + }); useVoiceConversationStore.setState({ status: { available: true, @@ -286,6 +416,11 @@ describe("voice transcript delivery coordination", () => { undefined, expect.objectContaining({ displayText: "keep this route" }), ); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1", revision: 1 }), + "waiting", + { mode: "working" }, + ); }); it("releases a retained transcript route when the chat becomes read-only", async () => { diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index a4e993c82..4a5cb37bb 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -7,6 +7,7 @@ import type { ChatInputVoiceConversation, } from "@/features/chat/types"; import { useChatStore } from "@/features/chat/stores/chatStore"; +import { isSessionRunning } from "@/features/chat/lib/sessionActivity"; import { createSystemNotificationMessage } from "@/shared/types/messages"; import { steerPromptInSession } from "@/features/chat/lib/steerCore"; import { @@ -23,9 +24,15 @@ import { confirmVoiceConversationForegroundSession, isVoiceMicrophoneCaptureError, setVoiceConversationControlsSuppressed, + updateVoiceConversationStatusSounds, type PendingVoiceTranscript, } from "../api/voiceConversation"; import { getMicrophonePermissionStatus } from "../api/microphonePermission"; +import { + getStatusSoundPreference, + subscribeToStatusSoundPreference, + type StatusSoundPreference, +} from "../lib/statusSoundPreference"; import type { VoiceInputBackend } from "../lib/voiceInputPreference"; import type { SiriVoiceSelection } from "../api/siriVoice"; @@ -409,43 +416,118 @@ export function waitForVoiceDeliveryOpportunity( }); } +export function publishChainedVoiceStatus( + sessionId: string, + conversationStatus: "working" | "waiting", + settings: StatusSoundPreference = getStatusSoundPreference(), +): void { + const status = useVoiceConversationStore.getState().status; + if (status.lifecycle !== "running" || status.sessionId !== sessionId) return; + void updateVoiceConversationStatusSounds( + status, + conversationStatus, + settings, + ).catch((error) => + console.warn( + `Could not publish chained voice ${conversationStatus} status`, + error, + ), + ); +} + +const chainedRunStatusObservers = new Map void>(); + +export function observeChainedVoiceStatus(): () => void { + let published: string | null = null; + const check = () => { + const { status } = useVoiceConversationStore.getState(); + if ( + status.lifecycle !== "running" || + !status.sessionId || + status.ownerWindowLabel !== getCurrentWindow().label + ) { + published = null; + return; + } + const runtime = useChatStore.getState().getSessionRuntime(status.sessionId); + const running = + runtime.activeRunId !== null || isSessionRunning(runtime.chatState); + const conversationStatus = + runtime.hasToolCallInRun && running ? "working" : "waiting"; + const settings = + running && !runtime.hasToolCallInRun + ? { mode: "off" as const } + : getStatusSoundPreference(); + const next = JSON.stringify([ + status.sessionId, + status.revision, + conversationStatus, + settings, + ]); + if (next === published) return; + published = next; + publishChainedVoiceStatus(status.sessionId, conversationStatus, settings); + }; + const unsubscribeChat = useChatStore.subscribe(check); + const unsubscribeVoice = useVoiceConversationStore.subscribe(check); + const unsubscribePreference = subscribeToStatusSoundPreference(check); + check(); + return () => { + unsubscribeChat(); + unsubscribeVoice(); + unsubscribePreference(); + }; +} + export function resetVoiceUiWhenRunSettles( sessionId: string, deliveryRevision: number, ): void { + chainedRunStatusObservers.get(sessionId)?.(); let sawRun = false; + let publishedStatus: "working" | "waiting" = "working"; + const cleanup = () => { + unsubscribeChat(); + unsubscribeVoice(); + if (chainedRunStatusObservers.get(sessionId) === cleanup) { + chainedRunStatusObservers.delete(sessionId); + } + }; const check = () => { const voice = useVoiceConversationStore.getState(); if ( voice.status.lifecycle !== "running" || voice.status.sessionId !== sessionId ) { - unsubscribeChat(); - unsubscribeVoice(); + cleanup(); return; } const runtime = useChatStore.getState().getSessionRuntime(sessionId); - if (runtime.activeRunId !== null || runtime.chatState !== "idle") { - sawRun = true; - return; - } + const working = + runtime.activeRunId !== null || isSessionRunning(runtime.chatState); + if (working) sawRun = true; if (!sawRun) return; - unsubscribeChat(); - unsubscribeVoice(); + const nextStatus = working ? "working" : "waiting"; + if (nextStatus === publishedStatus) return; if (voice.status.revision >= deliveryRevision) { - voice.setUiState("listening"); + publishedStatus = nextStatus; + voice.setUiState( + nextStatus === "working" ? "agent-working" : "listening", + ); } }; const unsubscribeChat = useChatStore.subscribe(check); const unsubscribeVoice = useVoiceConversationStore.subscribe(check); + chainedRunStatusObservers.set(sessionId, cleanup); queueMicrotask(check); } function ensureVoiceEventDeliveryInitialized() { if (deliveryInitialized) return; deliveryInitialized = true; + observeChainedVoiceStatus(); subscribeToVoiceConversationEvents(async (event) => { if (event.type === "cleanShutdown" || event.type === "controlsDismissed") { return; diff --git a/src/features/voice-conversation/lib/statusSoundPreference.test.ts b/src/features/voice-conversation/lib/statusSoundPreference.test.ts new file mode 100644 index 000000000..d01185eda --- /dev/null +++ b/src/features/voice-conversation/lib/statusSoundPreference.test.ts @@ -0,0 +1,71 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + getDefaultStatusSoundPreference, + getStatusSoundPreference, + setStatusSoundPreference, + subscribeToStatusSoundPreference, +} from "./statusSoundPreference"; + +describe("status sound preference", () => { + beforeEach(() => window.localStorage.clear()); + afterEach(() => vi.restoreAllMocks()); + + it("defaults to repeating only while working", () => { + expect(getDefaultStatusSoundPreference()).toEqual({ mode: "working" }); + expect(getStatusSoundPreference()).toEqual( + getDefaultStatusSoundPreference(), + ); + }); + + it("persists the working-and-waiting mode", () => { + setStatusSoundPreference({ mode: "working-and-waiting" }); + expect(getStatusSoundPreference()).toEqual({ + mode: "working-and-waiting", + }); + }); + + it("persists off without re-enabling sounds", () => { + setStatusSoundPreference({ mode: "off" }); + expect(getStatusSoundPreference()).toEqual({ mode: "off" }); + }); + + it("normalizes malformed persisted values", () => { + window.localStorage.setItem( + "goose:voice-status-sound-preference", + JSON.stringify({ mode: "unexpected" }), + ); + expect(getStatusSoundPreference()).toEqual({ mode: "working" }); + }); + + it.each([ + ["continuous", "working-and-waiting"], + ["continuous-while-working", "working"], + ["once", "working"], + ] as const)("migrates the legacy %s mode to %s", (legacy, expected) => { + window.localStorage.setItem( + "goose:voice-status-sound-preference", + JSON.stringify({ mode: legacy }), + ); + expect(getStatusSoundPreference()).toEqual({ mode: expected }); + }); + + it("notifies runtime subscribers with the applied preference", () => { + const listener = vi.fn(); + const unsubscribe = subscribeToStatusSoundPreference(listener); + + setStatusSoundPreference({ mode: "working-and-waiting" }); + + expect(listener).toHaveBeenCalledWith({ mode: "working-and-waiting" }); + unsubscribe(); + }); + + it("keeps the renderer preference usable when storage writes fail", () => { + vi.spyOn(window.localStorage, "setItem").mockImplementation(() => { + throw new Error("storage unavailable"); + }); + setStatusSoundPreference({ mode: "working-and-waiting" }); + expect(getStatusSoundPreference()).toEqual({ + mode: "working-and-waiting", + }); + }); +}); diff --git a/src/features/voice-conversation/lib/statusSoundPreference.ts b/src/features/voice-conversation/lib/statusSoundPreference.ts new file mode 100644 index 000000000..22cf70728 --- /dev/null +++ b/src/features/voice-conversation/lib/statusSoundPreference.ts @@ -0,0 +1,119 @@ +import { useCallback, useSyncExternalStore } from "react"; + +export type StatusSoundMode = "off" | "working" | "working-and-waiting"; + +export interface StatusSoundPreference { + mode: StatusSoundMode; +} + +const STORAGE_KEY = "goose:voice-status-sound-preference"; +const CHANGED_EVENT = "goose:voice-status-sound-preference-changed"; +const DEFAULT_PREFERENCE: StatusSoundPreference = { + mode: "working", +}; +const NORMALIZED_MODES: Record = { + continuous: "working-and-waiting", + "continuous-while-working": "working", + off: "off", + once: "working", + working: "working", + "working-and-waiting": "working-and-waiting", +}; +const DEFAULT_SNAPSHOT = JSON.stringify(DEFAULT_PREFERENCE); +let volatilePreference: StatusSoundPreference | undefined; + +function normalize(value: unknown): StatusSoundPreference { + if (!value || typeof value !== "object") return DEFAULT_PREFERENCE; + const candidate = value as { mode?: unknown }; + const mode = + typeof candidate.mode === "string" + ? (NORMALIZED_MODES[candidate.mode] ?? DEFAULT_PREFERENCE.mode) + : DEFAULT_PREFERENCE.mode; + return { mode }; +} + +export function getDefaultStatusSoundPreference(): StatusSoundPreference { + return DEFAULT_PREFERENCE; +} + +export function getStatusSoundPreference(): StatusSoundPreference { + if (typeof window === "undefined") return DEFAULT_PREFERENCE; + if (volatilePreference) return volatilePreference; + try { + const raw = window.localStorage.getItem(STORAGE_KEY); + return raw ? normalize(JSON.parse(raw)) : DEFAULT_PREFERENCE; + } catch { + return DEFAULT_PREFERENCE; + } +} + +function getSnapshot(): string { + return JSON.stringify(getStatusSoundPreference()); +} + +const listeners = new Set<() => void>(); +let removeWindowListeners: (() => void) | undefined; + +function notify() { + for (const listener of listeners) listener(); +} + +function subscribe(listener: () => void) { + if (typeof window === "undefined") return () => {}; + listeners.add(listener); + if (!removeWindowListeners) { + const handleStorage = (event: StorageEvent) => { + if (event.key === STORAGE_KEY || event.key === null) { + volatilePreference = undefined; + notify(); + } + }; + window.addEventListener(CHANGED_EVENT, notify); + window.addEventListener("storage", handleStorage); + removeWindowListeners = () => { + window.removeEventListener(CHANGED_EVENT, notify); + window.removeEventListener("storage", handleStorage); + }; + } + return () => { + listeners.delete(listener); + if (listeners.size === 0) { + removeWindowListeners?.(); + removeWindowListeners = undefined; + } + }; +} + +export function setStatusSoundPreference( + preference: StatusSoundPreference, +): void { + if (typeof window === "undefined") return; + const value = normalize(preference); + volatilePreference = value; + try { + window.localStorage.setItem(STORAGE_KEY, JSON.stringify(value)); + volatilePreference = undefined; + } catch { + // Keep the current renderer usable when persistent storage is unavailable. + } + window.dispatchEvent(new CustomEvent(CHANGED_EVENT, { detail: value })); +} + +export function subscribeToStatusSoundPreference( + listener: (preference: StatusSoundPreference) => void, +): () => void { + return subscribe(() => listener(getStatusSoundPreference())); +} + +export function useStatusSoundPreference() { + const snapshot = useSyncExternalStore( + subscribe, + getSnapshot, + () => DEFAULT_SNAPSHOT, + ); + const preference = normalize(JSON.parse(snapshot)); + const update = useCallback((patch: Partial) => { + setStatusSoundPreference({ ...getStatusSoundPreference(), ...patch }); + }, []); + return { ...preference, update }; +} diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 073aadced..f26a185ff 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -64,8 +64,12 @@ const preferenceMocks = vi.hoisted(() => ({ setOutputBackend: vi.fn(), setInterruptionMode: vi.fn(), setMode: vi.fn(), + setStatusSounds: vi.fn(), setRealtimePreference: vi.fn(), })); +const statusSoundState = vi.hoisted(() => ({ + mode: "working" as "off" | "working" | "working-and-waiting", +})); const interruptionState = vi.hoisted(() => ({ mode: "automatic" as "automatic" | "allowInterruptions" | "preventFeedback", })); @@ -176,6 +180,13 @@ vi.mock("../lib/voiceConversationModePreference", () => ({ setMode: preferenceMocks.setMode, }), })); +vi.mock("../lib/statusSoundPreference", () => ({ + getDefaultStatusSoundPreference: () => ({ mode: "working" }), + useStatusSoundPreference: () => ({ + ...statusSoundState, + update: preferenceMocks.setStatusSounds, + }), +})); vi.mock("../lib/realtimeVoicePreference", async (importOriginal) => ({ ...(await importOriginal()), setRealtimeVoicePreference: preferenceMocks.setRealtimePreference, @@ -327,6 +338,22 @@ describe("VoiceSettings", () => { preferenceMocks.setRealtimePreference.mockClear(); }); + it("offers off and both repeating status sound modes without a volume control", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + expect(screen.queryByText("Status sound volume")).not.toBeInTheDocument(); + await user.click(screen.getByRole("combobox", { name: "Status sounds" })); + expect( + screen.getByRole("option", { name: "While working" }), + ).toBeInTheDocument(); + expect( + screen.getByRole("option", { name: "While working and waiting" }), + ).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "Off" })).toBeInTheDocument(); + expect(screen.getAllByRole("option")).toHaveLength(3); + }); + it("describes voice modes by who the user talks with", () => { renderWithProviders(); diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index 2ea924829..eea396b17 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -30,6 +30,11 @@ import { usePocketVoiceSetup } from "../hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "../hooks/useMacSpeechSetup"; import { useMicrophonePermission } from "../hooks/useMicrophonePermission"; import { useSiriVoiceSetup } from "../hooks/useSiriVoiceSetup"; +import type { StatusSoundMode } from "../lib/statusSoundPreference"; +import { + getDefaultStatusSoundPreference, + useStatusSoundPreference, +} from "../lib/statusSoundPreference"; import type { VoiceInputBackend } from "../lib/voiceInputPreference"; import { getDefaultVoiceInputBackend, @@ -68,6 +73,12 @@ import { openAiVoiceOptions, } from "../lib/openAiVoiceOptions"; +const STATUS_SOUND_MODES: StatusSoundMode[] = [ + "off", + "working", + "working-and-waiting", +]; + const INTERRUPTION_MODES: VoiceInterruptionMode[] = [ "automatic", "allowInterruptions", @@ -162,6 +173,7 @@ export function VoiceSettings() { }, [openAiStatus]); const interruption = useVoiceInterruptionPreference(); const mode = useVoiceConversationModePreference(); + const statusSounds = useStatusSoundPreference(); const siriSetup = useSiriVoiceSetup(output.backend === "siri"); const siriSupported = getPlatform() === "mac"; const microphonePermission = useMicrophonePermission(siriSupported); @@ -233,6 +245,7 @@ export function VoiceSettings() { output.setBackend(getDefaultVoiceOutputBackend()); interruption.setMode(getDefaultVoiceInterruptionPreference().mode); mode.setMode(getDefaultVoiceConversationMode()); + statusSounds.update(getDefaultStatusSoundPreference()); setResetDialogOpen(false); } finally { setResetting(false); @@ -595,6 +608,41 @@ export function VoiceSettings() { ) : ( )} +
+ {t("voice.statusSounds")} + } + description={t( + `voice.statusSoundModeDescriptions.${statusSounds.mode}`, + )} + layout="responsive" + action={({ labelId, descriptionId }) => ( + + )} + /> +
{resetError ? (

{resetError} diff --git a/src/shared/api/openaiRealtime.ts b/src/shared/api/openaiRealtime.ts index 6a31162fa..356a01cc2 100644 --- a/src/shared/api/openaiRealtime.ts +++ b/src/shared/api/openaiRealtime.ts @@ -51,6 +51,20 @@ export function startOpenAiRealtimeSpokespersonRuntime( }); } +export function updateOpenAiRealtimeStatusSounds( + sessionId: string, + status: "working" | "waiting", + settings: { + mode: "off" | "working" | "working-and-waiting"; + volume?: number; + }, +): Promise { + return invoke("update_openai_realtime_status_sounds", { + sessionId, + update: { status, settings }, + }); +} + export function sendOpenAiRealtimeSpokespersonRuntimeEvent( sessionId: string, event: Record, diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index cfd8a9ffd..7658e72ee 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -1098,6 +1098,17 @@ "selectedVoice": "Selected voice: {{voice}}", "useVoice": "Use {{voice}}", "voice": "Voice", - "voiceLabel": "Pocket TTS voice" + "voiceLabel": "Pocket TTS voice", + "statusSounds": "Status sounds", + "statusSoundModes": { + "off": "Off", + "working": "While working", + "working-and-waiting": "While working and waiting" + }, + "statusSoundModeDescriptions": { + "off": "Do not play status sounds.", + "working": "Repeats the working sound while the agent is working.", + "working-and-waiting": "Repeats the working sound while the agent is working and the waiting sound while it waits." + } } } diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index 8778dba5b..adbace8e3 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -1097,6 +1097,17 @@ "selectedVoice": "Voz seleccionada: {{voice}}", "useVoice": "Usar {{voice}}", "voice": "Voz", - "voiceLabel": "Voz de Pocket TTS" + "voiceLabel": "Voz de Pocket TTS", + "statusSounds": "Sonidos de estado", + "statusSoundModes": { + "off": "Desactivados", + "working": "Mientras trabaja", + "working-and-waiting": "Mientras trabaja y espera" + }, + "statusSoundModeDescriptions": { + "off": "No reproducir sonidos de estado.", + "working": "Repite el sonido de trabajo mientras el agente está trabajando.", + "working-and-waiting": "Repite el sonido de trabajo mientras el agente trabaja y el sonido de espera mientras espera." + } } } diff --git a/src/shared/types/chat.ts b/src/shared/types/chat.ts index deacfff3d..1bb45851c 100644 --- a/src/shared/types/chat.ts +++ b/src/shared/types/chat.ts @@ -38,6 +38,7 @@ export interface SessionChatRuntime { hasUsageSnapshot: boolean; streamingMessageId: string | null; activeRunId: string | null; + hasToolCallInRun: boolean; isRunCancellationPending: boolean; pendingInterventionBoundary: { interventionMessageId: string; @@ -53,6 +54,7 @@ export const INITIAL_SESSION_CHAT_RUNTIME: SessionChatRuntime = { hasUsageSnapshot: false, streamingMessageId: null, activeRunId: null, + hasToolCallInRun: false, isRunCancellationPending: false, pendingInterventionBoundary: null, pendingAssistantProviderId: null,