From 1d685d70af693a2163b0768b6b926e3fabf5d931 Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Tue, 8 Sep 2026 19:18:36 -0400 Subject: [PATCH 01/29] Add runtime voice status sounds Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-voice/PROTOCOL.md | 42 +- src-tauri/crates/berd-voice/README.md | 8 +- src-tauri/crates/berd-voice/src/lib.rs | 1 + src-tauri/crates/berd-voice/src/main.rs | 62 ++- src-tauri/crates/berd-voice/src/protocol.rs | 21 +- .../crates/berd-voice/src/status_sounds.rs | 435 ++++++++++++++++++ .../berd-voice/tests/session_protocol.rs | 25 +- 7 files changed, 579 insertions(+), 15 deletions(-) create mode 100644 src-tauri/crates/berd-voice/src/status_sounds.rs diff --git a/src-tauri/crates/berd-voice/PROTOCOL.md b/src-tauri/crates/berd-voice/PROTOCOL.md index 786a3efcf..6d52f233e 100644 --- a/src-tauri/crates/berd-voice/PROTOCOL.md +++ b/src-tauri/crates/berd-voice/PROTOCOL.md @@ -3,8 +3,10 @@ `berd-voice session` is a development, full-authority voice session. The child owns speech recognition, finalized-input order, confirmation, speak admission, synthesis, source-frame delivery, playback lifecycle, and barge-in. The parent -owns capture and playback devices: it writes normalized microphone PCM on stdin -and consumes synthesized PCM from a dedicated inherited pipe. The child writes +owns capture and conversational playback devices: it writes normalized microphone PCM on +stdin and consumes synthesized PCM from a dedicated inherited pipe. Status cues are +the narrow exception: the child plays them on the output-device name supplied by the +parent, or the system default when none is supplied. The child writes flushed JSONL events to stdout. Diagnostics go only to stderr. ## Startup @@ -44,17 +46,21 @@ remain terminal speech events. has no device-owning or stdout-multiplexed fallback. The first request must be `hello`. `input_during_tts` is the host's resolved -initial policy; a host-specific `auto` mode must be resolved before the request: +initial policy; a host-specific `auto` mode must be resolved before the request. +`status_sounds` is the host's persisted effective preference. `status_sound_output_device` +is the host's selected playback-device name; omitting it uses the system default. +Omitting `status_sounds` uses `continuous-while-working` at volume `0.4`: ```json -{"type":"hello","id":1,"input_during_tts":"allow_barge_in"} +{"type":"hello","id":1,"input_during_tts":"allow_barge_in","status_sounds":{"mode":"continuous-while-working","volume":0.4},"status_sound_output_device":"MacBook Pro Speakers"} ``` -The response uses `protocol:4` as the exact session message-set version. The -parent must reject a version it does not support: +The response uses `protocol:5` as the exact session message-set version. The +parent must reject a version it does not support; the JSON version is independent +of the fixed binary framing marker described below: ```json -{"type":"ready","id":1,"protocol":4,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} +{"type":"ready","id":1,"protocol":5,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"status_sounds":{"mode":"continuous-while-working","volume":0.4}}} ``` The `session.tts` object is the authoritative, sanitized TTS configuration. @@ -64,7 +70,9 @@ and `rate`. Credentials, endpoints, and bundle paths never appear on stdout. Detailed backend errors are diagnostics on stderr only; protocol rejection and fatal messages are sanitized at the stdout boundary. `session.input_during_tts` is the authoritative effective assistant-input -policy and has its own revision. +policy and has its own revision. `session.status_sounds` is the host-provided +effective mode and volume. The runtime owns cue cadence and playback; it does not +persist preferences. ## Stdin framing @@ -179,9 +187,10 @@ it never admits a replacement while old host audio may still be active. ## Parent requests ```text -{"type":"hello","id":u64,"input_during_tts":"allow_barge_in"|"suppress_input"} +{"type":"hello","id":u64,"input_during_tts":"allow_barge_in"|"suppress_input","status_sounds":StatusSoundSettings} {"type":"set_paused","active":bool} {"type":"set_input_muted","id":u64,"active":bool} +{"type":"set_conversation_status","id":u64,"status":"working"|"waiting","settings":StatusSoundSettings} {"type":"set_tts_settings","id":u64,"expected_revision":u64,"settings":TtsSettings} {"type":"set_input_during_tts","id":u64,"expected_revision":u64,"policy":"allow_barge_in"|"suppress_input"} {"type":"reset_input","id":u64} @@ -198,6 +207,21 @@ Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. The parent cannot author speaking state or finalized input; those are derived only from PCM by the child runtime. +`StatusSoundSettings` has a `mode` of `continuous`, +`continuous-while-working`, `once`, or `off`, and a finite `volume` from `0` +through `1`. No cue is emitted until the first `set_conversation_status` request. +The runtime then ticks immediately and every five seconds. `continuous` emits the +current cue every tick; `continuous-while-working` repeats working and emits +waiting once; `once` emits only when the requested status differs from the last +emitted cue; `off` emits nothing. Audible user or assistant conversation audio +suppresses a tick without consuming its pending cue. On macOS, working uses the +system Pop sound and waiting uses Purr through the native PCM player. The applied +request is acknowledged with: + +```text +{"type":"conversation_status_applied","id":u64,"status":"working"|"waiting","settings":StatusSoundSettings} +``` + `set_tts_settings` accepts the same tagged public object projected by `ready`, without `revision`. It changes settings only for the already-active backend: diff --git a/src-tauri/crates/berd-voice/README.md b/src-tauri/crates/berd-voice/README.md index 3e14b5b3c..5e32f9800 100644 --- a/src-tauri/crates/berd-voice/README.md +++ b/src-tauri/crates/berd-voice/README.md @@ -18,8 +18,12 @@ Siri bridge emits normalized 48 kHz mono Float32 PCM without opening an audio device; the existing Berd Siri player and the CLI use the same decoder. `berd-voice session` exposes the development voice-session protocol documented -in [PROTOCOL.md](PROTOCOL.md). Siri TTS and macOS speech recognition are the -defaults: +in [PROTOCOL.md](PROTOCOL.md). The host supplies persisted status-sound settings +and semantic `working` / `waiting` updates through that protocol; the runtime +owns the five-second cadence, speech suppression, and macOS Pop/Purr playback. +The default mode is `continuous-while-working` at volume `0.4`. + +Siri TTS and macOS speech recognition are the defaults: ```text berd-voice session --voice Aaron --language en-US --rate 1.0 diff --git a/src-tauri/crates/berd-voice/src/lib.rs b/src-tauri/crates/berd-voice/src/lib.rs index f5a792564..b85dfe41c 100644 --- a/src-tauri/crates/berd-voice/src/lib.rs +++ b/src-tauri/crates/berd-voice/src/lib.rs @@ -32,6 +32,7 @@ pub mod realtime_pipe; pub mod session; pub mod siri; pub mod spokesperson_voice_update; +pub mod status_sounds; mod synthesis; mod tts; diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index 481c8806c..a99dba9b7 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -46,6 +46,7 @@ use berd_voice::spokesperson_voice_update::{ validate_voice_update_settings, VoiceBarrierAction, VoiceUpdateAction, VoiceUpdatePurpose, VoiceUpdateQueue, VoiceUpdateRequest, VoiceUpdateTransaction, }; +use berd_voice::status_sounds::StatusSoundRuntime; use berd_voice::{ estimated_spoken_through_utf8, local_assets::{ @@ -65,7 +66,7 @@ use session_audio::{ AUDIO_CANCELLED, }; -const SESSION_PROTOCOL_VERSION: u32 = 4; +const SESSION_PROTOCOL_VERSION: u32 = 5; const INPUT_FRAME_MARKER: u8 = 3; const MAX_LINE_BYTES: usize = 1024 * 1024; const FRAME_MAGIC: [u8; 2] = *b"BV"; @@ -1259,6 +1260,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let mut processed_pcm = 0_u64; let mut held: Option = None; let mut active: Option = None; + let mut status_sound_runtime = StatusSoundRuntime::default(); loop { if let Some(events) = input_events.as_mut() { @@ -1328,6 +1330,11 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String }, )?; } + if let Err(message) = status_sound_runtime + .poll(core.user_speaking() || core.recognition_pending() || active.is_some()) + { + eprintln!("status sound playback disabled: {message}"); + } let Some(input) = receive_session_input( &control_rx, @@ -1424,6 +1431,8 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String Input::Request(SessionRequest::Hello { id, input_during_tts, + status_sounds, + status_sound_output_device, }) => { if initialized { write_message( @@ -1474,10 +1483,12 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String input_runtime = Some(runtime); input_events = Some(events); initialized = true; + status_sound_runtime.set_output_device(status_sound_output_device); let input_policy = InputDuringTtsSlot::new(input_during_tts); let session = VoiceSessionSnapshot { tts: slot.snapshot()?, input_during_tts: input_policy.snapshot()?, + status_sounds, }; tts_slot = Some(slot); input_during_tts_slot = Some(input_policy); @@ -1500,6 +1511,21 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String )?; return Ok(()); } + Input::Request(SessionRequest::SetConversationStatus { + id, + status, + settings, + }) => { + status_sound_runtime.update(status, settings); + write_message( + &mut writer, + &SessionMessage::ConversationStatusApplied { + id, + status, + settings, + }, + )?; + } Input::Request(SessionRequest::SetInputMuted { id, active: muted }) => { handle_input_muted( id, @@ -2445,6 +2471,7 @@ fn run_expert_spokesperson_session( let mut input_muted = false; let mut queued_tts_settings = VoiceUpdateQueue::::default(); let mut pending_voice_update: Option = None; + let mut status_sound_runtime = StatusSoundRuntime::default(); loop { if initialized { @@ -3300,6 +3327,11 @@ fn run_expert_spokesperson_session( )?; } } + if let Err(message) = + status_sound_runtime.poll(turn_gate.input_blocks_output() || active.is_some()) + { + eprintln!("status sound playback disabled: {message}"); + } let Some(input) = receive_session_input( &control_rx, @@ -3393,6 +3425,8 @@ fn run_expert_spokesperson_session( Input::Request(SessionRequest::Hello { id, input_during_tts, + status_sounds, + status_sound_output_device, }) => { if initialized { write_protocol_fatal( @@ -3416,6 +3450,7 @@ fn run_expert_spokesperson_session( let snapshot = VoiceSessionSnapshot { tts: tts.clone(), input_during_tts: input_policy.snapshot()?, + status_sounds, }; let (created, events) = OpenAiSpokespersonRuntime::spawn_observed(spokesperson_config.clone())?; @@ -3443,6 +3478,7 @@ fn run_expert_spokesperson_session( session_tts = Some(tts); input_during_tts_slot = Some(input_policy); initialized = true; + status_sound_runtime.set_output_device(status_sound_output_device); turn_gate.lifecycle.session_started(Instant::now()); write_message( &mut writer, @@ -3821,6 +3857,21 @@ fn run_expert_spokesperson_session( cancel_live_playback(&mut active); } } + Input::Request(SessionRequest::SetConversationStatus { + id, + status, + settings, + }) => { + status_sound_runtime.update(status, settings); + write_message( + &mut writer, + &SessionMessage::ConversationStatusApplied { + id, + status, + settings, + }, + )?; + } Input::Request(SessionRequest::SetInputMuted { id, active: muted }) => { if pending_voice_update.is_some() { rollback_spokesperson_voice_update( @@ -6495,6 +6546,7 @@ fn validate_request(request: SessionRequest) -> Result { let id = match &request { SessionRequest::Hello { id, .. } | SessionRequest::SetInputMuted { id, .. } + | SessionRequest::SetConversationStatus { id, .. } | SessionRequest::SetTtsSettings { id, .. } | SessionRequest::SetInputDuringTts { id, .. } | SessionRequest::ResetInput { id } @@ -6521,6 +6573,13 @@ fn validate_request(request: SessionRequest) -> Result { return Err("request id must be positive".into()); } match &request { + SessionRequest::Hello { status_sounds, .. } + | SessionRequest::SetConversationStatus { + settings: status_sounds, + .. + } => { + status_sounds.validate()?; + } SessionRequest::PrepareSpeak { text, .. } if text.len() > MAX_SPEAK_TEXT_BYTES => { return Err("speak text exceeds 16 KiB".into()) } @@ -8124,6 +8183,7 @@ mod tests { session: VoiceSessionSnapshot { tts: snapshot.clone(), input_during_tts: test_input_policy(), + status_sounds: Default::default(), }, }) .unwrap(); diff --git a/src-tauri/crates/berd-voice/src/protocol.rs b/src-tauri/crates/berd-voice/src/protocol.rs index 125f8363d..4cf30e2d7 100644 --- a/src-tauri/crates/berd-voice/src/protocol.rs +++ b/src-tauri/crates/berd-voice/src/protocol.rs @@ -3,6 +3,7 @@ use serde::{Deserialize, Serialize}; use crate::{ input::{InputDuringTtsPolicy, InputDuringTtsSnapshot}, openai_realtime_protocol::RealtimeExpertDeliveryEvent, + status_sounds::{ConversationStatus, StatusSoundSettings}, TtsConfigurationSnapshot, TtsSettings, }; @@ -12,6 +13,15 @@ pub enum SessionRequest { Hello { id: u64, input_during_tts: InputDuringTtsPolicy, + #[serde(default)] + status_sounds: StatusSoundSettings, + #[serde(default)] + status_sound_output_device: Option, + }, + SetConversationStatus { + id: u64, + status: ConversationStatus, + settings: StatusSoundSettings, }, SetPaused { active: bool, @@ -152,6 +162,7 @@ pub enum OutputReadyOutcome { pub struct VoiceSessionSnapshot { pub tts: TtsConfigurationSnapshot, pub input_during_tts: InputDuringTtsSnapshot, + pub status_sounds: StatusSoundSettings, } #[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] @@ -192,6 +203,11 @@ pub enum SessionMessage { protocol: u32, session: VoiceSessionSnapshot, }, + ConversationStatusApplied { + id: u64, + status: ConversationStatus, + settings: StatusSoundSettings, + }, TtsSettingsResult { id: u64, outcome: TtsSettingsOutcome, @@ -341,7 +357,7 @@ mod tests { assert_eq!( serde_json::to_string(&SessionMessage::Ready { id: 4, - protocol: 4, + protocol: 5, session: VoiceSessionSnapshot { tts: TtsConfigurationSnapshot { revision: 1, @@ -355,10 +371,11 @@ mod tests { revision: 1, policy: InputDuringTtsPolicy::AllowBargeIn, }, + status_sounds: StatusSoundSettings::default(), }, }) .unwrap(), - r#"{"type":"ready","id":4,"protocol":4,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"# + r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"status_sounds":{"mode":"continuous-while-working","volume":0.4}}}"# ); assert_eq!( serde_json::from_str::( diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs new file mode 100644 index 000000000..edf4471ab --- /dev/null +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -0,0 +1,435 @@ +use std::time::{Duration, Instant}; + +use serde::{Deserialize, Serialize}; + +pub const DEFAULT_STATUS_SOUND_VOLUME: f32 = 0.4; +pub const STATUS_SOUND_INTERVAL: Duration = Duration::from_secs(5); + +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum StatusSoundMode { + Continuous, + #[default] + ContinuousWhileWorking, + Once, + Off, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ConversationStatus { + Working, + Waiting, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct StatusSoundSettings { + pub mode: StatusSoundMode, + pub volume: f32, +} + +impl Default for StatusSoundSettings { + fn default() -> Self { + Self { + mode: StatusSoundMode::default(), + volume: DEFAULT_STATUS_SOUND_VOLUME, + } + } +} + +impl StatusSoundSettings { + pub fn validate(self) -> Result { + if !self.volume.is_finite() || !(0.0..=1.0).contains(&self.volume) { + return Err("status sound volume must be finite and between 0 and 1"); + } + Ok(self) + } +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct StatusSoundCue { + pub status: ConversationStatus, + pub volume: f32, +} + +/// Pure policy for deciding which cue, if any, a fixed-cadence runtime tick plays. +/// The host owns the timer and supplies whether conversation audio is currently audible. +#[derive(Debug, Default)] +pub struct StatusSoundStateMachine { + current: Option<(ConversationStatus, StatusSoundSettings)>, + last_played: Option, +} + +impl StatusSoundStateMachine { + pub fn update(&mut self, status: ConversationStatus, settings: StatusSoundSettings) { + self.current = Some((status, settings)); + } + + pub fn tick(&mut self, conversation_audio_audible: bool) -> Option { + let (status, settings) = self.current?; + if conversation_audio_audible || settings.mode == StatusSoundMode::Off { + return None; + } + let should_play = match settings.mode { + StatusSoundMode::Continuous => true, + StatusSoundMode::ContinuousWhileWorking => { + status == ConversationStatus::Working || self.last_played != Some(status) + } + StatusSoundMode::Once => self.last_played != Some(status), + StatusSoundMode::Off => false, + }; + if !should_play { + return None; + } + self.last_played = Some(status); + Some(StatusSoundCue { + status, + volume: settings.volume, + }) + } + + pub fn clear(&mut self) { + self.current = None; + self.last_played = None; + } +} + +/// Owns status cadence and cue playback for one running voice session. +/// Producers report semantic state; they never choose or play a sound. +pub struct StatusSoundRuntime { + machine: StatusSoundStateMachine, + next_tick: Option, + player: StatusSoundPlayer, + output_device: Option, + playback_available: bool, + conversation_audio_audible: bool, +} + +impl Default for StatusSoundRuntime { + fn default() -> Self { + Self { + machine: StatusSoundStateMachine::default(), + next_tick: None, + player: StatusSoundPlayer::default(), + output_device: None, + playback_available: cfg!(target_os = "macos"), + conversation_audio_audible: false, + } + } +} + +impl StatusSoundRuntime { + pub fn set_output_device(&mut self, output_device: Option) { + if self.output_device != output_device { + self.player.stop(); + self.output_device = output_device; + } + } + + pub fn update(&mut self, status: ConversationStatus, settings: StatusSoundSettings) { + let changed = self.machine.current != Some((status, settings)); + self.machine.update(status, settings); + if changed { + self.player.stop(); + self.next_tick = Some(Instant::now()); + } + } + + pub fn poll(&mut self, conversation_audio_audible: bool) -> Result<(), String> { + if conversation_audio_audible && !self.conversation_audio_audible { + self.player.stop(); + } + self.conversation_audio_audible = conversation_audio_audible; + if !self.playback_available { + return Ok(()); + } + self.player.reap(); + let now = Instant::now(); + if self.next_tick.is_none_or(|deadline| now < deadline) { + return Ok(()); + } + self.next_tick = Some(now + STATUS_SOUND_INTERVAL); + if let Some(cue) = self.machine.tick(conversation_audio_audible) { + if let Err(message) = self.player.play(cue, self.output_device.as_deref()) { + self.playback_available = false; + return Err(message); + } + } + Ok(()) + } + + pub fn clear(&mut self) { + self.machine.clear(); + self.next_tick = None; + self.player.stop(); + self.conversation_audio_audible = false; + } +} + +#[cfg(not(target_os = "macos"))] +#[derive(Default)] +struct StatusSoundPlayer; + +#[cfg(not(target_os = "macos"))] +impl StatusSoundPlayer { + fn play(&mut self, _cue: StatusSoundCue, _output_device: Option<&str>) -> Result<(), String> { + Err("status sound playback is only available on macOS".into()) + } + + fn reap(&mut self) {} + + fn stop(&mut self) {} +} + +#[cfg(target_os = "macos")] +struct StatusSoundPlayer { + working: Result, + waiting: Result, + active: Vec, +} + +#[cfg(target_os = "macos")] +impl Default for StatusSoundPlayer { + fn default() -> Self { + Self { + working: load_system_sound("Pop"), + waiting: load_system_sound("Purr"), + active: Vec::new(), + } + } +} + +#[cfg(target_os = "macos")] +impl StatusSoundPlayer { + fn play(&mut self, cue: StatusSoundCue, output_device: Option<&str>) -> Result<(), String> { + let asset = match cue.status { + ConversationStatus::Working => &self.working, + ConversationStatus::Waiting => &self.waiting, + } + .as_ref() + .map_err(Clone::clone)?; + let player = crate::macos_audio_output::PocketAudioPlayer::new( + asset.sample_rate, + 1.0, + output_device, + )?; + let samples = asset + .samples + .iter() + .map(|sample| sample * cue.volume) + .collect::>(); + player.enqueue(&samples)?; + self.active.push(player); + Ok(()) + } + + fn reap(&mut self) { + self.active.retain(|player| !player.is_empty()); + } + + fn stop(&mut self) { + for player in self.active.drain(..) { + player.stop(); + } + } +} + +#[cfg(target_os = "macos")] +struct StatusSoundAsset { + sample_rate: u32, + samples: Vec, +} + +#[cfg(target_os = "macos")] +fn load_system_sound(name: &str) -> Result { + use std::process::Command; + + let source = format!("/System/Library/Sounds/{name}.aiff"); + let directory = + tempfile::tempdir().map_err(|error| format!("could not prepare status sound: {error}"))?; + let output = directory.path().join("status.wav"); + let destination = output.to_str().ok_or("status sound path is not UTF-8")?; + let result = Command::new("/usr/bin/afconvert") + .args([ + source.as_str(), + destination, + "-d", + "LEI16@22050", + "-c", + "1", + "-f", + "WAVE", + ]) + .output() + .map_err(|error| format!("could not convert {name} status sound: {error}"))?; + if !result.status.success() { + return Err(format!("could not convert {name} status sound")); + } + let mut reader = hound::WavReader::open(&output) + .map_err(|error| format!("could not open {name} status sound: {error}"))?; + let spec = reader.spec(); + if spec.channels != 1 || spec.bits_per_sample != 16 { + return Err(format!( + "converted {name} status sound has an unsupported format" + )); + } + let samples = reader + .samples::() + .map(|sample| sample.map(|sample| f32::from(sample) / f32::from(i16::MAX))) + .collect::, _>>() + .map_err(|error| format!("could not decode {name} status sound: {error}"))?; + if samples.is_empty() { + return Err(format!("converted {name} status sound is empty")); + } + Ok(StatusSoundAsset { + sample_rate: spec.sample_rate, + samples, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn settings(mode: StatusSoundMode) -> StatusSoundSettings { + StatusSoundSettings { mode, volume: 0.4 } + } + + #[test] + fn defaults_to_continuous_while_working() { + assert_eq!( + StatusSoundSettings::default(), + settings(StatusSoundMode::ContinuousWhileWorking) + ); + } + + #[test] + fn continuous_plays_every_tick() { + let mut machine = StatusSoundStateMachine::default(); + machine.update( + ConversationStatus::Waiting, + settings(StatusSoundMode::Continuous), + ); + assert!(machine.tick(false).is_some()); + assert!(machine.tick(false).is_some()); + } + + #[test] + fn continuous_while_working_repeats_working_and_plays_waiting_once() { + let mut machine = StatusSoundStateMachine::default(); + let settings = settings(StatusSoundMode::ContinuousWhileWorking); + machine.update(ConversationStatus::Working, settings); + assert_eq!( + machine.tick(false).unwrap().status, + ConversationStatus::Working + ); + assert_eq!( + machine.tick(false).unwrap().status, + ConversationStatus::Working + ); + machine.update(ConversationStatus::Waiting, settings); + assert_eq!( + machine.tick(false).unwrap().status, + ConversationStatus::Waiting + ); + assert_eq!(machine.tick(false), None); + } + + #[test] + fn once_plays_only_when_status_differs_from_last_cue() { + let mut machine = StatusSoundStateMachine::default(); + let settings = settings(StatusSoundMode::Once); + machine.update(ConversationStatus::Working, settings); + assert!(machine.tick(false).is_some()); + assert_eq!(machine.tick(false), None); + machine.update(ConversationStatus::Waiting, settings); + assert!(machine.tick(false).is_some()); + assert_eq!(machine.tick(false), None); + } + + #[test] + fn off_never_plays() { + let mut machine = StatusSoundStateMachine::default(); + machine.update(ConversationStatus::Working, settings(StatusSoundMode::Off)); + assert_eq!(machine.tick(false), None); + } + + #[test] + fn audible_conversation_audio_suppresses_without_consuming_cue() { + let mut machine = StatusSoundStateMachine::default(); + machine.update(ConversationStatus::Waiting, settings(StatusSoundMode::Once)); + assert_eq!(machine.tick(true), None); + assert!(machine.tick(false).is_some()); + } + + #[test] + fn no_status_event_means_no_startup_cue() { + assert_eq!(StatusSoundStateMachine::default().tick(false), None); + } + + #[test] + fn validates_volume() { + for volume in [f32::NAN, f32::INFINITY, -0.1, 1.1] { + assert!(StatusSoundSettings { + mode: StatusSoundMode::Once, + volume + } + .validate() + .is_err()); + } + assert!(StatusSoundSettings { + mode: StatusSoundMode::Once, + volume: 1.0 + } + .validate() + .is_ok()); + } + + #[test] + fn duplicate_updates_preserve_the_existing_cadence() { + let mut runtime = StatusSoundRuntime::default(); + runtime.playback_available = false; + let settings = settings(StatusSoundMode::Continuous); + runtime.update(ConversationStatus::Working, settings); + let deadline = runtime.next_tick; + runtime.update(ConversationStatus::Working, settings); + assert_eq!(runtime.next_tick, deadline); + } + + #[test] + fn changed_updates_restart_the_cadence() { + let mut runtime = StatusSoundRuntime::default(); + runtime.playback_available = false; + runtime.update( + ConversationStatus::Working, + settings(StatusSoundMode::Continuous), + ); + runtime.next_tick = Some(Instant::now() + Duration::from_secs(60)); + runtime.update( + ConversationStatus::Waiting, + settings(StatusSoundMode::Continuous), + ); + assert!(runtime.next_tick.unwrap() < Instant::now() + Duration::from_secs(1)); + } + + #[cfg(target_os = "macos")] + #[test] + #[ignore = "opens the default CoreAudio output and plays the macOS Pop and Purr cues"] + fn macos_player_decodes_and_queues_both_status_cues() { + let mut player = StatusSoundPlayer::default(); + for status in [ConversationStatus::Working, ConversationStatus::Waiting] { + player + .play( + StatusSoundCue { + status, + volume: DEFAULT_STATUS_SOUND_VOLUME, + }, + None, + ) + .unwrap(); + } + assert_eq!(player.active.len(), 2); + player.stop(); + } +} diff --git a/src-tauri/crates/berd-voice/tests/session_protocol.rs b/src-tauri/crates/berd-voice/tests/session_protocol.rs index 3b1f66c21..0f42395de 100644 --- a/src-tauri/crates/berd-voice/tests/session_protocol.rs +++ b/src-tauri/crates/berd-voice/tests/session_protocol.rs @@ -2510,7 +2510,7 @@ fn siri_session_reaches_ready_without_openai_credentials() { stdin.flush().unwrap(); let ready = receive(); assert_eq!(ready["type"], "ready"); - assert_eq!(ready["protocol"], 4); + assert_eq!(ready["protocol"], 5); assert_eq!(ready["session"]["tts"]["backend"], "siri"); assert_eq!(ready["session"]["tts"]["voice"], voice); assert_eq!(ready["session"]["tts"]["language"], language); @@ -2519,6 +2519,29 @@ fn siri_session_reaches_ready_without_openai_credentials() { ready["session"]["input_during_tts"], json!({"revision":1,"policy":"allow_barge_in"}) ); + assert_eq!( + ready["session"]["status_sounds"], + json!({"mode":"continuous-while-working","volume":0.4}) + ); + write_session_json( + &mut stdin, + &json!({ + "type":"set_conversation_status", + "id":19, + "status":"working", + "settings":{"mode":"once","volume":0.25} + }), + ); + stdin.flush().unwrap(); + assert_eq!( + receive(), + json!({ + "type":"conversation_status_applied", + "id":19, + "status":"working", + "settings":{"mode":"once","volume":0.25} + }) + ); write_session_json( &mut stdin, &json!({ From 28dfba0a31193f06ee4a5e6f7cc5cc1eaeb7a83c Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Tue, 8 Sep 2026 22:09:23 -0400 Subject: [PATCH 02/29] refactor(voice): tighten status sound boundaries Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-voice/PROTOCOL.md | 20 +++--- .../berd-voice/native/siri_tts_bridge.h | 10 +++ .../berd-voice/native/siri_tts_bridge.m | 48 +++++++++++++ src-tauri/crates/berd-voice/src/lib.rs | 5 +- .../berd-voice/src/macos_audio_output.rs | 31 +++++++++ src-tauri/crates/berd-voice/src/main.rs | 36 ++++++---- src-tauri/crates/berd-voice/src/protocol.rs | 9 +-- .../crates/berd-voice/src/status_sounds.rs | 68 ++++--------------- .../berd-voice/tests/session_protocol.rs | 4 -- 9 files changed, 139 insertions(+), 92 deletions(-) diff --git a/src-tauri/crates/berd-voice/PROTOCOL.md b/src-tauri/crates/berd-voice/PROTOCOL.md index 6d52f233e..a8aa7d6bf 100644 --- a/src-tauri/crates/berd-voice/PROTOCOL.md +++ b/src-tauri/crates/berd-voice/PROTOCOL.md @@ -47,12 +47,10 @@ has no device-owning or stdout-multiplexed fallback. The first request must be `hello`. `input_during_tts` is the host's resolved initial policy; a host-specific `auto` mode must be resolved before the request. -`status_sounds` is the host's persisted effective preference. `status_sound_output_device` -is the host's selected playback-device name; omitting it uses the system default. -Omitting `status_sounds` uses `continuous-while-working` at volume `0.4`: +`status_sound_output_device` is the host's selected playback-device name; omitting it uses the system default: ```json -{"type":"hello","id":1,"input_during_tts":"allow_barge_in","status_sounds":{"mode":"continuous-while-working","volume":0.4},"status_sound_output_device":"MacBook Pro Speakers"} +{"type":"hello","id":1,"input_during_tts":"allow_barge_in","status_sound_output_device":"MacBook Pro Speakers"} ``` The response uses `protocol:5` as the exact session message-set version. The @@ -60,7 +58,7 @@ parent must reject a version it does not support; the JSON version is independen of the fixed binary framing marker described below: ```json -{"type":"ready","id":1,"protocol":5,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"status_sounds":{"mode":"continuous-while-working","volume":0.4}}} +{"type":"ready","id":1,"protocol":5,"session":{"tts":{"revision":1,"backend":"siri","voice":"Aaron","language":"en-US","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}} ``` The `session.tts` object is the authoritative, sanitized TTS configuration. @@ -70,9 +68,9 @@ and `rate`. Credentials, endpoints, and bundle paths never appear on stdout. Detailed backend errors are diagnostics on stderr only; protocol rejection and fatal messages are sanitized at the stdout boundary. `session.input_during_tts` is the authoritative effective assistant-input -policy and has its own revision. `session.status_sounds` is the host-provided -effective mode and volume. The runtime owns cue cadence and playback; it does not -persist preferences. +policy and has its own revision. Status-sound settings are per-update parameters, +not session snapshot configuration. The runtime owns cue cadence and playback; it +does not persist preferences. ## Stdin framing @@ -187,7 +185,7 @@ it never admits a replacement while old host audio may still be active. ## Parent requests ```text -{"type":"hello","id":u64,"input_during_tts":"allow_barge_in"|"suppress_input","status_sounds":StatusSoundSettings} +{"type":"hello","id":u64,"input_during_tts":"allow_barge_in"|"suppress_input","status_sound_output_device":string?} {"type":"set_paused","active":bool} {"type":"set_input_muted","id":u64,"active":bool} {"type":"set_conversation_status","id":u64,"status":"working"|"waiting","settings":StatusSoundSettings} @@ -213,9 +211,7 @@ through `1`. No cue is emitted until the first `set_conversation_status` request The runtime then ticks immediately and every five seconds. `continuous` emits the current cue every tick; `continuous-while-working` repeats working and emits waiting once; `once` emits only when the requested status differs from the last -emitted cue; `off` emits nothing. Audible user or assistant conversation audio -suppresses a tick without consuming its pending cue. On macOS, working uses the -system Pop sound and waiting uses Purr through the native PCM player. The applied +emitted cue; `off` emits nothing. Active user input, pending recognition, or assistant output suppresses a tick without consuming its pending cue. On macOS, working uses the system Pop sound and waiting uses Purr through the native PCM player. The applied request is acknowledged with: ```text diff --git a/src-tauri/crates/berd-voice/native/siri_tts_bridge.h b/src-tauri/crates/berd-voice/native/siri_tts_bridge.h index 153c51815..7db985d66 100644 --- a/src-tauri/crates/berd-voice/native/siri_tts_bridge.h +++ b/src-tauri/crates/berd-voice/native/siri_tts_bridge.h @@ -101,6 +101,16 @@ bool berd_siri_tts_speak( char **error_out ); +/// Decodes an audio file through AVFoundation into malloc-owned mono Float32 PCM. +/// The caller releases successful samples with `berd_audio_free_samples`. +float *berd_audio_file_load_mono_pcm( + const char *path, + uint32_t *sample_rate_out, + uint32_t *frame_count_out, + char **error_out +); +void berd_audio_free_samples(float *samples); + /// Opaque Pocket PCM player backed by AVAudioUnitTimePitch. Samples are /// mono, noninterleaved float PCM. Device ID 0 uses the system default. void *berd_pocket_audio_player_create( diff --git a/src-tauri/crates/berd-voice/native/siri_tts_bridge.m b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m index ec80eb370..c5b4b77f2 100644 --- a/src-tauri/crates/berd-voice/native/siri_tts_bridge.m +++ b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m @@ -1604,6 +1604,54 @@ bool berd_siri_tts_speak( } } +float *berd_audio_file_load_mono_pcm( + const char *pathValue, + uint32_t *sampleRateOut, + uint32_t *frameCountOut, + char **errorOut +) { + @autoreleasepool { + if (errorOut) *errorOut = NULL; + if (!pathValue || !sampleRateOut || !frameCountOut) { + BerdSetError(errorOut, BerdError(38, @"An audio path and output pointers are required.")); + return NULL; + } + NSURL *url = [NSURL fileURLWithPath:[NSString stringWithUTF8String:pathValue]]; + NSError *error = nil; + AVAudioFile *file = [[AVAudioFile alloc] initForReading:url error:&error]; + if (!file) { + BerdSetError(errorOut, error ?: BerdError(39, @"Could not open the audio file.")); + return NULL; + } + AVAudioFormat *format = file.processingFormat; + AVAudioFrameCount capacity = (AVAudioFrameCount)file.length; + AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] + initWithPCMFormat:format frameCapacity:capacity]; + if (!buffer || ![file readIntoBuffer:buffer error:&error]) { + BerdSetError(errorOut, error ?: BerdError(40, @"Could not decode the audio file.")); + return NULL; + } + uint32_t frameCount = buffer.frameLength; + if (frameCount == 0 || !buffer.floatChannelData) { + BerdSetError(errorOut, BerdError(41, @"The decoded audio file is empty.")); + return NULL; + } + float *samples = malloc((size_t)frameCount * sizeof(float)); + if (!samples) { + BerdSetError(errorOut, BerdError(42, @"Could not allocate decoded audio samples.")); + return NULL; + } + memcpy(samples, buffer.floatChannelData[0], (size_t)frameCount * sizeof(float)); + *sampleRateOut = (uint32_t)format.sampleRate; + *frameCountOut = frameCount; + return samples; + } +} + +void berd_audio_free_samples(float *samples) { + free(samples); +} + void *berd_pocket_audio_player_create( uint32_t sampleRate, float rate, diff --git a/src-tauri/crates/berd-voice/src/lib.rs b/src-tauri/crates/berd-voice/src/lib.rs index b85dfe41c..9eb7d059d 100644 --- a/src-tauri/crates/berd-voice/src/lib.rs +++ b/src-tauri/crates/berd-voice/src/lib.rs @@ -32,7 +32,7 @@ pub mod realtime_pipe; pub mod session; pub mod siri; pub mod spokesperson_voice_update; -pub mod status_sounds; +mod status_sounds; mod synthesis; mod tts; @@ -55,6 +55,9 @@ pub use pocket::{ }; #[cfg(target_os = "macos")] pub use siri::SiriTts; +pub use status_sounds::{ + ConversationStatus, StatusSoundMode, StatusSoundRuntime, StatusSoundSettings, +}; pub use synthesis::{synthesize_pcm16_wav, WavSynthesis, WavSynthesisError, WavSynthesisErrorKind}; pub use tts::{ OpenAiTts, PocketTtsBackend, StreamingTextChunk, StreamingTextChunks, StreamingTtsText, diff --git a/src-tauri/crates/berd-voice/src/macos_audio_output.rs b/src-tauri/crates/berd-voice/src/macos_audio_output.rs index f48a6eeaa..7e02c8a99 100644 --- a/src-tauri/crates/berd-voice/src/macos_audio_output.rs +++ b/src-tauri/crates/berd-voice/src/macos_audio_output.rs @@ -5,6 +5,13 @@ use std::ffi::{c_char, c_void, CStr}; use crate::PcmAudioOutput; unsafe extern "C" { + fn berd_audio_file_load_mono_pcm( + path: *const c_char, + sample_rate_out: *mut u32, + frame_count_out: *mut u32, + error_out: *mut *mut c_char, + ) -> *mut f32; + fn berd_audio_free_samples(samples: *mut f32); fn berd_pocket_audio_player_create( sample_rate: u32, rate: f32, @@ -25,6 +32,30 @@ unsafe extern "C" { fn berd_siri_tts_free_string(value: *mut c_char); } +pub(crate) fn load_mono_audio_file(path: &str) -> Result<(u32, Vec), String> { + let path = std::ffi::CString::new(path).map_err(|_| "audio path contains NUL".to_string())?; + let mut sample_rate = 0; + let mut frame_count = 0; + let mut error = std::ptr::null_mut(); + // SAFETY: The bridge copies the path and returns an owned allocation with + // the reported frame count, released below by its paired free function. + let raw = unsafe { + berd_audio_file_load_mono_pcm( + path.as_ptr(), + &mut sample_rate, + &mut frame_count, + &mut error, + ) + }; + if raw.is_null() { + return Err(take_error(error, "Could not decode audio file")); + } + // SAFETY: A successful bridge call returns exactly `frame_count` initialized samples. + let samples = unsafe { std::slice::from_raw_parts(raw, frame_count as usize) }.to_vec(); + unsafe { berd_audio_free_samples(raw) }; + Ok((sample_rate, samples)) +} + pub struct PocketAudioPlayer { raw: *mut c_void, delivery_safety_frames: u64, diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index a99dba9b7..68c273695 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -46,7 +46,7 @@ use berd_voice::spokesperson_voice_update::{ validate_voice_update_settings, VoiceBarrierAction, VoiceUpdateAction, VoiceUpdatePurpose, VoiceUpdateQueue, VoiceUpdateRequest, VoiceUpdateTransaction, }; -use berd_voice::status_sounds::StatusSoundRuntime; +use berd_voice::StatusSoundRuntime; use berd_voice::{ estimated_spoken_through_utf8, local_assets::{ @@ -1236,6 +1236,14 @@ fn run_management_command(command: ManagementCommand) -> Result<(), ManagementFa } } +fn conversation_activity_suppresses_status_cues( + input_active: bool, + recognition_pending: bool, + output_active: bool, +) -> bool { + input_active || recognition_pending || output_active +} + fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String> { let (control_tx, control_rx) = mpsc::channel(); let (pcm_tx, pcm_rx) = mpsc::sync_channel(INPUT_QUEUE_CAPACITY); @@ -1330,9 +1338,12 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String }, )?; } - if let Err(message) = status_sound_runtime - .poll(core.user_speaking() || core.recognition_pending() || active.is_some()) - { + let conversation_active = conversation_activity_suppresses_status_cues( + core.user_speaking(), + core.recognition_pending(), + active.is_some(), + ); + if let Err(message) = status_sound_runtime.poll(conversation_active) { eprintln!("status sound playback disabled: {message}"); } @@ -1431,7 +1442,6 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String Input::Request(SessionRequest::Hello { id, input_during_tts, - status_sounds, status_sound_output_device, }) => { if initialized { @@ -1488,7 +1498,6 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let session = VoiceSessionSnapshot { tts: slot.snapshot()?, input_during_tts: input_policy.snapshot()?, - status_sounds, }; tts_slot = Some(slot); input_during_tts_slot = Some(input_policy); @@ -3327,9 +3336,12 @@ fn run_expert_spokesperson_session( )?; } } - if let Err(message) = - status_sound_runtime.poll(turn_gate.input_blocks_output() || active.is_some()) - { + let conversation_active = conversation_activity_suppresses_status_cues( + turn_gate.input_blocks_output(), + false, + active.is_some(), + ); + if let Err(message) = status_sound_runtime.poll(conversation_active) { eprintln!("status sound playback disabled: {message}"); } @@ -3425,7 +3437,6 @@ fn run_expert_spokesperson_session( Input::Request(SessionRequest::Hello { id, input_during_tts, - status_sounds, status_sound_output_device, }) => { if initialized { @@ -3450,7 +3461,6 @@ fn run_expert_spokesperson_session( let snapshot = VoiceSessionSnapshot { tts: tts.clone(), input_during_tts: input_policy.snapshot()?, - status_sounds, }; let (created, events) = OpenAiSpokespersonRuntime::spawn_observed(spokesperson_config.clone())?; @@ -6573,8 +6583,7 @@ fn validate_request(request: SessionRequest) -> Result { return Err("request id must be positive".into()); } match &request { - SessionRequest::Hello { status_sounds, .. } - | SessionRequest::SetConversationStatus { + SessionRequest::SetConversationStatus { settings: status_sounds, .. } => { @@ -8183,7 +8192,6 @@ mod tests { session: VoiceSessionSnapshot { tts: snapshot.clone(), input_during_tts: test_input_policy(), - status_sounds: Default::default(), }, }) .unwrap(); diff --git a/src-tauri/crates/berd-voice/src/protocol.rs b/src-tauri/crates/berd-voice/src/protocol.rs index 4cf30e2d7..c2103889c 100644 --- a/src-tauri/crates/berd-voice/src/protocol.rs +++ b/src-tauri/crates/berd-voice/src/protocol.rs @@ -3,8 +3,7 @@ use serde::{Deserialize, Serialize}; use crate::{ input::{InputDuringTtsPolicy, InputDuringTtsSnapshot}, openai_realtime_protocol::RealtimeExpertDeliveryEvent, - status_sounds::{ConversationStatus, StatusSoundSettings}, - TtsConfigurationSnapshot, TtsSettings, + ConversationStatus, StatusSoundSettings, TtsConfigurationSnapshot, TtsSettings, }; #[derive(Clone, Debug, Deserialize, PartialEq)] @@ -14,8 +13,6 @@ pub enum SessionRequest { id: u64, input_during_tts: InputDuringTtsPolicy, #[serde(default)] - status_sounds: StatusSoundSettings, - #[serde(default)] status_sound_output_device: Option, }, SetConversationStatus { @@ -162,7 +159,6 @@ pub enum OutputReadyOutcome { pub struct VoiceSessionSnapshot { pub tts: TtsConfigurationSnapshot, pub input_during_tts: InputDuringTtsSnapshot, - pub status_sounds: StatusSoundSettings, } #[derive(Clone, Copy, Debug, Serialize, PartialEq, Eq)] @@ -371,11 +367,10 @@ mod tests { revision: 1, policy: InputDuringTtsPolicy::AllowBargeIn, }, - status_sounds: StatusSoundSettings::default(), }, }) .unwrap(), - r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"},"status_sounds":{"mode":"continuous-while-working","volume":0.4}}}"# + r#"{"type":"ready","id":4,"protocol":5,"session":{"tts":{"revision":1,"backend":"openai","model":"gpt-4o-mini-tts","voice":"marin","rate":1.0},"input_during_tts":{"revision":1,"policy":"allow_barge_in"}}}"# ); assert_eq!( serde_json::from_str::( diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index edf4471ab..ba9c233fd 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -54,7 +54,7 @@ pub struct StatusSoundCue { } /// Pure policy for deciding which cue, if any, a fixed-cadence runtime tick plays. -/// The host owns the timer and supplies whether conversation audio is currently audible. +/// The host owns the timer and supplies whether conversation activity should suppress cues. #[derive(Debug, Default)] pub struct StatusSoundStateMachine { current: Option<(ConversationStatus, StatusSoundSettings)>, @@ -62,13 +62,16 @@ pub struct StatusSoundStateMachine { } impl StatusSoundStateMachine { - pub fn update(&mut self, status: ConversationStatus, settings: StatusSoundSettings) { - self.current = Some((status, settings)); + pub fn update(&mut self, status: ConversationStatus, settings: StatusSoundSettings) -> bool { + let next = (status, settings); + let changed = self.current != Some(next); + self.current = Some(next); + changed } - pub fn tick(&mut self, conversation_audio_audible: bool) -> Option { + pub fn tick(&mut self, conversation_active: bool) -> Option { let (status, settings) = self.current?; - if conversation_audio_audible || settings.mode == StatusSoundMode::Off { + if conversation_active || settings.mode == StatusSoundMode::Off { return None; } let should_play = match settings.mode { @@ -103,7 +106,6 @@ pub struct StatusSoundRuntime { player: StatusSoundPlayer, output_device: Option, playback_available: bool, - conversation_audio_audible: bool, } impl Default for StatusSoundRuntime { @@ -114,7 +116,6 @@ impl Default for StatusSoundRuntime { player: StatusSoundPlayer::default(), output_device: None, playback_available: cfg!(target_os = "macos"), - conversation_audio_audible: false, } } } @@ -128,19 +129,17 @@ impl StatusSoundRuntime { } pub fn update(&mut self, status: ConversationStatus, settings: StatusSoundSettings) { - let changed = self.machine.current != Some((status, settings)); - self.machine.update(status, settings); + let changed = self.machine.update(status, settings); if changed { self.player.stop(); self.next_tick = Some(Instant::now()); } } - pub fn poll(&mut self, conversation_audio_audible: bool) -> Result<(), String> { - if conversation_audio_audible && !self.conversation_audio_audible { + pub fn poll(&mut self, conversation_active: bool) -> Result<(), String> { + if conversation_active { self.player.stop(); } - self.conversation_audio_audible = conversation_audio_audible; if !self.playback_available { return Ok(()); } @@ -150,7 +149,7 @@ impl StatusSoundRuntime { return Ok(()); } self.next_tick = Some(now + STATUS_SOUND_INTERVAL); - if let Some(cue) = self.machine.tick(conversation_audio_audible) { + if let Some(cue) = self.machine.tick(conversation_active) { if let Err(message) = self.player.play(cue, self.output_device.as_deref()) { self.playback_available = false; return Err(message); @@ -163,7 +162,6 @@ impl StatusSoundRuntime { self.machine.clear(); self.next_tick = None; self.player.stop(); - self.conversation_audio_audible = false; } } @@ -243,47 +241,11 @@ struct StatusSoundAsset { #[cfg(target_os = "macos")] fn load_system_sound(name: &str) -> Result { - use std::process::Command; - let source = format!("/System/Library/Sounds/{name}.aiff"); - let directory = - tempfile::tempdir().map_err(|error| format!("could not prepare status sound: {error}"))?; - let output = directory.path().join("status.wav"); - let destination = output.to_str().ok_or("status sound path is not UTF-8")?; - let result = Command::new("/usr/bin/afconvert") - .args([ - source.as_str(), - destination, - "-d", - "LEI16@22050", - "-c", - "1", - "-f", - "WAVE", - ]) - .output() - .map_err(|error| format!("could not convert {name} status sound: {error}"))?; - if !result.status.success() { - return Err(format!("could not convert {name} status sound")); - } - let mut reader = hound::WavReader::open(&output) - .map_err(|error| format!("could not open {name} status sound: {error}"))?; - let spec = reader.spec(); - if spec.channels != 1 || spec.bits_per_sample != 16 { - return Err(format!( - "converted {name} status sound has an unsupported format" - )); - } - let samples = reader - .samples::() - .map(|sample| sample.map(|sample| f32::from(sample) / f32::from(i16::MAX))) - .collect::, _>>() + let (sample_rate, samples) = crate::macos_audio_output::load_mono_audio_file(&source) .map_err(|error| format!("could not decode {name} status sound: {error}"))?; - if samples.is_empty() { - return Err(format!("converted {name} status sound is empty")); - } Ok(StatusSoundAsset { - sample_rate: spec.sample_rate, + sample_rate, samples, }) } @@ -389,7 +351,6 @@ mod tests { #[test] fn duplicate_updates_preserve_the_existing_cadence() { let mut runtime = StatusSoundRuntime::default(); - runtime.playback_available = false; let settings = settings(StatusSoundMode::Continuous); runtime.update(ConversationStatus::Working, settings); let deadline = runtime.next_tick; @@ -400,7 +361,6 @@ mod tests { #[test] fn changed_updates_restart_the_cadence() { let mut runtime = StatusSoundRuntime::default(); - runtime.playback_available = false; runtime.update( ConversationStatus::Working, settings(StatusSoundMode::Continuous), diff --git a/src-tauri/crates/berd-voice/tests/session_protocol.rs b/src-tauri/crates/berd-voice/tests/session_protocol.rs index 0f42395de..bfe76e612 100644 --- a/src-tauri/crates/berd-voice/tests/session_protocol.rs +++ b/src-tauri/crates/berd-voice/tests/session_protocol.rs @@ -2519,10 +2519,6 @@ fn siri_session_reaches_ready_without_openai_credentials() { ready["session"]["input_during_tts"], json!({"revision":1,"policy":"allow_barge_in"}) ); - assert_eq!( - ready["session"]["status_sounds"], - json!({"mode":"continuous-while-working","volume":0.4}) - ); write_session_json( &mut stdin, &json!({ From 9b00c9a83be2d659c4a3b683b8b7ebc69754e03e Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Tue, 8 Sep 2026 22:20:35 -0400 Subject: [PATCH 03/29] fix(voice): downmix status sound assets Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- .../berd-voice/native/siri_tts_bridge.m | 66 ++++++++++++++----- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/src-tauri/crates/berd-voice/native/siri_tts_bridge.m b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m index c5b4b77f2..a35fbc9b2 100644 --- a/src-tauri/crates/berd-voice/native/siri_tts_bridge.m +++ b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m @@ -1617,32 +1617,62 @@ bool berd_siri_tts_speak( return NULL; } NSURL *url = [NSURL fileURLWithPath:[NSString stringWithUTF8String:pathValue]]; - NSError *error = nil; - AVAudioFile *file = [[AVAudioFile alloc] initForReading:url error:&error]; - if (!file) { - BerdSetError(errorOut, error ?: BerdError(39, @"Could not open the audio file.")); + ExtAudioFileRef file = NULL; + OSStatus status = ExtAudioFileOpenURL((__bridge CFURLRef)url, &file); + if (status != noErr || !file) { + BerdSetError(errorOut, BerdError(39, @"Could not open the audio file.")); return NULL; } - AVAudioFormat *format = file.processingFormat; - AVAudioFrameCount capacity = (AVAudioFrameCount)file.length; - AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] - initWithPCMFormat:format frameCapacity:capacity]; - if (!buffer || ![file readIntoBuffer:buffer error:&error]) { - BerdSetError(errorOut, error ?: BerdError(40, @"Could not decode the audio file.")); - return NULL; + AudioStreamBasicDescription sourceFormat = {0}; + UInt32 propertySize = sizeof(sourceFormat); + status = ExtAudioFileGetProperty( + file, kExtAudioFileProperty_FileDataFormat, &propertySize, &sourceFormat); + SInt64 sourceFrames = 0; + propertySize = sizeof(sourceFrames); + if (status == noErr) { + status = ExtAudioFileGetProperty( + file, kExtAudioFileProperty_FileLengthFrames, &propertySize, &sourceFrames); + } + AudioStreamBasicDescription clientFormat = {0}; + clientFormat.mSampleRate = sourceFormat.mSampleRate; + clientFormat.mFormatID = kAudioFormatLinearPCM; + clientFormat.mFormatFlags = kAudioFormatFlagsNativeFloatPacked; + clientFormat.mBytesPerPacket = sizeof(float); + clientFormat.mFramesPerPacket = 1; + clientFormat.mBytesPerFrame = sizeof(float); + clientFormat.mChannelsPerFrame = 1; + clientFormat.mBitsPerChannel = 8 * sizeof(float); + if (status == noErr) { + status = ExtAudioFileSetProperty( + file, kExtAudioFileProperty_ClientDataFormat, + sizeof(clientFormat), &clientFormat); } - uint32_t frameCount = buffer.frameLength; - if (frameCount == 0 || !buffer.floatChannelData) { - BerdSetError(errorOut, BerdError(41, @"The decoded audio file is empty.")); + if (status != noErr || sourceFrames <= 0 || sourceFrames > UINT32_MAX) { + ExtAudioFileDispose(file); + BerdSetError(errorOut, BerdError(40, @"Could not prepare the audio file for decoding.")); return NULL; } - float *samples = malloc((size_t)frameCount * sizeof(float)); + uint32_t capacity = (uint32_t)sourceFrames; + float *samples = malloc((size_t)capacity * sizeof(float)); if (!samples) { - BerdSetError(errorOut, BerdError(42, @"Could not allocate decoded audio samples.")); + ExtAudioFileDispose(file); + BerdSetError(errorOut, BerdError(41, @"Could not allocate decoded audio samples.")); + return NULL; + } + AudioBufferList buffers = {0}; + buffers.mNumberBuffers = 1; + buffers.mBuffers[0].mNumberChannels = 1; + buffers.mBuffers[0].mDataByteSize = capacity * sizeof(float); + buffers.mBuffers[0].mData = samples; + UInt32 frameCount = capacity; + status = ExtAudioFileRead(file, &frameCount, &buffers); + ExtAudioFileDispose(file); + if (status != noErr || frameCount == 0) { + free(samples); + BerdSetError(errorOut, BerdError(42, @"Could not decode the audio file.")); return NULL; } - memcpy(samples, buffer.floatChannelData[0], (size_t)frameCount * sizeof(float)); - *sampleRateOut = (uint32_t)format.sampleRate; + *sampleRateOut = (uint32_t)clientFormat.mSampleRate; *frameCountOut = frameCount; return samples; } From d666e008444251369462926ac6794901cd44649d Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Tue, 8 Sep 2026 22:31:02 -0400 Subject: [PATCH 04/29] refactor(voice): simplify status cue policy Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- .../berd-voice/native/siri_tts_bridge.m | 22 ++++++++++++++----- .../berd-voice/src/macos_audio_output.rs | 2 +- src-tauri/crates/berd-voice/src/main.rs | 20 +++++++++++------ .../crates/berd-voice/src/status_sounds.rs | 13 +---------- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src-tauri/crates/berd-voice/native/siri_tts_bridge.m b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m index a35fbc9b2..8b709f3df 100644 --- a/src-tauri/crates/berd-voice/native/siri_tts_bridge.m +++ b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m @@ -1637,10 +1637,11 @@ bool berd_siri_tts_speak( clientFormat.mSampleRate = sourceFormat.mSampleRate; clientFormat.mFormatID = kAudioFormatLinearPCM; clientFormat.mFormatFlags = kAudioFormatFlagsNativeFloatPacked; - clientFormat.mBytesPerPacket = sizeof(float); + uint32_t channelCount = sourceFormat.mChannelsPerFrame; + clientFormat.mBytesPerPacket = sizeof(float) * channelCount; clientFormat.mFramesPerPacket = 1; - clientFormat.mBytesPerFrame = sizeof(float); - clientFormat.mChannelsPerFrame = 1; + clientFormat.mBytesPerFrame = sizeof(float) * channelCount; + clientFormat.mChannelsPerFrame = channelCount; clientFormat.mBitsPerChannel = 8 * sizeof(float); if (status == noErr) { status = ExtAudioFileSetProperty( @@ -1653,7 +1654,7 @@ bool berd_siri_tts_speak( return NULL; } uint32_t capacity = (uint32_t)sourceFrames; - float *samples = malloc((size_t)capacity * sizeof(float)); + float *samples = malloc((size_t)capacity * channelCount * sizeof(float)); if (!samples) { ExtAudioFileDispose(file); BerdSetError(errorOut, BerdError(41, @"Could not allocate decoded audio samples.")); @@ -1661,8 +1662,8 @@ bool berd_siri_tts_speak( } AudioBufferList buffers = {0}; buffers.mNumberBuffers = 1; - buffers.mBuffers[0].mNumberChannels = 1; - buffers.mBuffers[0].mDataByteSize = capacity * sizeof(float); + buffers.mBuffers[0].mNumberChannels = channelCount; + buffers.mBuffers[0].mDataByteSize = capacity * channelCount * sizeof(float); buffers.mBuffers[0].mData = samples; UInt32 frameCount = capacity; status = ExtAudioFileRead(file, &frameCount, &buffers); @@ -1672,6 +1673,15 @@ bool berd_siri_tts_speak( BerdSetError(errorOut, BerdError(42, @"Could not decode the audio file.")); return NULL; } + if (channelCount > 1) { + for (uint32_t frame = 0; frame < frameCount; frame++) { + float mixed = 0; + for (uint32_t channel = 0; channel < channelCount; channel++) { + mixed += samples[frame * channelCount + channel]; + } + samples[frame] = mixed / channelCount; + } + } *sampleRateOut = (uint32_t)clientFormat.mSampleRate; *frameCountOut = frameCount; return samples; diff --git a/src-tauri/crates/berd-voice/src/macos_audio_output.rs b/src-tauri/crates/berd-voice/src/macos_audio_output.rs index 7e02c8a99..57a2bf73a 100644 --- a/src-tauri/crates/berd-voice/src/macos_audio_output.rs +++ b/src-tauri/crates/berd-voice/src/macos_audio_output.rs @@ -1,4 +1,4 @@ -//! Safe ownership wrapper for the shared macOS AVAudioUnitTimePitch PCM player. +//! Safe wrappers for the shared macOS audio FFI boundary. use std::ffi::{c_char, c_void, CStr}; diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index 68c273695..09a95e0e8 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -1236,12 +1236,19 @@ fn run_management_command(command: ManagementCommand) -> Result<(), ManagementFa } } -fn conversation_activity_suppresses_status_cues( - input_active: bool, +fn standard_session_status_cues_suppressed( + user_speaking: bool, recognition_pending: bool, - output_active: bool, + assistant_output_active: bool, ) -> bool { - input_active || recognition_pending || output_active + user_speaking || recognition_pending || assistant_output_active +} + +fn expert_session_status_cues_suppressed( + input_blocks_output: bool, + assistant_output_active: bool, +) -> bool { + input_blocks_output || assistant_output_active } fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String> { @@ -1338,7 +1345,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String }, )?; } - let conversation_active = conversation_activity_suppresses_status_cues( + let conversation_active = standard_session_status_cues_suppressed( core.user_speaking(), core.recognition_pending(), active.is_some(), @@ -3336,9 +3343,8 @@ fn run_expert_spokesperson_session( )?; } } - let conversation_active = conversation_activity_suppresses_status_cues( + let conversation_active = expert_session_status_cues_suppressed( turn_gate.input_blocks_output(), - false, active.is_some(), ); if let Err(message) = status_sound_runtime.poll(conversation_active) { diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index ba9c233fd..2112a7279 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -91,11 +91,6 @@ impl StatusSoundStateMachine { volume: settings.volume, }) } - - pub fn clear(&mut self) { - self.current = None; - self.last_played = None; - } } /// Owns status cadence and cue playback for one running voice session. @@ -115,7 +110,7 @@ impl Default for StatusSoundRuntime { next_tick: None, player: StatusSoundPlayer::default(), output_device: None, - playback_available: cfg!(target_os = "macos"), + playback_available: true, } } } @@ -157,12 +152,6 @@ impl StatusSoundRuntime { } Ok(()) } - - pub fn clear(&mut self) { - self.machine.clear(); - self.next_tick = None; - self.player.stop(); - } } #[cfg(not(target_os = "macos"))] From b4815b991b443f9f24194fcc53119b461d20bc64 Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Tue, 8 Sep 2026 22:48:42 -0400 Subject: [PATCH 05/29] fix(voice): isolate status cues from input Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- .../berd-voice/native/siri_tts_bridge.m | 21 +++++---- src-tauri/crates/berd-voice/src/main.rs | 45 ++++++++++++++++--- .../crates/berd-voice/src/status_sounds.rs | 29 +++++++----- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/src-tauri/crates/berd-voice/native/siri_tts_bridge.m b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m index 8b709f3df..74404c1f6 100644 --- a/src-tauri/crates/berd-voice/native/siri_tts_bridge.m +++ b/src-tauri/crates/berd-voice/native/siri_tts_bridge.m @@ -1660,15 +1660,20 @@ bool berd_siri_tts_speak( BerdSetError(errorOut, BerdError(41, @"Could not allocate decoded audio samples.")); return NULL; } - AudioBufferList buffers = {0}; - buffers.mNumberBuffers = 1; - buffers.mBuffers[0].mNumberChannels = channelCount; - buffers.mBuffers[0].mDataByteSize = capacity * channelCount * sizeof(float); - buffers.mBuffers[0].mData = samples; - UInt32 frameCount = capacity; - status = ExtAudioFileRead(file, &frameCount, &buffers); + uint32_t frameCount = 0; + while (frameCount < capacity) { + UInt32 requestedFrames = capacity - frameCount; + AudioBufferList buffers = {0}; + buffers.mNumberBuffers = 1; + buffers.mBuffers[0].mNumberChannels = channelCount; + buffers.mBuffers[0].mDataByteSize = requestedFrames * channelCount * sizeof(float); + buffers.mBuffers[0].mData = samples + ((size_t)frameCount * channelCount); + status = ExtAudioFileRead(file, &requestedFrames, &buffers); + if (status != noErr || requestedFrames == 0) break; + frameCount += requestedFrames; + } ExtAudioFileDispose(file); - if (status != noErr || frameCount == 0) { + if (status != noErr || frameCount != capacity) { free(samples); BerdSetError(errorOut, BerdError(42, @"Could not decode the audio file.")); return NULL; diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index 09a95e0e8..499bfeba4 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -17,8 +17,8 @@ use berd_voice::benchmark::{ }; use berd_voice::expert_spokesperson::{ExpertDirectiveOutcome, LiveSideEvent}; use berd_voice::input::{ - AssistantActivityGuard, InputDuringTtsSlot, InputDuringTtsSnapshot, VoiceInputConfig, - VoiceInputControls, VoiceInputEngineConfig, VoiceInputEvent, VoiceInputFrame, + AssistantActivityGuard, InputDuringTtsPolicy, InputDuringTtsSlot, InputDuringTtsSnapshot, + VoiceInputConfig, VoiceInputControls, VoiceInputEngineConfig, VoiceInputEvent, VoiceInputFrame, VoiceInputRuntime, INPUT_FRAME_SAMPLES, }; use berd_voice::openai_realtime_protocol::{ @@ -1276,6 +1276,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let mut held: Option = None; let mut active: Option = None; let mut status_sound_runtime = StatusSoundRuntime::default(); + let mut status_sound_activity: Option = None; loop { if let Some(events) = input_events.as_mut() { @@ -1350,8 +1351,20 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String core.recognition_pending(), active.is_some(), ); - if let Err(message) = status_sound_runtime.poll(conversation_active) { - eprintln!("status sound playback disabled: {message}"); + match status_sound_runtime.poll(conversation_active) { + Ok(true) if status_sound_activity.is_none() => { + status_sound_activity = input_controls.as_ref().map(|controls| { + controls + .begin_assistant_activity(0.65, InputDuringTtsPolicy::SuppressInput) + .expect("balanced assistant threshold is valid") + }); + } + Ok(false) => status_sound_activity = None, + Ok(true) => {} + Err(message) => { + status_sound_activity = None; + eprintln!("status sound playback disabled: {message}"); + } } let Some(input) = receive_session_input( @@ -2291,9 +2304,11 @@ fn stage_live_audio_delta( fn spokesperson_pcm_allowed( input_muted: bool, playback_active: bool, + status_sound_active: bool, input_policy: InputDuringTtsSnapshot, ) -> bool { !(input_muted + || status_sound_active || playback_active && input_policy.policy == berd_voice::input::InputDuringTtsPolicy::SuppressInput) } @@ -3347,9 +3362,13 @@ fn run_expert_spokesperson_session( turn_gate.input_blocks_output(), active.is_some(), ); - if let Err(message) = status_sound_runtime.poll(conversation_active) { - eprintln!("status sound playback disabled: {message}"); - } + let status_sound_active = match status_sound_runtime.poll(conversation_active) { + Ok(active) => active, + Err(message) => { + eprintln!("status sound playback disabled: {message}"); + false + } + }; let Some(input) = receive_session_input( &control_rx, @@ -3399,6 +3418,7 @@ fn run_expert_spokesperson_session( if spokesperson_pcm_allowed( input_muted, active.is_some(), + status_sound_active, input_during_tts_slot .as_ref() .expect("hello initialized input policy") @@ -7376,9 +7396,11 @@ mod tests { assert!(!spokesperson_pcm_allowed( false, true, + false, slot.snapshot().unwrap() )); assert!(spokesperson_pcm_allowed( + false, false, false, slot.snapshot().unwrap() @@ -7386,8 +7408,17 @@ mod tests { assert!(!spokesperson_pcm_allowed( true, false, + false, slot.snapshot().unwrap() )); + assert!(!spokesperson_pcm_allowed( + false, + false, + true, + InputDuringTtsSlot::new(InputDuringTtsPolicy::AllowBargeIn) + .snapshot() + .unwrap() + )); let messages = messages(&output); assert_eq!( messages[0], diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 2112a7279..fdf8435ec 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -131,26 +131,25 @@ impl StatusSoundRuntime { } } - pub fn poll(&mut self, conversation_active: bool) -> Result<(), String> { + pub fn poll(&mut self, conversation_active: bool) -> Result { if conversation_active { self.player.stop(); } if !self.playback_available { - return Ok(()); + return Ok(false); } self.player.reap(); let now = Instant::now(); - if self.next_tick.is_none_or(|deadline| now < deadline) { - return Ok(()); - } - self.next_tick = Some(now + STATUS_SOUND_INTERVAL); - if let Some(cue) = self.machine.tick(conversation_active) { - if let Err(message) = self.player.play(cue, self.output_device.as_deref()) { - self.playback_available = false; - return Err(message); + if self.next_tick.is_some_and(|deadline| now >= deadline) { + self.next_tick = Some(now + STATUS_SOUND_INTERVAL); + if let Some(cue) = self.machine.tick(conversation_active) { + if let Err(message) = self.player.play(cue, self.output_device.as_deref()) { + self.playback_available = false; + return Err(message); + } } } - Ok(()) + Ok(self.player.is_active()) } } @@ -166,6 +165,10 @@ impl StatusSoundPlayer { fn reap(&mut self) {} + fn is_active(&self) -> bool { + false + } + fn stop(&mut self) {} } @@ -215,6 +218,10 @@ impl StatusSoundPlayer { self.active.retain(|player| !player.is_empty()); } + fn is_active(&self) -> bool { + !self.active.is_empty() + } + fn stop(&mut self) { for player in self.active.drain(..) { player.stop(); From 4c94f5315cf42773704ad3d0cdb68b096a757d37 Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Tue, 8 Sep 2026 23:00:12 -0400 Subject: [PATCH 06/29] fix(voice): cover status cue output tail Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- .../crates/berd-voice/src/status_sounds.rs | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index fdf8435ec..c58e979e2 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -172,11 +172,20 @@ impl StatusSoundPlayer { fn stop(&mut self) {} } +#[cfg(target_os = "macos")] +const STATUS_SOUND_OUTPUT_TAIL: Duration = Duration::from_millis(100); + +#[cfg(target_os = "macos")] +struct ActiveStatusSound { + player: crate::macos_audio_output::PocketAudioPlayer, + output_tail_deadline: Option, +} + #[cfg(target_os = "macos")] struct StatusSoundPlayer { working: Result, waiting: Result, - active: Vec, + active: Vec, } #[cfg(target_os = "macos")] @@ -210,12 +219,25 @@ impl StatusSoundPlayer { .map(|sample| sample * cue.volume) .collect::>(); player.enqueue(&samples)?; - self.active.push(player); + self.active.push(ActiveStatusSound { + player, + output_tail_deadline: None, + }); Ok(()) } fn reap(&mut self) { - self.active.retain(|player| !player.is_empty()); + let now = Instant::now(); + self.active.retain_mut(|sound| { + if !sound.player.is_empty() { + sound.output_tail_deadline = None; + return true; + } + let deadline = sound + .output_tail_deadline + .get_or_insert(now + STATUS_SOUND_OUTPUT_TAIL); + now < *deadline + }); } fn is_active(&self) -> bool { @@ -223,8 +245,8 @@ impl StatusSoundPlayer { } fn stop(&mut self) { - for player in self.active.drain(..) { - player.stop(); + for sound in self.active.drain(..) { + sound.player.stop(); } } } From c049a3048d2b0e500dcab5337d41ec9053e027a9 Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Tue, 8 Sep 2026 23:12:55 -0400 Subject: [PATCH 07/29] fix(voice): refine status cue input gating Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-voice/src/main.rs | 23 ++++++++++++++----- .../crates/berd-voice/src/status_sounds.rs | 14 +++++++---- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index 499bfeba4..13cca399d 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -1351,7 +1351,13 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String core.recognition_pending(), active.is_some(), ); - match status_sound_runtime.poll(conversation_active) { + let status_sound_result = if conversation_active { + status_sound_runtime.stop(); + Ok(false) + } else { + status_sound_runtime.poll(false) + }; + match status_sound_result { Ok(true) if status_sound_activity.is_none() => { status_sound_activity = input_controls.as_ref().map(|controls| { controls @@ -3362,11 +3368,16 @@ fn run_expert_spokesperson_session( turn_gate.input_blocks_output(), active.is_some(), ); - let status_sound_active = match status_sound_runtime.poll(conversation_active) { - Ok(active) => active, - Err(message) => { - eprintln!("status sound playback disabled: {message}"); - false + let status_sound_active = if conversation_active { + status_sound_runtime.stop(); + false + } else { + match status_sound_runtime.poll(false) { + Ok(active) => active, + Err(message) => { + eprintln!("status sound playback disabled: {message}"); + false + } } }; diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index c58e979e2..23a456e9e 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -131,9 +131,13 @@ impl StatusSoundRuntime { } } + pub fn stop(&mut self) { + self.player.stop(); + } + pub fn poll(&mut self, conversation_active: bool) -> Result { if conversation_active { - self.player.stop(); + self.stop(); } if !self.playback_available { return Ok(false); @@ -143,9 +147,11 @@ impl StatusSoundRuntime { if self.next_tick.is_some_and(|deadline| now >= deadline) { self.next_tick = Some(now + STATUS_SOUND_INTERVAL); if let Some(cue) = self.machine.tick(conversation_active) { - if let Err(message) = self.player.play(cue, self.output_device.as_deref()) { - self.playback_available = false; - return Err(message); + if cue.volume > 0.0 { + if let Err(message) = self.player.play(cue, self.output_device.as_deref()) { + self.playback_available = false; + return Err(message); + } } } } From 11745315e3460f9516023ea1aa3678b8f3ce06fb Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Wed, 9 Sep 2026 09:54:48 -0400 Subject: [PATCH 08/29] fix(voice): construct platform status player directly Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-voice/src/status_sounds.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 23a456e9e..8b5263ec2 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -108,7 +108,7 @@ impl Default for StatusSoundRuntime { Self { machine: StatusSoundStateMachine::default(), next_tick: None, - player: StatusSoundPlayer::default(), + player: StatusSoundPlayer::new(), output_device: None, playback_available: true, } @@ -160,11 +160,14 @@ impl StatusSoundRuntime { } #[cfg(not(target_os = "macos"))] -#[derive(Default)] struct StatusSoundPlayer; #[cfg(not(target_os = "macos"))] impl StatusSoundPlayer { + fn new() -> Self { + Self + } + fn play(&mut self, _cue: StatusSoundCue, _output_device: Option<&str>) -> Result<(), String> { Err("status sound playback is only available on macOS".into()) } @@ -195,18 +198,15 @@ struct StatusSoundPlayer { } #[cfg(target_os = "macos")] -impl Default for StatusSoundPlayer { - fn default() -> Self { +impl StatusSoundPlayer { + fn new() -> Self { Self { working: load_system_sound("Pop"), waiting: load_system_sound("Purr"), active: Vec::new(), } } -} -#[cfg(target_os = "macos")] -impl StatusSoundPlayer { fn play(&mut self, cue: StatusSoundCue, output_device: Option<&str>) -> Result<(), String> { let asset = match cue.status { ConversationStatus::Working => &self.working, @@ -401,7 +401,7 @@ mod tests { #[test] #[ignore = "opens the default CoreAudio output and plays the macOS Pop and Purr cues"] fn macos_player_decodes_and_queues_both_status_cues() { - let mut player = StatusSoundPlayer::default(); + let mut player = StatusSoundPlayer::new(); for status in [ConversationStatus::Working, ConversationStatus::Waiting] { player .play( From 6728e5f1f2912706a9634a6dce34b2b6e93cfa5a Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Wed, 9 Sep 2026 13:28:47 -0400 Subject: [PATCH 09/29] fix(voice): integrate runtime status sounds Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-voice/src/lib.rs | 3 +- src-tauri/crates/berd-voice/src/main.rs | 50 +---- .../crates/berd-voice/src/status_sounds.rs | 88 +++++++- src-tauri/src/commands/native_voice.rs | 207 ++++++++++++------ src-tauri/src/commands/openai_realtime.rs | 47 ++++ src-tauri/src/lib.rs | 2 + .../api/voiceConversation.ts | 23 ++ .../useOpenAiRealtimeConversation.test.ts | 17 ++ .../hooks/useOpenAiRealtimeConversation.ts | 44 +++- .../useVoiceConversationController.test.ts | 54 +++++ .../hooks/useVoiceConversationController.ts | 45 ++++ .../lib/statusSoundPreference.test.ts | 56 +++++ .../lib/statusSoundPreference.ts | 125 +++++++++++ .../ui/VoiceSettings.test.tsx | 19 ++ .../voice-conversation/ui/VoiceSettings.tsx | 70 ++++++ src/shared/api/openaiRealtime.ts | 14 ++ src/shared/i18n/locales/en/settings.json | 16 +- src/shared/i18n/locales/es/settings.json | 16 +- 18 files changed, 779 insertions(+), 117 deletions(-) create mode 100644 src/features/voice-conversation/lib/statusSoundPreference.test.ts create mode 100644 src/features/voice-conversation/lib/statusSoundPreference.ts diff --git a/src-tauri/crates/berd-voice/src/lib.rs b/src-tauri/crates/berd-voice/src/lib.rs index 9eb7d059d..86298f5cc 100644 --- a/src-tauri/crates/berd-voice/src/lib.rs +++ b/src-tauri/crates/berd-voice/src/lib.rs @@ -56,7 +56,8 @@ pub use pocket::{ #[cfg(target_os = "macos")] pub use siri::SiriTts; pub use status_sounds::{ - ConversationStatus, StatusSoundMode, StatusSoundRuntime, StatusSoundSettings, + ConversationStatus, ManagedStatusSoundRuntime, StatusSoundMode, + StatusSoundRuntime, StatusSoundSettings, }; pub use synthesis::{synthesize_pcm16_wav, WavSynthesis, WavSynthesisError, WavSynthesisErrorKind}; pub use tts::{ diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index 13cca399d..278f581d2 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -17,8 +17,8 @@ use berd_voice::benchmark::{ }; use berd_voice::expert_spokesperson::{ExpertDirectiveOutcome, LiveSideEvent}; use berd_voice::input::{ - AssistantActivityGuard, InputDuringTtsPolicy, InputDuringTtsSlot, InputDuringTtsSnapshot, - VoiceInputConfig, VoiceInputControls, VoiceInputEngineConfig, VoiceInputEvent, VoiceInputFrame, + AssistantActivityGuard, InputDuringTtsSlot, InputDuringTtsSnapshot, VoiceInputConfig, + VoiceInputControls, VoiceInputEngineConfig, VoiceInputEvent, VoiceInputFrame, VoiceInputRuntime, INPUT_FRAME_SAMPLES, }; use berd_voice::openai_realtime_protocol::{ @@ -1276,7 +1276,6 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String let mut held: Option = None; let mut active: Option = None; let mut status_sound_runtime = StatusSoundRuntime::default(); - let mut status_sound_activity: Option = None; loop { if let Some(events) = input_events.as_mut() { @@ -1357,20 +1356,8 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String } else { status_sound_runtime.poll(false) }; - match status_sound_result { - Ok(true) if status_sound_activity.is_none() => { - status_sound_activity = input_controls.as_ref().map(|controls| { - controls - .begin_assistant_activity(0.65, InputDuringTtsPolicy::SuppressInput) - .expect("balanced assistant threshold is valid") - }); - } - Ok(false) => status_sound_activity = None, - Ok(true) => {} - Err(message) => { - status_sound_activity = None; - eprintln!("status sound playback disabled: {message}"); - } + if let Err(message) = status_sound_result { + eprintln!("status sound playback disabled: {message}"); } let Some(input) = receive_session_input( @@ -2310,11 +2297,9 @@ fn stage_live_audio_delta( fn spokesperson_pcm_allowed( input_muted: bool, playback_active: bool, - status_sound_active: bool, input_policy: InputDuringTtsSnapshot, ) -> bool { !(input_muted - || status_sound_active || playback_active && input_policy.policy == berd_voice::input::InputDuringTtsPolicy::SuppressInput) } @@ -3368,18 +3353,15 @@ fn run_expert_spokesperson_session( turn_gate.input_blocks_output(), active.is_some(), ); - let status_sound_active = if conversation_active { + let status_sound_result = if conversation_active { status_sound_runtime.stop(); - false + Ok(false) } else { - match status_sound_runtime.poll(false) { - Ok(active) => active, - Err(message) => { - eprintln!("status sound playback disabled: {message}"); - false - } - } + status_sound_runtime.poll(false) }; + if let Err(message) = status_sound_result { + eprintln!("status sound playback disabled: {message}"); + } let Some(input) = receive_session_input( &control_rx, @@ -3429,7 +3411,6 @@ fn run_expert_spokesperson_session( if spokesperson_pcm_allowed( input_muted, active.is_some(), - status_sound_active, input_during_tts_slot .as_ref() .expect("hello initialized input policy") @@ -7407,11 +7388,9 @@ mod tests { assert!(!spokesperson_pcm_allowed( false, true, - false, slot.snapshot().unwrap() )); assert!(spokesperson_pcm_allowed( - false, false, false, slot.snapshot().unwrap() @@ -7419,17 +7398,8 @@ mod tests { assert!(!spokesperson_pcm_allowed( true, false, - false, slot.snapshot().unwrap() )); - assert!(!spokesperson_pcm_allowed( - false, - false, - true, - InputDuringTtsSlot::new(InputDuringTtsPolicy::AllowBargeIn) - .snapshot() - .unwrap() - )); let messages = messages(&output); assert_eq!( messages[0], diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 8b5263ec2..72d2c2448 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -1,4 +1,8 @@ -use std::time::{Duration, Instant}; +use std::{ + sync::mpsc::{self, RecvTimeoutError, Sender}, + thread, + time::{Duration, Instant}, +}; use serde::{Deserialize, Serialize}; @@ -159,6 +163,88 @@ impl StatusSoundRuntime { } } +enum StatusSoundCommand { + Update(ConversationStatus, StatusSoundSettings), + ConversationActive(bool), + OutputDevice(Option), + Shutdown, +} + +/// Thread-safe status-sound service for hosts that do not own a polling loop. +pub struct ManagedStatusSoundRuntime { + commands: Sender, +} + +impl ManagedStatusSoundRuntime { + pub fn spawn(output_device: Option) -> Result { + let (commands, receiver) = mpsc::channel(); + let worker = thread::Builder::new() + .name("berd-status-sounds".into()) + .spawn(move || { + let mut runtime = StatusSoundRuntime::default(); + let mut conversation_active = false; + runtime.set_output_device(output_device); + loop { + match receiver.recv_timeout(Duration::from_millis(10)) { + Ok(StatusSoundCommand::Update(status, settings)) => { + runtime.update(status, settings); + } + Ok(StatusSoundCommand::ConversationActive(active)) => { + conversation_active = active; + } + Ok(StatusSoundCommand::OutputDevice(device)) => { + runtime.set_output_device(device); + } + Ok(StatusSoundCommand::Shutdown) | Err(RecvTimeoutError::Disconnected) => { + runtime.stop(); + break; + } + Err(RecvTimeoutError::Timeout) => {} + } + if let Err(message) = runtime.poll(conversation_active) { + eprintln!("status sound playback disabled: {message}"); + } + } + }) + .map_err(|error| format!("Could not start status sound runtime: {error}"))?; + drop(worker); + Ok(Self { commands }) + } + + pub fn update( + &self, + status: ConversationStatus, + settings: StatusSoundSettings, + ) -> Result<(), String> { + settings.validate().map_err(str::to_string)?; + self.send(StatusSoundCommand::Update(status, settings)) + } + + pub fn set_conversation_active(&self, active: bool) -> Result<(), String> { + self.send(StatusSoundCommand::ConversationActive(active)) + } + + pub fn set_output_device(&self, output_device: Option) -> Result<(), String> { + self.send(StatusSoundCommand::OutputDevice(output_device)) + } + + fn send(&self, command: StatusSoundCommand) -> Result<(), String> { + self.commands + .send(command) + .map_err(|_| "Status sound runtime is unavailable".to_string()) + } + + pub fn finish(&self) { + let _ = self.commands.send(StatusSoundCommand::Shutdown); + } +} + +impl Drop for ManagedStatusSoundRuntime { + fn drop(&mut self) { + let _ = self.commands.send(StatusSoundCommand::Shutdown); + } +} + #[cfg(not(target_os = "macos"))] struct StatusSoundPlayer; diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index b20ef1d9f..1c0e71488 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -77,6 +77,17 @@ pub struct MicrophoneMuteRequest { renderer_epoch: u64, } +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StatusSoundUpdateRequest { + session_id: String, + expected_revision: u64, + status: berd_voice::ConversationStatus, + settings: berd_voice::StatusSoundSettings, + renderer_id: String, + renderer_epoch: u64, +} + #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct AssistantSpeakingRequest { @@ -242,6 +253,10 @@ struct Runtime { controls_visibility_generation: u64, controls_window_revision: Option, native_microphone_mute_control: bool, + status_sounds: Option, + status_sound_user_speaking: bool, + status_sound_recognition_pending: bool, + status_sound_assistant_speaking: bool, admission: Option>, voice_input_quarantined: bool, } @@ -529,6 +544,7 @@ type StopSnapshot = ( Option, u64, Option, + Option, Option<(RuntimeOwner, String)>, ); @@ -997,32 +1013,6 @@ impl NativeVoiceState { Ok(()) } - fn assistant_activity_target( - &self, - caller_window_label: &str, - session_id: &str, - expected_revision: u64, - ) -> Result, String> { - let runtime = self - .runtime - .lock() - .map_err(|_| "native voice state lock was poisoned".to_string())?; - if runtime.session_id.as_deref() != Some(session_id) - || runtime.revision != expected_revision - { - return Ok(None); - } - let owner_window_label = runtime - .owner - .as_ref() - .map(|owner| owner.window_label.clone()) - .ok_or_else(|| "The native voice conversation has no owning window.".to_string())?; - if owner_window_label != caller_window_label { - return Err("Only the voice conversation owner can report assistant activity.".into()); - } - Ok(Some((owner_window_label, runtime.revision))) - } - fn set_assistant_speaking( &self, app: &AppHandle, @@ -1031,10 +1021,34 @@ impl NativeVoiceState { expected_revision: u64, speaking: bool, ) -> Result<(), String> { - let Some((owner_window_label, revision)) = - self.assistant_activity_target(caller_window_label, session_id, expected_revision)? - else { - return Ok(()); + let (owner_window_label, revision) = { + let mut runtime = self + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(session_id) + || runtime.revision != expected_revision + { + return Ok(()); + } + let owner_window_label = runtime + .owner + .as_ref() + .map(|owner| owner.window_label.clone()) + .ok_or_else(|| "The native voice conversation has no owning window.".to_string())?; + if owner_window_label != caller_window_label { + return Err( + "Only the voice conversation owner can report assistant activity.".into(), + ); + } + runtime.status_sound_assistant_speaking = speaking; + let conversation_active = runtime.status_sound_user_speaking + || runtime.status_sound_recognition_pending + || runtime.status_sound_assistant_speaking; + if let Some(status_sounds) = runtime.status_sounds.as_ref() { + status_sounds.set_conversation_active(conversation_active)?; + } + (owner_window_label, runtime.revision) }; let event = NativeVoiceEvent::Activity { session_id: session_id.to_string(), @@ -1052,6 +1066,35 @@ impl NativeVoiceState { Ok(()) } + fn set_status_sound_input_activity( + &self, + session_id: &str, + revision: u64, + user_speaking: Option, + recognition_pending: Option, + ) { + let Ok(mut runtime) = self.runtime.lock() else { + return; + }; + if runtime.session_id.as_deref() != Some(session_id) || runtime.revision != revision { + return; + } + if let Some(speaking) = user_speaking { + runtime.status_sound_user_speaking = speaking; + } + if let Some(pending) = recognition_pending { + runtime.status_sound_recognition_pending = pending; + } + let conversation_active = runtime.status_sound_user_speaking + || runtime.status_sound_recognition_pending + || runtime.status_sound_assistant_speaking; + if let Some(status_sounds) = runtime.status_sounds.as_ref() { + if let Err(error) = status_sounds.set_conversation_active(conversation_active) { + log::warn!("Could not update status sound activity: {error}"); + } + } + } + fn take_stop_snapshot( &self, expected_lifecycle: Option<(&str, u64)>, @@ -1075,6 +1118,7 @@ impl NativeVoiceState { session_id, runtime.revision, runtime.pipeline.take(), + runtime.status_sounds.take(), owner.zip(owner_id), ))) } @@ -1591,6 +1635,14 @@ pub async fn start_native_voice_conversation( window_label: window_label.clone(), }); runtime.pipeline = pipeline.take(); + runtime.status_sounds = berd_voice::ManagedStatusSoundRuntime::spawn( + super::pocket_voice::selected_output_device(), + ) + .map_err(|error| log::warn!("Status sounds unavailable: {error}")) + .ok(); + runtime.status_sound_user_speaking = false; + runtime.status_sound_recognition_pending = false; + runtime.status_sound_assistant_speaking = false; runtime.admission = Some(Arc::new(BerdAdmissionCoordinator::default())); runtime.controls_ready = false; // Voice always starts from its owning session, where the in-session @@ -1707,6 +1759,12 @@ pub async fn start_native_voice_conversation( } berd_voice::input::VoiceInputEvent::SpeakingChanged(speaking) => { admission.set_user_speaking(speaking); + event_state.set_status_sound_input_activity( + &session_id, + revision, + Some(speaking), + None, + ); let event = NativeVoiceEvent::Activity { session_id: session_id.clone(), activity: if speaking { @@ -1721,6 +1779,12 @@ pub async fn start_native_voice_conversation( } berd_voice::input::VoiceInputEvent::RecognitionPendingChanged(pending) => { admission.set_recognition_pending(pending); + event_state.set_status_sound_input_activity( + &session_id, + revision, + None, + Some(pending), + ); // The runtime owns recognition-pending sequencing. Berd's // renderer does not project that state yet. } @@ -1790,6 +1854,9 @@ pub async fn start_native_voice_conversation( if let Some(admission) = current.admission.take() { admission.close(); } + if let Some(status_sounds) = current.status_sounds.take() { + status_sounds.finish(); + } current.session_id = None; current.lifecycle_id = None; current.owner = None; @@ -1863,6 +1930,44 @@ pub async fn set_native_voice_microphone_muted( Ok(status(&app, &state).await) } +#[tauri::command] +pub fn update_native_voice_status_sounds( + state: State<'_, NativeVoiceState>, + capture: State<'_, VoiceCaptureState>, + webview_window: WebviewWindow, + request: StatusSoundUpdateRequest, +) -> Result<(), String> { + capture.with_active_renderer( + webview_window.label(), + &request.renderer_id, + request.renderer_epoch, + || { + let runtime = state + .runtime + .lock() + .map_err(|_| "native voice state lock was poisoned".to_string())?; + if runtime.session_id.as_deref() != Some(request.session_id.as_str()) + || runtime.revision != request.expected_revision + { + return Ok(()); + } + if runtime + .owner + .as_ref() + .map(|owner| owner.window_label.as_str()) + != Some(webview_window.label()) + { + return Err("Only the voice conversation owner can update status sounds.".into()); + } + runtime + .status_sounds + .as_ref() + .ok_or_else(|| "Status sound runtime is unavailable".to_string())? + .update(request.status, request.settings) + }, + ) +} + #[tauri::command] pub fn set_native_voice_assistant_speaking( app: AppHandle, @@ -2191,11 +2296,14 @@ impl NativeVoiceState { &self, expected_lifecycle: Option<(&str, u64)>, ) -> Result, String> { - let Some((session_id, revision, pipeline, owner)) = + let Some((session_id, revision, pipeline, status_sounds, owner)) = self.take_stop_snapshot(expected_lifecycle)? else { return Ok(None); }; + if let Some(status_sounds) = status_sounds { + status_sounds.finish(); + } // Keep the lifecycle current through the bounded shutdown window so a // cooperative worker can flush its final utterance durably. A worker // that misses the deadline is quarantined; its revision-bound late @@ -3297,43 +3405,6 @@ mod tests { assert!(!state.input_controls.is_muted()); } - #[test] - fn assistant_activity_is_bound_to_the_exact_voice_lifecycle() { - let state = NativeVoiceState::default(); - { - let mut runtime = state.runtime.lock().expect("lock native runtime"); - runtime.session_id = Some("session-1".to_string()); - runtime.revision = 7; - runtime.owner = Some(RuntimeOwner { - window_label: "main".to_string(), - }); - } - - assert_eq!( - state - .assistant_activity_target("main", "session-1", 7) - .expect("current activity target"), - Some(("main".to_string(), 7)), - ); - assert_eq!( - state - .assistant_activity_target("main", "session-1", 6) - .expect("stale activity is ignored"), - None, - ); - assert!(state - .assistant_activity_target("session:other", "session-1", 7) - .is_err()); - - state.runtime.lock().expect("lock native runtime").revision = 8; - assert_eq!( - state - .assistant_activity_target("main", "session-1", 7) - .expect("prior lifecycle activity is ignored after restart"), - None, - ); - } - #[test] fn stale_controls_watchdog_cannot_take_a_restarted_voice_lifecycle() { let state = NativeVoiceState::default(); diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index f179adf3d..b3a0550b2 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -33,6 +33,7 @@ pub struct OpenAiRealtimeRuntimeState { struct NativeRealtimeRuntime { owner_window: String, runtime: Arc, + status_sounds: Option, protocol: RealtimeExpertSpokespersonSession, semantic_revision: Arc, } @@ -137,11 +138,16 @@ pub fn start_openai_realtime_spokesperson_runtime( log::info!( "Starting Expert-Spokesperson session {session_id} with execution_path=berd_voice_in_process transport=websocket playback=native_pcm" ); + let status_sounds = + berd_voice::ManagedStatusSoundRuntime::spawn(super::pocket_voice::selected_output_device()) + .map_err(|error| log::warn!("Status sounds unavailable: {error}")) + .ok(); sessions.insert( session_id.clone(), NativeRealtimeRuntime { owner_window: webview_window.label().into(), runtime, + status_sounds, protocol: RealtimeExpertSpokespersonSession::new(initial_cursor, call_id), semantic_revision, }, @@ -159,6 +165,29 @@ fn ensure_native_realtime_playback_supported() -> Result<(), String> { Err("Native OpenAI Realtime playback is not supported on this platform".into()) } +#[derive(serde::Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RealtimeStatusSoundUpdate { + status: berd_voice::ConversationStatus, + settings: berd_voice::StatusSoundSettings, +} + +#[tauri::command] +pub fn update_openai_realtime_status_sounds( + state: State<'_, OpenAiRealtimeRuntimeState>, + webview_window: WebviewWindow, + session_id: String, + update: RealtimeStatusSoundUpdate, +) -> Result<(), String> { + with_runtime_entry(state, session_id, webview_window.label(), |entry| { + entry + .status_sounds + .as_ref() + .ok_or_else(|| "Status sound runtime is unavailable".to_string())? + .update(update.status, update.settings) + }) +} + #[tauri::command] pub fn send_openai_realtime_spokesperson_runtime_event( state: State<'_, OpenAiRealtimeRuntimeState>, @@ -275,6 +304,24 @@ pub fn handle_owner_window_destroyed(app: &AppHandle, window_label: &str) { } } +fn with_runtime_entry( + state: State<'_, OpenAiRealtimeRuntimeState>, + session_id: String, + owner_window: &str, + operation: impl FnOnce(&NativeRealtimeRuntime) -> Result, +) -> Result { + let session_id = non_empty_session_id(session_id)?; + let sessions = state + .sessions + .lock() + .map_err(|_| "OpenAI Realtime runtime state is unavailable".to_string())?; + let entry = sessions + .get(&session_id) + .ok_or("OpenAI Realtime runtime session is not active")?; + ensure_runtime_owner(&entry.owner_window, owner_window)?; + operation(entry) +} + fn with_runtime( state: State<'_, OpenAiRealtimeRuntimeState>, session_id: String, diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 9187d5239..79cdcf904 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -612,6 +612,7 @@ pub fn run() { commands::openai_realtime::stop_openai_realtime_spokesperson_runtime, commands::openai_realtime::release_openai_realtime_spokesperson_runtime, commands::openai_realtime::update_openai_realtime_spokesperson_settings, + commands::openai_realtime::update_openai_realtime_status_sounds, commands::openai_realtime::create_openai_realtime_expert_instructions, commands::openai_realtime::create_openai_realtime_transcript_seed, commands::openai_realtime::deliver_openai_realtime_expert_message, @@ -713,6 +714,7 @@ pub fn run() { commands::native_voice::cancel_native_voice_assistant_speech, commands::native_voice::set_native_voice_microphone_muted, commands::native_voice::set_native_voice_assistant_speaking, + commands::native_voice::update_native_voice_status_sounds, commands::native_voice::drain_native_voice_conversation_transcripts, commands::native_voice::acknowledge_native_voice_conversation_transcript, commands::native_voice::reject_native_voice_conversation_transcript, diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index c06c825c1..9ea7033ab 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -589,6 +589,29 @@ export function rejectVoiceConversationTranscript( ); } +export interface VoiceStatusSoundSettings { + mode: "continuous" | "continuous-while-working" | "once" | "off"; + volume: number; +} + +export async function updateVoiceConversationStatusSounds( + status: VoiceConversationStatus, + conversationStatus: "working" | "waiting", + settings: VoiceStatusSoundSettings, +): Promise { + const { rendererId, rendererEpoch } = await getRendererInstance(); + return invoke("update_native_voice_status_sounds", { + request: { + sessionId: status.sessionId, + expectedRevision: status.revision, + status: conversationStatus, + settings, + rendererId, + rendererEpoch, + }, + }); +} + export async function startVoiceConversation( sessionId: string, inputBackend: "parakeet" | "macos" | "openai" = "parakeet", diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts index 67d7e348d..878fcd901 100644 --- a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts @@ -3,6 +3,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { useChatSessionStore } from "@/features/chat/stores/chatSessionStore"; import type { OpenAiRealtimeProtocolEvent } from "@/shared/api/openaiRealtime"; +import { setStatusSoundPreference } from "../lib/statusSoundPreference"; import { collectRealtimeTranscriptSeedTurns, requestOpenAiRealtimeConversationStart, @@ -200,6 +201,7 @@ const mocks = vi.hoisted(() => ({ setControlsSuppressed: vi.fn(), startControls: vi.fn(), startRuntime: vi.fn(), + updateStatusSounds: vi.fn<() => Promise>(), stopControls: vi.fn(), stopRuntime: vi.fn(), updateRuntimeSettings: vi.fn(), @@ -358,6 +360,7 @@ vi.mock("@/shared/api/openaiRealtime", () => ({ stopOpenAiRealtimeVoiceControls: mocks.stopControls, stopOpenAiRealtimeSpokespersonRuntime: mocks.stopRuntime, updateOpenAiRealtimeSpokespersonSettings: mocks.updateRuntimeSettings, + updateOpenAiRealtimeStatusSounds: mocks.updateStatusSounds, unknownOpenAiRealtimeHandoffIds: mocks.unknownHandoffIds, })); @@ -510,6 +513,7 @@ describe("collectRealtimeTranscriptSeedTurns", () => { }); beforeEach(() => { + window.localStorage.clear(); vi.clearAllMocks(); mocks.activeEmissary = null; mocks.createResponse = true; @@ -538,6 +542,8 @@ beforeEach(() => { value: { getUserMedia: vi.fn().mockResolvedValue(stream) }, }); mocks.appendSessionSystemPrompt.mockResolvedValue(undefined); + mocks.updateStatusSounds.mockReset(); + mocks.updateStatusSounds.mockResolvedValue(undefined); mocks.claimMicrophone.mockResolvedValue(undefined); mocks.createHandoffToolOutput.mockReturnValue({ type: "conversation.item.create", @@ -910,6 +916,17 @@ describe("useOpenAiRealtimeConversation lifecycle", () => { await act(async () => owner.result.current.onToggle()); await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(mocks.updateStatusSounds).toHaveBeenCalledWith( + "session-a", + "waiting", + { mode: "continuous-while-working", volume: 0.4 }, + ); + act(() => setStatusSoundPreference({ mode: "once", volume: 0.7 })); + expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( + "session-a", + "waiting", + { mode: "once", volume: 0.7 }, + ); act(() => { mocks.preferenceListener?.({ voice: "cedar", speed: 1.5 }); }); diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts index 1d7a06a0a..e3a2e4071 100644 --- a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts @@ -33,6 +33,7 @@ import { stopOpenAiRealtimeSpokespersonRuntime, releaseOpenAiRealtimeSpokespersonRuntime, updateOpenAiRealtimeSpokespersonSettings, + updateOpenAiRealtimeStatusSounds, type OpenAiRealtimeTranscriptSeedTurn, type OpenAiRealtimeExpertDeliveryEvent, } from "@/shared/api/openaiRealtime"; @@ -57,6 +58,15 @@ import { type MasterMessageMode, sendRealtimeEvents, } from "../lib/realtimeEmissaryProtocol"; +import { + getStatusSoundPreference, + subscribeToStatusSoundPreference, + type StatusSoundPreference, +} from "../lib/statusSoundPreference"; +import { + getRealtimeVoicePreference, + subscribeToRealtimeVoicePreference, +} from "../lib/realtimeVoicePreference"; import { requestVoiceConversationEnd, trackVoiceAssistantResponse, @@ -64,10 +74,6 @@ import { trackVoiceConversationStarted, trackVoiceUserUtterance, } from "../lib/voiceTelemetry"; -import { - getRealtimeVoicePreference, - subscribeToRealtimeVoicePreference, -} from "../lib/realtimeVoicePreference"; import { beginVoiceControlsVisibilityLease, observeVoiceConversationControlVisibility, @@ -439,12 +445,24 @@ const OFF_SNAPSHOT: Snapshot = { ownerWindowLabel: null, }; +function publishRealtimeStatus( + sessionId: string, + status: "working" | "waiting", + settings: StatusSoundPreference = getStatusSoundPreference(), +): void { + void updateOpenAiRealtimeStatusSounds(sessionId, status, settings).catch( + (error) => + console.warn(`Could not publish Realtime voice ${status} status`, error), + ); +} + class OpenAiRealtimeConversationRuntime { private snapshot: Snapshot = OFF_SNAPSHOT; private readonly listeners = new Set<() => void>(); private nativeMicrophone: NativeMicrophone | null = null; private releaseRuntimeListener: (() => void) | null = null; private releaseVoicePreferenceListener: (() => void) | null = null; + private releaseStatusSoundPreferenceListener: (() => void) | null = null; private realtimeSettingsRevision = 1; private realtimeSettingsQueue = Promise.resolve(); private realtimeRuntimeSessionId: string | null = null; @@ -884,6 +902,17 @@ class OpenAiRealtimeConversationRuntime { this.bridgeCallScope.id, runtimeOptions, ); + publishRealtimeStatus(sessionId, "waiting"); + this.releaseStatusSoundPreferenceListener = + subscribeToStatusSoundPreference((settings) => { + if (isStale() || this.realtimeRuntimeSessionId !== sessionId) + return; + publishRealtimeStatus( + sessionId, + this.snapshot.state === "agent-working" ? "working" : "waiting", + settings, + ); + }); } catch (error) { if (this.realtimeRuntimeSessionId === sessionId) { this.realtimeRuntimeSessionId = null; @@ -1243,6 +1272,7 @@ class OpenAiRealtimeConversationRuntime { }; if (!continueAfterStop) { this.setSnapshot({ ...this.snapshot, state: "agent-working" }); + publishRealtimeStatus(sessionId, "working"); } for (;;) { const opportunity = await waitForMasterDeliveryOpportunity( @@ -1281,8 +1311,10 @@ class OpenAiRealtimeConversationRuntime { } } onDelivered?.(); - if (this.snapshot.boundSessionId === sessionId) + if (this.snapshot.boundSessionId === sessionId) { this.setSnapshot({ ...this.snapshot, state: "listening" }); + publishRealtimeStatus(sessionId, "waiting"); + } }) .catch((error) => { if (isAbortError(error)) return; @@ -1336,6 +1368,7 @@ class OpenAiRealtimeConversationRuntime { this.nativeMicrophone?.stop(); this.releaseRuntimeListener?.(); this.releaseVoicePreferenceListener?.(); + this.releaseStatusSoundPreferenceListener?.(); const realtimeRuntimeSessionId = this.realtimeRuntimeSessionId; this.realtimeRuntimeSessionId = null; this.releaseControlsListener?.(); @@ -1350,6 +1383,7 @@ class OpenAiRealtimeConversationRuntime { this.nativeMicrophone = null; this.releaseRuntimeListener = null; this.releaseVoicePreferenceListener = null; + this.releaseStatusSoundPreferenceListener = null; this.realtimeSettingsQueue = Promise.resolve(); if (controlsRevision > 0) { await stopOpenAiRealtimeVoiceControls( diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 1bbd56015..1992f11cd 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -3,6 +3,7 @@ import { act, renderHook, waitFor } from "@testing-library/react"; import { useChatStore } from "@/features/chat/stores/chatStore"; import { useVoiceConversationStore } from "../stores/voiceConversationStore"; import { VoiceMicrophoneCaptureError } from "../api/voiceConversation"; +import { setStatusSoundPreference } from "../lib/statusSoundPreference"; const nativeAssistantSpeechMocks = vi.hoisted(() => ({ capture: vi.fn(() => []), @@ -13,6 +14,7 @@ const nativeAssistantSpeechMocks = vi.hoisted(() => ({ const tauriWindowMocks = vi.hoisted(() => ({ label: "main" })); const voiceApiMocks = vi.hoisted(() => ({ confirmForegroundSession: vi.fn<() => Promise>(), + updateStatusSounds: vi.fn<() => Promise>().mockResolvedValue(undefined), })); const microphonePermissionMocks = vi.hoisted(() => ({ getStatus: vi.fn<() => Promise<"authorized" | "denied">>(), @@ -38,6 +40,7 @@ vi.mock("../api/voiceConversation", async (importOriginal) => ({ ...(await importOriginal()), confirmVoiceConversationForegroundSession: voiceApiMocks.confirmForegroundSession, + updateVoiceConversationStatusSounds: voiceApiMocks.updateStatusSounds, })); vi.mock("../api/microphonePermission", () => ({ @@ -220,9 +223,15 @@ describe("voice transcript delivery coordination", () => { useChatStore.getState().setActiveRunId("session-1", null); expect(useVoiceConversationStore.getState().uiState).toBe("listening"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1", revision: 3 }), + "waiting", + { mode: "continuous-while-working", volume: 0.4 }, + ); }); beforeEach(() => { + window.localStorage.clear(); tauriWindowMocks.label = "main"; nativeAssistantSpeechMocks.capture.mockClear(); nativeAssistantSpeechMocks.start.mockClear(); @@ -231,11 +240,51 @@ describe("voice transcript delivery coordination", () => { nativeAssistantSpeechMocks.takeNotices.mockReturnValue(null); voiceApiMocks.confirmForegroundSession.mockReset(); voiceApiMocks.confirmForegroundSession.mockResolvedValue(1); + voiceApiMocks.updateStatusSounds.mockReset(); + voiceApiMocks.updateStatusSounds.mockResolvedValue(undefined); microphonePermissionMocks.getStatus.mockReset(); microphonePermissionMocks.getStatus.mockResolvedValue("authorized"); useChatStore.setState({ messagesBySession: {}, sessionStateById: {} }); }); + it("applies settings changes to an active chained runtime", async () => { + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 3, + }, + uiState: "agent-working", + hydrated: true, + init: vi.fn().mockResolvedValue(undefined), + }); + const { unmount } = renderHook(() => + useVoiceConversationController({ + sessionId: "session-1", + onSend: vi.fn(), + enabled: true, + isGooseSession: true, + pocketReady: true, + onPocketSetupRequired: vi.fn(), + }), + ); + + act(() => setStatusSoundPreference({ mode: "once", volume: 0.7 })); + + await waitFor(() => + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1", revision: 3 }), + "working", + { mode: "once", volume: 0.7 }, + ), + ); + unmount(); + }); + it("delivers a queued transcript after its chat becomes temporarily ineligible", async () => { const onSend = vi.fn().mockResolvedValue(true); useVoiceConversationStore.setState({ @@ -286,6 +335,11 @@ describe("voice transcript delivery coordination", () => { undefined, expect.objectContaining({ displayText: "keep this route" }), ); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1", revision: 1 }), + "working", + { mode: "continuous-while-working", volume: 0.4 }, + ); }); it("releases a retained transcript route when the chat becomes read-only", async () => { diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index a4e993c82..be01d1ea9 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -23,9 +23,15 @@ import { confirmVoiceConversationForegroundSession, isVoiceMicrophoneCaptureError, setVoiceConversationControlsSuppressed, + updateVoiceConversationStatusSounds, type PendingVoiceTranscript, } from "../api/voiceConversation"; import { getMicrophonePermissionStatus } from "../api/microphonePermission"; +import { + getStatusSoundPreference, + subscribeToStatusSoundPreference, + type StatusSoundPreference, +} from "../lib/statusSoundPreference"; import type { VoiceInputBackend } from "../lib/voiceInputPreference"; import type { SiriVoiceSelection } from "../api/siriVoice"; @@ -409,6 +415,25 @@ export function waitForVoiceDeliveryOpportunity( }); } +export function publishChainedVoiceStatus( + sessionId: string, + conversationStatus: "working" | "waiting", + settings: StatusSoundPreference = getStatusSoundPreference(), +): void { + const status = useVoiceConversationStore.getState().status; + if (status.lifecycle !== "running" || status.sessionId !== sessionId) return; + void updateVoiceConversationStatusSounds( + status, + conversationStatus, + settings, + ).catch((error) => + console.warn( + `Could not publish chained voice ${conversationStatus} status`, + error, + ), + ); +} + export function resetVoiceUiWhenRunSettles( sessionId: string, deliveryRevision: number, @@ -436,6 +461,7 @@ export function resetVoiceUiWhenRunSettles( unsubscribeVoice(); if (voice.status.revision >= deliveryRevision) { voice.setUiState("listening"); + publishChainedVoiceStatus(sessionId, "waiting"); } }; const unsubscribeChat = useChatStore.subscribe(check); @@ -531,6 +557,7 @@ function ensureVoiceEventDeliveryInitialized() { displayText: event.text, }; store.setUiState("agent-working"); + publishChainedVoiceStatus(event.sessionId, "working"); const delivered = opportunity === "steer" ? await steerPromptInSession( @@ -987,6 +1014,7 @@ export function useVoiceConversationController({ return "not-completed"; } startAssistantSpeech(assistantSpeechHistory); + publishChainedVoiceStatus(sessionId, "waiting"); return "completed"; } catch (startError) { const backendStatus = useVoiceConversationStore.getState().status; @@ -1083,6 +1111,23 @@ export function useVoiceConversationController({ stop, ]); + useEffect(() => { + if ( + status.lifecycle !== "running" || + status.sessionId !== sessionId || + status.ownerWindowLabel !== getCurrentWindow().label + ) + return; + return subscribeToStatusSoundPreference((preference) => { + const uiState = useVoiceConversationStore.getState().uiState; + publishChainedVoiceStatus( + sessionId, + uiState === "agent-working" ? "working" : "waiting", + preference, + ); + }); + }, [sessionId, status.lifecycle, status.ownerWindowLabel, status.sessionId]); + useEffect(() => { if ( status.lifecycle !== "running" || diff --git a/src/features/voice-conversation/lib/statusSoundPreference.test.ts b/src/features/voice-conversation/lib/statusSoundPreference.test.ts new file mode 100644 index 000000000..969b0e7f3 --- /dev/null +++ b/src/features/voice-conversation/lib/statusSoundPreference.test.ts @@ -0,0 +1,56 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + getDefaultStatusSoundPreference, + getStatusSoundPreference, + setStatusSoundPreference, + subscribeToStatusSoundPreference, +} from "./statusSoundPreference"; + +describe("status sound preference", () => { + beforeEach(() => window.localStorage.clear()); + afterEach(() => vi.restoreAllMocks()); + + it("defaults to working pulses and one waiting cue", () => { + expect(getDefaultStatusSoundPreference()).toEqual({ + mode: "continuous-while-working", + volume: 0.4, + }); + expect(getStatusSoundPreference()).toEqual( + getDefaultStatusSoundPreference(), + ); + }); + + it("persists mode and volume", () => { + setStatusSoundPreference({ mode: "once", volume: 0.65 }); + expect(getStatusSoundPreference()).toEqual({ mode: "once", volume: 0.65 }); + }); + + it("normalizes malformed persisted values", () => { + window.localStorage.setItem( + "goose:voice-status-sound-preference", + JSON.stringify({ mode: "unexpected", volume: 4 }), + ); + expect(getStatusSoundPreference()).toEqual({ + mode: "continuous-while-working", + volume: 1, + }); + }); + + it("notifies runtime subscribers with the applied preference", () => { + const listener = vi.fn(); + const unsubscribe = subscribeToStatusSoundPreference(listener); + + setStatusSoundPreference({ mode: "continuous", volume: 0.7 }); + + expect(listener).toHaveBeenCalledWith({ mode: "continuous", volume: 0.7 }); + unsubscribe(); + }); + + it("keeps the renderer preference usable when storage writes fail", () => { + vi.spyOn(window.localStorage, "setItem").mockImplementation(() => { + throw new Error("storage unavailable"); + }); + setStatusSoundPreference({ mode: "off", volume: 0.2 }); + expect(getStatusSoundPreference()).toEqual({ mode: "off", volume: 0.2 }); + }); +}); diff --git a/src/features/voice-conversation/lib/statusSoundPreference.ts b/src/features/voice-conversation/lib/statusSoundPreference.ts new file mode 100644 index 000000000..bd2ba9ca5 --- /dev/null +++ b/src/features/voice-conversation/lib/statusSoundPreference.ts @@ -0,0 +1,125 @@ +import { useCallback, useSyncExternalStore } from "react"; + +export type StatusSoundMode = + | "continuous" + | "continuous-while-working" + | "once" + | "off"; + +export interface StatusSoundPreference { + mode: StatusSoundMode; + volume: number; +} + +const STORAGE_KEY = "goose:voice-status-sound-preference"; +const CHANGED_EVENT = "goose:voice-status-sound-preference-changed"; +const DEFAULT_PREFERENCE: StatusSoundPreference = { + mode: "continuous-while-working", + volume: 0.4, +}; +const DEFAULT_SNAPSHOT = JSON.stringify(DEFAULT_PREFERENCE); +let volatilePreference: StatusSoundPreference | undefined; + +function normalize(value: unknown): StatusSoundPreference { + if (!value || typeof value !== "object") return DEFAULT_PREFERENCE; + const candidate = value as Partial; + const mode = + candidate.mode === "continuous" || + candidate.mode === "continuous-while-working" || + candidate.mode === "once" || + candidate.mode === "off" + ? candidate.mode + : DEFAULT_PREFERENCE.mode; + const volume = + typeof candidate.volume === "number" && Number.isFinite(candidate.volume) + ? Math.min(1, Math.max(0, candidate.volume)) + : DEFAULT_PREFERENCE.volume; + return { mode, volume }; +} + +export function getDefaultStatusSoundPreference(): StatusSoundPreference { + return DEFAULT_PREFERENCE; +} + +export function getStatusSoundPreference(): StatusSoundPreference { + if (typeof window === "undefined") return DEFAULT_PREFERENCE; + try { + const raw = window.localStorage.getItem(STORAGE_KEY); + return raw + ? normalize(JSON.parse(raw)) + : (volatilePreference ?? DEFAULT_PREFERENCE); + } catch { + return volatilePreference ?? DEFAULT_PREFERENCE; + } +} + +function getSnapshot(): string { + return JSON.stringify(getStatusSoundPreference()); +} + +const listeners = new Set<() => void>(); +let removeWindowListeners: (() => void) | undefined; + +function notify() { + for (const listener of listeners) listener(); +} + +function subscribe(listener: () => void) { + if (typeof window === "undefined") return () => {}; + listeners.add(listener); + if (!removeWindowListeners) { + const handleStorage = (event: StorageEvent) => { + if (event.key === STORAGE_KEY || event.key === null) { + volatilePreference = undefined; + notify(); + } + }; + window.addEventListener(CHANGED_EVENT, notify); + window.addEventListener("storage", handleStorage); + removeWindowListeners = () => { + window.removeEventListener(CHANGED_EVENT, notify); + window.removeEventListener("storage", handleStorage); + }; + } + return () => { + listeners.delete(listener); + if (listeners.size === 0) { + removeWindowListeners?.(); + removeWindowListeners = undefined; + } + }; +} + +export function setStatusSoundPreference( + preference: StatusSoundPreference, +): void { + if (typeof window === "undefined") return; + const value = normalize(preference); + volatilePreference = value; + try { + window.localStorage.setItem(STORAGE_KEY, JSON.stringify(value)); + volatilePreference = undefined; + } catch { + // Keep the current renderer usable when persistent storage is unavailable. + } + window.dispatchEvent(new CustomEvent(CHANGED_EVENT, { detail: value })); +} + +export function subscribeToStatusSoundPreference( + listener: (preference: StatusSoundPreference) => void, +): () => void { + return subscribe(() => listener(getStatusSoundPreference())); +} + +export function useStatusSoundPreference() { + const snapshot = useSyncExternalStore( + subscribe, + getSnapshot, + () => DEFAULT_SNAPSHOT, + ); + const preference = normalize(JSON.parse(snapshot)); + const update = useCallback((patch: Partial) => { + setStatusSoundPreference({ ...getStatusSoundPreference(), ...patch }); + }, []); + return { ...preference, update }; +} diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 073aadced..84d27a1f4 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -64,8 +64,17 @@ const preferenceMocks = vi.hoisted(() => ({ setOutputBackend: vi.fn(), setInterruptionMode: vi.fn(), setMode: vi.fn(), + setStatusSounds: vi.fn(), setRealtimePreference: vi.fn(), })); +const statusSoundState = vi.hoisted(() => ({ + mode: "continuous-while-working" as + | "continuous" + | "continuous-while-working" + | "once" + | "off", + volume: 0.4, +})); const interruptionState = vi.hoisted(() => ({ mode: "automatic" as "automatic" | "allowInterruptions" | "preventFeedback", })); @@ -176,6 +185,16 @@ vi.mock("../lib/voiceConversationModePreference", () => ({ setMode: preferenceMocks.setMode, }), })); +vi.mock("../lib/statusSoundPreference", () => ({ + getDefaultStatusSoundPreference: () => ({ + mode: "continuous-while-working", + volume: 0.4, + }), + useStatusSoundPreference: () => ({ + ...statusSoundState, + update: preferenceMocks.setStatusSounds, + }), +})); vi.mock("../lib/realtimeVoicePreference", async (importOriginal) => ({ ...(await importOriginal()), setRealtimeVoicePreference: preferenceMocks.setRealtimePreference, diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index 2ea924829..162a526ac 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -9,6 +9,7 @@ import { Badge } from "@/shared/ui/badge"; import { ConfirmDialog } from "@/shared/ui/confirm-dialog"; import { RadioGroup, RadioGroupCard } from "@/shared/ui/radio-group"; import { SettingsRow } from "@/shared/ui/settings-row"; +import { Slider } from "@/shared/ui/slider"; import { Select, SelectContent, @@ -30,6 +31,11 @@ import { usePocketVoiceSetup } from "../hooks/usePocketVoiceSetup"; import { useMacSpeechSetup } from "../hooks/useMacSpeechSetup"; import { useMicrophonePermission } from "../hooks/useMicrophonePermission"; import { useSiriVoiceSetup } from "../hooks/useSiriVoiceSetup"; +import type { StatusSoundMode } from "../lib/statusSoundPreference"; +import { + getDefaultStatusSoundPreference, + useStatusSoundPreference, +} from "../lib/statusSoundPreference"; import type { VoiceInputBackend } from "../lib/voiceInputPreference"; import { getDefaultVoiceInputBackend, @@ -68,6 +74,13 @@ import { openAiVoiceOptions, } from "../lib/openAiVoiceOptions"; +const STATUS_SOUND_MODES: StatusSoundMode[] = [ + "continuous", + "continuous-while-working", + "once", + "off", +]; + const INTERRUPTION_MODES: VoiceInterruptionMode[] = [ "automatic", "allowInterruptions", @@ -162,6 +175,7 @@ export function VoiceSettings() { }, [openAiStatus]); const interruption = useVoiceInterruptionPreference(); const mode = useVoiceConversationModePreference(); + const statusSounds = useStatusSoundPreference(); const siriSetup = useSiriVoiceSetup(output.backend === "siri"); const siriSupported = getPlatform() === "mac"; const microphonePermission = useMicrophonePermission(siriSupported); @@ -233,6 +247,7 @@ export function VoiceSettings() { output.setBackend(getDefaultVoiceOutputBackend()); interruption.setMode(getDefaultVoiceInterruptionPreference().mode); mode.setMode(getDefaultVoiceConversationMode()); + statusSounds.update(getDefaultStatusSoundPreference()); setResetDialogOpen(false); } finally { setResetting(false); @@ -595,6 +610,61 @@ export function VoiceSettings() { ) : ( )} +
+ {t("voice.statusSounds")} + } + description={t( + `voice.statusSoundModeDescriptions.${statusSounds.mode}`, + )} + layout="responsive" + action={({ labelId, descriptionId }) => ( + + )} + details={ + statusSounds.mode === "off" ? null : ( +
+
+ {t("voice.statusSoundVolume")} + + {Math.round(statusSounds.volume * 100)}% + +
+ statusSounds.update({ volume })} + aria-label={t("voice.statusSoundVolume")} + /> +
+ ) + } + /> +
{resetError ? (

{resetError} diff --git a/src/shared/api/openaiRealtime.ts b/src/shared/api/openaiRealtime.ts index 6a31162fa..bb3552c31 100644 --- a/src/shared/api/openaiRealtime.ts +++ b/src/shared/api/openaiRealtime.ts @@ -51,6 +51,20 @@ export function startOpenAiRealtimeSpokespersonRuntime( }); } +export function updateOpenAiRealtimeStatusSounds( + sessionId: string, + status: "working" | "waiting", + settings: { + mode: "continuous" | "continuous-while-working" | "once" | "off"; + volume: number; + }, +): Promise { + return invoke("update_openai_realtime_status_sounds", { + sessionId, + update: { status, settings }, + }); +} + export function sendOpenAiRealtimeSpokespersonRuntimeEvent( sessionId: string, event: Record, diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index cfd8a9ffd..0ec2efc8a 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -1098,6 +1098,20 @@ "selectedVoice": "Selected voice: {{voice}}", "useVoice": "Use {{voice}}", "voice": "Voice", - "voiceLabel": "Pocket TTS voice" + "voiceLabel": "Pocket TTS voice", + "statusSounds": "Status sounds", + "statusSoundVolume": "Status sound volume", + "statusSoundModes": { + "continuous": "Always repeat", + "continuous-while-working": "Repeat while working", + "once": "Play once", + "off": "Off" + }, + "statusSoundModeDescriptions": { + "continuous": "Plays the current working or waiting cue every five seconds.", + "continuous-while-working": "Repeats the working cue every five seconds, then plays the waiting cue once.", + "once": "Plays one cue whenever the agent changes between working and waiting.", + "off": "Disables agent status sounds." + } } } diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index 8778dba5b..ce580648c 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -1097,6 +1097,20 @@ "selectedVoice": "Voz seleccionada: {{voice}}", "useVoice": "Usar {{voice}}", "voice": "Voz", - "voiceLabel": "Voz de Pocket TTS" + "voiceLabel": "Voz de Pocket TTS", + "statusSounds": "Sonidos de estado", + "statusSoundVolume": "Volumen de los sonidos de estado", + "statusSoundModes": { + "continuous": "Repetir siempre", + "continuous-while-working": "Repetir mientras trabaja", + "once": "Reproducir una vez", + "off": "Desactivados" + }, + "statusSoundModeDescriptions": { + "continuous": "Reproduce la señal actual de trabajo o espera cada cinco segundos.", + "continuous-while-working": "Repite la señal de trabajo cada cinco segundos y luego reproduce una vez la señal de espera.", + "once": "Reproduce una señal cuando el agente cambia entre trabajo y espera.", + "off": "Desactiva los sonidos de estado del agente." + } } } From d1278da0a0331c0868c8ba0fafdb3de16a24c968 Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Wed, 9 Sep 2026 13:40:37 -0400 Subject: [PATCH 10/29] Fix status sound runtime lifecycle Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- .../crates/berd-voice/src/status_sounds.rs | 7 +- src-tauri/src/commands/openai_realtime.rs | 107 +++++++++++++++++- .../useOpenAiRealtimeConversation.test.ts | 41 +++++++ .../hooks/useOpenAiRealtimeConversation.ts | 32 +++++- .../hooks/useVoiceConversationController.ts | 27 ++--- .../lib/statusSoundPreference.ts | 7 +- 6 files changed, 186 insertions(+), 35 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 72d2c2448..8635d8266 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -171,6 +171,7 @@ enum StatusSoundCommand { } /// Thread-safe status-sound service for hosts that do not own a polling loop. +#[derive(Clone)] pub struct ManagedStatusSoundRuntime { commands: Sender, } @@ -239,12 +240,6 @@ impl ManagedStatusSoundRuntime { } } -impl Drop for ManagedStatusSoundRuntime { - fn drop(&mut self) { - let _ = self.commands.send(StatusSoundCommand::Shutdown); - } -} - #[cfg(not(target_os = "macos"))] struct StatusSoundPlayer; diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index b3a0550b2..c4ce8a02c 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -129,19 +129,28 @@ pub fn start_openai_realtime_spokesperson_runtime( let semantic_revision = Arc::new(AtomicU64::new(0)); let event_window = webview_window.clone(); let event_session_id = session_id.clone(); + let status_sounds = + berd_voice::ManagedStatusSoundRuntime::spawn(super::pocket_voice::selected_output_device()) + .map_err(|error| log::warn!("Status sounds unavailable: {error}")) + .ok(); + let runtime_status_sounds = status_sounds.clone(); + let mut status_sound_activity = RealtimeStatusSoundActivity::default(); let runtime = Arc::new(ManagedRealtimeHost::spawn( config, Arc::clone(&semantic_revision), create_native_realtime_output, - move |event| emit_runtime_provider_event(&event_window, &event_session_id, event), + move |event| { + update_realtime_status_sound_activity( + runtime_status_sounds.as_ref(), + &event, + &mut status_sound_activity, + ); + emit_runtime_provider_event(&event_window, &event_session_id, event) + }, )?); log::info!( "Starting Expert-Spokesperson session {session_id} with execution_path=berd_voice_in_process transport=websocket playback=native_pcm" ); - let status_sounds = - berd_voice::ManagedStatusSoundRuntime::spawn(super::pocket_voice::selected_output_device()) - .map_err(|error| log::warn!("Status sounds unavailable: {error}")) - .ok(); sessions.insert( session_id.clone(), NativeRealtimeRuntime { @@ -237,6 +246,9 @@ pub async fn stop_openai_realtime_spokesperson_runtime( .get(&session_id) .map(|entry| { ensure_runtime_owner(&entry.owner_window, webview_window.label())?; + if let Some(status_sounds) = entry.status_sounds.as_ref() { + status_sounds.finish(); + } Ok::<_, String>(Arc::clone(&entry.runtime)) }) .transpose()?; @@ -266,6 +278,9 @@ pub async fn release_openai_realtime_spokesperson_runtime( sessions.remove(&session_id) }; if let Some(entry) = entry { + if let Some(status_sounds) = entry.status_sounds.as_ref() { + status_sounds.finish(); + } tauri::async_runtime::spawn_blocking(move || entry.runtime.finish()) .await .map_err(|error| format!("OpenAI Realtime runtime release task failed: {error}"))??; @@ -285,7 +300,12 @@ pub fn handle_owner_window_destroyed(app: &AppHandle, window_label: &str) { owned_session_ids .into_iter() .filter_map(|session_id| sessions.remove(&session_id)) - .map(|entry| entry.runtime) + .map(|entry| { + if let Some(status_sounds) = entry.status_sounds.as_ref() { + status_sounds.finish(); + } + entry.runtime + }) .collect::>() } Err(_) => { @@ -391,6 +411,43 @@ pub async fn update_openai_realtime_spokesperson_settings( .map_err(|error| format!("Spokesperson settings task failed: {error}"))? } +#[derive(Default)] +struct RealtimeStatusSoundActivity { + user_speaking: bool, + playback_active: bool, +} + +impl RealtimeStatusSoundActivity { + fn update(&mut self, event: &serde_json::Value) -> Option { + match event.get("type").and_then(serde_json::Value::as_str) { + Some("input_audio_buffer.speech_started") => self.user_speaking = true, + Some("input_audio_buffer.speech_stopped") => self.user_speaking = false, + Some("output_audio_buffer.started") => self.playback_active = true, + Some("output_audio_buffer.stopped" | "output_audio_buffer.cleared") => { + self.playback_active = false; + } + _ => return None, + } + Some(self.user_speaking || self.playback_active) + } +} + +fn update_realtime_status_sound_activity( + status_sounds: Option<&berd_voice::ManagedStatusSoundRuntime>, + event: &serde_json::Value, + activity: &mut RealtimeStatusSoundActivity, +) { + let Some(conversation_active) = activity.update(event) else { + return; + }; + let Some(status_sounds) = status_sounds else { + return; + }; + if let Err(error) = status_sounds.set_conversation_active(conversation_active) { + log::warn!("Could not update Realtime status sound activity: {error}"); + } +} + fn emit_runtime_provider_event( window: &WebviewWindow, session_id: &str, @@ -761,6 +818,7 @@ fn client_secret_value(value: &serde_json::Value) -> Option<&str> { mod tests { use super::{ ensure_runtime_owner, parse_client_secret, realtime_transcription_client_secret_request, + RealtimeStatusSoundActivity, }; use serde_json::json; @@ -775,6 +833,43 @@ mod tests { } } + #[test] + fn realtime_status_sound_activity_aggregates_input_and_output() { + let mut activity = RealtimeStatusSoundActivity::default(); + + assert_eq!( + activity.update(&json!({ "type": "input_audio_buffer.speech_started" })), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "output_audio_buffer.started" })), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "input_audio_buffer.speech_stopped" })), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "output_audio_buffer.stopped" })), + Some(false) + ); + assert_eq!(activity.update(&json!({ "type": "response.done" })), None); + } + + #[test] + fn realtime_status_sound_activity_treats_cleared_output_as_inactive() { + let mut activity = RealtimeStatusSoundActivity::default(); + + assert_eq!( + activity.update(&json!({ "type": "output_audio_buffer.started" })), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "output_audio_buffer.cleared" })), + Some(false) + ); + } + #[test] fn parses_supported_client_secret_shapes() { assert_eq!( diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts index 878fcd901..a0da0c64b 100644 --- a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts @@ -941,6 +941,47 @@ describe("useOpenAiRealtimeConversation lifecycle", () => { ); }); + it("keeps working status until the admitted master run settles", async () => { + const onSend = vi.fn().mockImplementation(async () => { + useChatStore.getState().setChatState("session-a", "thinking"); + useChatStore.getState().setActiveRunId("session-a", "run-1"); + return true; + }); + const owner = renderConversation("session-a", onSend); + + await act(async () => owner.result.current.onToggle()); + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + mocks.updateStatusSounds.mockClear(); + + act(() => { + channel.dispatchEvent( + new MessageEvent("message", { + data: JSON.stringify({ type: "test.emissary" }), + }), + ); + }); + + await waitFor(() => expect(onSend).toHaveBeenCalledOnce()); + expect(owner.result.current.state).toBe("agent-working"); + expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( + "session-a", + "working", + { mode: "continuous-while-working", volume: 0.4 }, + ); + + act(() => { + useChatStore.getState().setActiveRunId("session-a", null); + useChatStore.getState().setChatState("session-a", "idle"); + }); + + await waitFor(() => expect(owner.result.current.state).toBe("listening")); + expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( + "session-a", + "waiting", + { mode: "continuous-while-working", volume: 0.4 }, + ); + }); + it("serializes rapid voice settings changes against each applied revision", async () => { let resolveFirst!: (snapshot: { revision: number; diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts index e3a2e4071..d85d70092 100644 --- a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts @@ -456,6 +456,29 @@ function publishRealtimeStatus( ); } +function resetRealtimeStatusWhenRunSettles( + runtime: OpenAiRealtimeConversationRuntime, + sessionId: string, +): void { + let sawRun = false; + const check = () => { + if (runtime.getSnapshot().boundSessionId !== sessionId) { + unsubscribe(); + return; + } + const master = useChatStore.getState().getSessionRuntime(sessionId); + if (master.activeRunId !== null || isSessionRunning(master.chatState)) { + sawRun = true; + return; + } + if (!sawRun) return; + unsubscribe(); + runtime.markWaiting(sessionId); + }; + const unsubscribe = useChatStore.subscribe(check); + queueMicrotask(check); +} + class OpenAiRealtimeConversationRuntime { private snapshot: Snapshot = OFF_SNAPSHOT; private readonly listeners = new Set<() => void>(); @@ -1205,6 +1228,12 @@ class OpenAiRealtimeConversationRuntime { this.setSnapshot(OFF_SNAPSHOT); } + markWaiting(sessionId: string): void { + if (this.snapshot.boundSessionId !== sessionId) return; + this.setSnapshot({ ...this.snapshot, state: "listening" }); + publishRealtimeStatus(sessionId, "waiting"); + } + private deliverToMaster( sessionId: string, text: string, @@ -1312,8 +1341,7 @@ class OpenAiRealtimeConversationRuntime { } onDelivered?.(); if (this.snapshot.boundSessionId === sessionId) { - this.setSnapshot({ ...this.snapshot, state: "listening" }); - publishRealtimeStatus(sessionId, "waiting"); + resetRealtimeStatusWhenRunSettles(this, sessionId); } }) .catch((error) => { diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index be01d1ea9..23aa77d0a 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -472,6 +472,16 @@ export function resetVoiceUiWhenRunSettles( function ensureVoiceEventDeliveryInitialized() { if (deliveryInitialized) return; deliveryInitialized = true; + subscribeToStatusSoundPreference((preference) => { + const voice = useVoiceConversationStore.getState(); + const activeSessionId = voice.status.sessionId; + if (voice.status.lifecycle !== "running" || !activeSessionId) return; + publishChainedVoiceStatus( + activeSessionId, + voice.uiState === "agent-working" ? "working" : "waiting", + preference, + ); + }); subscribeToVoiceConversationEvents(async (event) => { if (event.type === "cleanShutdown" || event.type === "controlsDismissed") { return; @@ -1111,23 +1121,6 @@ export function useVoiceConversationController({ stop, ]); - useEffect(() => { - if ( - status.lifecycle !== "running" || - status.sessionId !== sessionId || - status.ownerWindowLabel !== getCurrentWindow().label - ) - return; - return subscribeToStatusSoundPreference((preference) => { - const uiState = useVoiceConversationStore.getState().uiState; - publishChainedVoiceStatus( - sessionId, - uiState === "agent-working" ? "working" : "waiting", - preference, - ); - }); - }, [sessionId, status.lifecycle, status.ownerWindowLabel, status.sessionId]); - useEffect(() => { if ( status.lifecycle !== "running" || diff --git a/src/features/voice-conversation/lib/statusSoundPreference.ts b/src/features/voice-conversation/lib/statusSoundPreference.ts index bd2ba9ca5..44a48e20c 100644 --- a/src/features/voice-conversation/lib/statusSoundPreference.ts +++ b/src/features/voice-conversation/lib/statusSoundPreference.ts @@ -43,13 +43,12 @@ export function getDefaultStatusSoundPreference(): StatusSoundPreference { export function getStatusSoundPreference(): StatusSoundPreference { if (typeof window === "undefined") return DEFAULT_PREFERENCE; + if (volatilePreference) return volatilePreference; try { const raw = window.localStorage.getItem(STORAGE_KEY); - return raw - ? normalize(JSON.parse(raw)) - : (volatilePreference ?? DEFAULT_PREFERENCE); + return raw ? normalize(JSON.parse(raw)) : DEFAULT_PREFERENCE; } catch { - return volatilePreference ?? DEFAULT_PREFERENCE; + return DEFAULT_PREFERENCE; } } From 53b505a9f73ee8a3d680d7b2f8e499273fca0dbc Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Wed, 9 Sep 2026 13:47:52 -0400 Subject: [PATCH 11/29] Make status sound exports rustfmt-stable Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-voice/src/lib.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/lib.rs b/src-tauri/crates/berd-voice/src/lib.rs index 86298f5cc..ba7ddd06d 100644 --- a/src-tauri/crates/berd-voice/src/lib.rs +++ b/src-tauri/crates/berd-voice/src/lib.rs @@ -55,10 +55,11 @@ pub use pocket::{ }; #[cfg(target_os = "macos")] pub use siri::SiriTts; -pub use status_sounds::{ - ConversationStatus, ManagedStatusSoundRuntime, StatusSoundMode, - StatusSoundRuntime, StatusSoundSettings, -}; +pub use status_sounds::ConversationStatus; +pub use status_sounds::ManagedStatusSoundRuntime; +pub use status_sounds::StatusSoundMode; +pub use status_sounds::StatusSoundRuntime; +pub use status_sounds::StatusSoundSettings; pub use synthesis::{synthesize_pcm16_wav, WavSynthesis, WavSynthesisError, WavSynthesisErrorKind}; pub use tts::{ OpenAiTts, PocketTtsBackend, StreamingTextChunk, StreamingTextChunks, StreamingTtsText, From 1930da02b7db16746e0db279b5d732b15937b0ce Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Thu, 10 Sep 2026 12:46:08 -0400 Subject: [PATCH 12/29] Simplify continuous voice status sounds Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-voice/PROTOCOL.md | 12 +- src-tauri/crates/berd-voice/README.md | 2 +- src-tauri/crates/berd-voice/src/main.rs | 6 - .../crates/berd-voice/src/status_sounds.rs | 147 ++++-------------- .../berd-voice/tests/session_protocol.rs | 4 +- src-tauri/src/commands/openai_realtime.rs | 69 +++++++- .../api/voiceConversation.ts | 3 +- .../useOpenAiRealtimeConversation.test.ts | 23 ++- .../hooks/useOpenAiRealtimeConversation.ts | 34 +++- .../useVoiceConversationController.test.ts | 30 +++- .../hooks/useVoiceConversationController.ts | 36 +++-- .../lib/statusSoundPreference.test.ts | 32 ++-- .../lib/statusSoundPreference.ts | 38 ++--- .../ui/VoiceSettings.test.tsx | 27 ++-- .../voice-conversation/ui/VoiceSettings.tsx | 27 +--- src/shared/api/openaiRealtime.ts | 3 +- src/shared/i18n/locales/en/settings.json | 13 +- src/shared/i18n/locales/es/settings.json | 13 +- 18 files changed, 259 insertions(+), 260 deletions(-) diff --git a/src-tauri/crates/berd-voice/PROTOCOL.md b/src-tauri/crates/berd-voice/PROTOCOL.md index a8aa7d6bf..80190f763 100644 --- a/src-tauri/crates/berd-voice/PROTOCOL.md +++ b/src-tauri/crates/berd-voice/PROTOCOL.md @@ -205,13 +205,11 @@ Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. The parent cannot author speaking state or finalized input; those are derived only from PCM by the child runtime. -`StatusSoundSettings` has a `mode` of `continuous`, -`continuous-while-working`, `once`, or `off`, and a finite `volume` from `0` -through `1`. No cue is emitted until the first `set_conversation_status` request. -The runtime then ticks immediately and every five seconds. `continuous` emits the -current cue every tick; `continuous-while-working` repeats working and emits -waiting once; `once` emits only when the requested status differs from the last -emitted cue; `off` emits nothing. Active user input, pending recognition, or assistant output suppresses a tick without consuming its pending cue. On macOS, working uses the system Pop sound and waiting uses Purr through the native PCM player. The applied +`StatusSoundSettings` has a `mode` of `working` or `working-and-waiting`. No cue +is emitted until the first `set_conversation_status` request. The runtime then +ticks immediately and every five seconds. `working` repeats only the working cue +and stays silent while waiting; `working-and-waiting` repeats the current working +or waiting cue. Active user input, pending recognition, or assistant output suppresses a tick without consuming its pending cue. On macOS, working uses the system Pop sound and waiting uses Purr through the native PCM player. The applied request is acknowledged with: ```text diff --git a/src-tauri/crates/berd-voice/README.md b/src-tauri/crates/berd-voice/README.md index 5e32f9800..12da7725a 100644 --- a/src-tauri/crates/berd-voice/README.md +++ b/src-tauri/crates/berd-voice/README.md @@ -21,7 +21,7 @@ device; the existing Berd Siri player and the CLI use the same decoder. in [PROTOCOL.md](PROTOCOL.md). The host supplies persisted status-sound settings and semantic `working` / `waiting` updates through that protocol; the runtime owns the five-second cadence, speech suppression, and macOS Pop/Purr playback. -The default mode is `continuous-while-working` at volume `0.4`. +The default mode is `working`; status sounds use a fixed gain of `0.4`. Siri TTS and macOS speech recognition are the defaults: diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index 278f581d2..d9749e507 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -6601,12 +6601,6 @@ fn validate_request(request: SessionRequest) -> Result { return Err("request id must be positive".into()); } match &request { - SessionRequest::SetConversationStatus { - settings: status_sounds, - .. - } => { - status_sounds.validate()?; - } SessionRequest::PrepareSpeak { text, .. } if text.len() > MAX_SPEAK_TEXT_BYTES => { return Err("speak text exceeds 16 KiB".into()) } diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 8635d8266..489a0826b 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -6,17 +6,15 @@ use std::{ use serde::{Deserialize, Serialize}; -pub const DEFAULT_STATUS_SOUND_VOLUME: f32 = 0.4; +const STATUS_SOUND_GAIN: f32 = 0.4; pub const STATUS_SOUND_INTERVAL: Duration = Duration::from_secs(5); #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub enum StatusSoundMode { - Continuous, + WorkingAndWaiting, #[default] - ContinuousWhileWorking, - Once, - Off, + Working, } #[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)] @@ -30,31 +28,19 @@ pub enum ConversationStatus { #[serde(deny_unknown_fields)] pub struct StatusSoundSettings { pub mode: StatusSoundMode, - pub volume: f32, } impl Default for StatusSoundSettings { fn default() -> Self { Self { mode: StatusSoundMode::default(), - volume: DEFAULT_STATUS_SOUND_VOLUME, - } - } -} - -impl StatusSoundSettings { - pub fn validate(self) -> Result { - if !self.volume.is_finite() || !(0.0..=1.0).contains(&self.volume) { - return Err("status sound volume must be finite and between 0 and 1"); } - Ok(self) } } #[derive(Clone, Copy, Debug, PartialEq)] pub struct StatusSoundCue { pub status: ConversationStatus, - pub volume: f32, } /// Pure policy for deciding which cue, if any, a fixed-cadence runtime tick plays. @@ -62,7 +48,6 @@ pub struct StatusSoundCue { #[derive(Debug, Default)] pub struct StatusSoundStateMachine { current: Option<(ConversationStatus, StatusSoundSettings)>, - last_played: Option, } impl StatusSoundStateMachine { @@ -75,25 +60,13 @@ impl StatusSoundStateMachine { pub fn tick(&mut self, conversation_active: bool) -> Option { let (status, settings) = self.current?; - if conversation_active || settings.mode == StatusSoundMode::Off { + if conversation_active + || (settings.mode == StatusSoundMode::Working + && status == ConversationStatus::Waiting) + { return None; } - let should_play = match settings.mode { - StatusSoundMode::Continuous => true, - StatusSoundMode::ContinuousWhileWorking => { - status == ConversationStatus::Working || self.last_played != Some(status) - } - StatusSoundMode::Once => self.last_played != Some(status), - StatusSoundMode::Off => false, - }; - if !should_play { - return None; - } - self.last_played = Some(status); - Some(StatusSoundCue { - status, - volume: settings.volume, - }) + Some(StatusSoundCue { status }) } } @@ -104,7 +77,6 @@ pub struct StatusSoundRuntime { next_tick: Option, player: StatusSoundPlayer, output_device: Option, - playback_available: bool, } impl Default for StatusSoundRuntime { @@ -114,7 +86,6 @@ impl Default for StatusSoundRuntime { next_tick: None, player: StatusSoundPlayer::new(), output_device: None, - playback_available: true, } } } @@ -142,8 +113,7 @@ impl StatusSoundRuntime { pub fn poll(&mut self, conversation_active: bool) -> Result { if conversation_active { self.stop(); - } - if !self.playback_available { + self.player.reap(); return Ok(false); } self.player.reap(); @@ -151,12 +121,7 @@ impl StatusSoundRuntime { if self.next_tick.is_some_and(|deadline| now >= deadline) { self.next_tick = Some(now + STATUS_SOUND_INTERVAL); if let Some(cue) = self.machine.tick(conversation_active) { - if cue.volume > 0.0 { - if let Err(message) = self.player.play(cue, self.output_device.as_deref()) { - self.playback_available = false; - return Err(message); - } - } + self.player.play(cue, self.output_device.as_deref())?; } } Ok(self.player.is_active()) @@ -217,7 +182,6 @@ impl ManagedStatusSoundRuntime { status: ConversationStatus, settings: StatusSoundSettings, ) -> Result<(), String> { - settings.validate().map_err(str::to_string)?; self.send(StatusSoundCommand::Update(status, settings)) } @@ -303,7 +267,7 @@ impl StatusSoundPlayer { let samples = asset .samples .iter() - .map(|sample| sample * cue.volume) + .map(|sample| sample * STATUS_SOUND_GAIN) .collect::>(); player.enqueue(&samples)?; self.active.push(ActiveStatusSound { @@ -360,72 +324,48 @@ mod tests { use super::*; fn settings(mode: StatusSoundMode) -> StatusSoundSettings { - StatusSoundSettings { mode, volume: 0.4 } + StatusSoundSettings { mode } } #[test] - fn defaults_to_continuous_while_working() { + fn defaults_to_working_only() { assert_eq!( StatusSoundSettings::default(), - settings(StatusSoundMode::ContinuousWhileWorking) + settings(StatusSoundMode::Working) ); } #[test] - fn continuous_plays_every_tick() { + fn working_and_waiting_repeats_both_statuses() { let mut machine = StatusSoundStateMachine::default(); - machine.update( - ConversationStatus::Waiting, - settings(StatusSoundMode::Continuous), - ); - assert!(machine.tick(false).is_some()); - assert!(machine.tick(false).is_some()); - } - - #[test] - fn continuous_while_working_repeats_working_and_plays_waiting_once() { - let mut machine = StatusSoundStateMachine::default(); - let settings = settings(StatusSoundMode::ContinuousWhileWorking); + let settings = settings(StatusSoundMode::WorkingAndWaiting); machine.update(ConversationStatus::Working, settings); - assert_eq!( - machine.tick(false).unwrap().status, - ConversationStatus::Working - ); - assert_eq!( - machine.tick(false).unwrap().status, - ConversationStatus::Working - ); + assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Working); + assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Working); machine.update(ConversationStatus::Waiting, settings); - assert_eq!( - machine.tick(false).unwrap().status, - ConversationStatus::Waiting - ); - assert_eq!(machine.tick(false), None); + assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Waiting); + assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Waiting); } #[test] - fn once_plays_only_when_status_differs_from_last_cue() { + fn working_only_repeats_working_and_stays_silent_while_waiting() { let mut machine = StatusSoundStateMachine::default(); - let settings = settings(StatusSoundMode::Once); + let settings = settings(StatusSoundMode::Working); machine.update(ConversationStatus::Working, settings); - assert!(machine.tick(false).is_some()); - assert_eq!(machine.tick(false), None); + assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Working); + assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Working); machine.update(ConversationStatus::Waiting, settings); - assert!(machine.tick(false).is_some()); assert_eq!(machine.tick(false), None); - } - - #[test] - fn off_never_plays() { - let mut machine = StatusSoundStateMachine::default(); - machine.update(ConversationStatus::Working, settings(StatusSoundMode::Off)); assert_eq!(machine.tick(false), None); } #[test] - fn audible_conversation_audio_suppresses_without_consuming_cue() { + fn conversation_audio_suppresses_without_changing_the_status() { let mut machine = StatusSoundStateMachine::default(); - machine.update(ConversationStatus::Waiting, settings(StatusSoundMode::Once)); + machine.update( + ConversationStatus::Working, + settings(StatusSoundMode::Working), + ); assert_eq!(machine.tick(true), None); assert!(machine.tick(false).is_some()); } @@ -435,28 +375,10 @@ mod tests { assert_eq!(StatusSoundStateMachine::default().tick(false), None); } - #[test] - fn validates_volume() { - for volume in [f32::NAN, f32::INFINITY, -0.1, 1.1] { - assert!(StatusSoundSettings { - mode: StatusSoundMode::Once, - volume - } - .validate() - .is_err()); - } - assert!(StatusSoundSettings { - mode: StatusSoundMode::Once, - volume: 1.0 - } - .validate() - .is_ok()); - } - #[test] fn duplicate_updates_preserve_the_existing_cadence() { let mut runtime = StatusSoundRuntime::default(); - let settings = settings(StatusSoundMode::Continuous); + let settings = settings(StatusSoundMode::WorkingAndWaiting); runtime.update(ConversationStatus::Working, settings); let deadline = runtime.next_tick; runtime.update(ConversationStatus::Working, settings); @@ -468,12 +390,12 @@ mod tests { let mut runtime = StatusSoundRuntime::default(); runtime.update( ConversationStatus::Working, - settings(StatusSoundMode::Continuous), + settings(StatusSoundMode::WorkingAndWaiting), ); runtime.next_tick = Some(Instant::now() + Duration::from_secs(60)); runtime.update( ConversationStatus::Waiting, - settings(StatusSoundMode::Continuous), + settings(StatusSoundMode::WorkingAndWaiting), ); assert!(runtime.next_tick.unwrap() < Instant::now() + Duration::from_secs(1)); } @@ -486,10 +408,7 @@ mod tests { for status in [ConversationStatus::Working, ConversationStatus::Waiting] { player .play( - StatusSoundCue { - status, - volume: DEFAULT_STATUS_SOUND_VOLUME, - }, + StatusSoundCue { status }, None, ) .unwrap(); diff --git a/src-tauri/crates/berd-voice/tests/session_protocol.rs b/src-tauri/crates/berd-voice/tests/session_protocol.rs index bfe76e612..5bf1b8fb8 100644 --- a/src-tauri/crates/berd-voice/tests/session_protocol.rs +++ b/src-tauri/crates/berd-voice/tests/session_protocol.rs @@ -2525,7 +2525,7 @@ fn siri_session_reaches_ready_without_openai_credentials() { "type":"set_conversation_status", "id":19, "status":"working", - "settings":{"mode":"once","volume":0.25} + "settings":{"mode":"working"} }), ); stdin.flush().unwrap(); @@ -2535,7 +2535,7 @@ fn siri_session_reaches_ready_without_openai_credentials() { "type":"conversation_status_applied", "id":19, "status":"working", - "settings":{"mode":"once","volume":0.25} + "settings":{"mode":"working"} }) ); write_session_json( diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index c4ce8a02c..b91778f74 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -413,22 +413,55 @@ pub async fn update_openai_realtime_spokesperson_settings( #[derive(Default)] struct RealtimeStatusSoundActivity { - user_speaking: bool, + speaking_item_ids: std::collections::HashSet, + transcription_item_ids: std::collections::HashSet, playback_active: bool, } impl RealtimeStatusSoundActivity { fn update(&mut self, event: &serde_json::Value) -> Option { + let item_id = || { + event + .get("item_id") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + }; match event.get("type").and_then(serde_json::Value::as_str) { - Some("input_audio_buffer.speech_started") => self.user_speaking = true, - Some("input_audio_buffer.speech_stopped") => self.user_speaking = false, + Some("input_audio_buffer.speech_started") => { + if let Some(item_id) = item_id() { + self.speaking_item_ids.insert(item_id.clone()); + self.transcription_item_ids.insert(item_id); + } + } + Some("input_audio_buffer.speech_stopped") => { + if let Some(item_id) = item_id() { + self.speaking_item_ids.remove(&item_id); + } + } + Some( + "conversation.item.input_audio_transcription.completed" + | "conversation.item.input_audio_transcription.failed", + ) => { + if let Some(item_id) = item_id() { + self.speaking_item_ids.remove(&item_id); + self.transcription_item_ids.remove(&item_id); + } + } + Some("input_audio_buffer.cleared") => { + self.speaking_item_ids.clear(); + self.transcription_item_ids.clear(); + } Some("output_audio_buffer.started") => self.playback_active = true, Some("output_audio_buffer.stopped" | "output_audio_buffer.cleared") => { self.playback_active = false; } _ => return None, } - Some(self.user_speaking || self.playback_active) + Some( + !self.speaking_item_ids.is_empty() + || !self.transcription_item_ids.is_empty() + || self.playback_active, + ) } } @@ -838,7 +871,9 @@ mod tests { let mut activity = RealtimeStatusSoundActivity::default(); assert_eq!( - activity.update(&json!({ "type": "input_audio_buffer.speech_started" })), + activity.update( + &json!({ "type": "input_audio_buffer.speech_started", "item_id": "user-1" }) + ), Some(true) ); assert_eq!( @@ -846,16 +881,38 @@ mod tests { Some(true) ); assert_eq!( - activity.update(&json!({ "type": "input_audio_buffer.speech_stopped" })), + activity.update( + &json!({ "type": "input_audio_buffer.speech_stopped", "item_id": "user-1" }) + ), Some(true) ); assert_eq!( activity.update(&json!({ "type": "output_audio_buffer.stopped" })), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "conversation.item.input_audio_transcription.completed", "item_id": "user-1" })), Some(false) ); assert_eq!(activity.update(&json!({ "type": "response.done" })), None); } + #[test] + fn realtime_status_sound_activity_releases_abandoned_input_when_cleared() { + let mut activity = RealtimeStatusSoundActivity::default(); + + assert_eq!( + activity.update( + &json!({ "type": "input_audio_buffer.speech_started", "item_id": "user-1" }) + ), + Some(true) + ); + assert_eq!( + activity.update(&json!({ "type": "input_audio_buffer.cleared" })), + Some(false) + ); + } + #[test] fn realtime_status_sound_activity_treats_cleared_output_as_inactive() { let mut activity = RealtimeStatusSoundActivity::default(); diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index 9ea7033ab..59d1394a7 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -590,8 +590,7 @@ export function rejectVoiceConversationTranscript( } export interface VoiceStatusSoundSettings { - mode: "continuous" | "continuous-while-working" | "once" | "off"; - volume: number; + mode: "working" | "working-and-waiting"; } export async function updateVoiceConversationStatusSounds( diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts index a0da0c64b..a64cfb065 100644 --- a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.test.ts @@ -919,13 +919,13 @@ describe("useOpenAiRealtimeConversation lifecycle", () => { expect(mocks.updateStatusSounds).toHaveBeenCalledWith( "session-a", "waiting", - { mode: "continuous-while-working", volume: 0.4 }, + { mode: "working" }, ); - act(() => setStatusSoundPreference({ mode: "once", volume: 0.7 })); + act(() => setStatusSoundPreference({ mode: "working-and-waiting" })); expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( "session-a", "waiting", - { mode: "once", volume: 0.7 }, + { mode: "working-and-waiting" }, ); act(() => { mocks.preferenceListener?.({ voice: "cedar", speed: 1.5 }); @@ -966,7 +966,7 @@ describe("useOpenAiRealtimeConversation lifecycle", () => { expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( "session-a", "working", - { mode: "continuous-while-working", volume: 0.4 }, + { mode: "working" }, ); act(() => { @@ -978,7 +978,20 @@ describe("useOpenAiRealtimeConversation lifecycle", () => { expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( "session-a", "waiting", - { mode: "continuous-while-working", volume: 0.4 }, + { mode: "working" }, + ); + + act(() => { + useChatStore.getState().setChatState("session-a", "thinking"); + useChatStore.getState().setActiveRunId("session-a", "run-2"); + }); + await waitFor(() => + expect(owner.result.current.state).toBe("agent-working"), + ); + expect(mocks.updateStatusSounds).toHaveBeenLastCalledWith( + "session-a", + "working", + { mode: "working" }, ); }); diff --git a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts index d85d70092..736d86460 100644 --- a/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts +++ b/src/features/voice-conversation/hooks/useOpenAiRealtimeConversation.ts @@ -461,19 +461,22 @@ function resetRealtimeStatusWhenRunSettles( sessionId: string, ): void { let sawRun = false; + let publishedStatus: "working" | "waiting" = "working"; const check = () => { if (runtime.getSnapshot().boundSessionId !== sessionId) { unsubscribe(); return; } const master = useChatStore.getState().getSessionRuntime(sessionId); - if (master.activeRunId !== null || isSessionRunning(master.chatState)) { - sawRun = true; - return; - } + const working = + master.activeRunId !== null || isSessionRunning(master.chatState); + if (working) sawRun = true; if (!sawRun) return; - unsubscribe(); - runtime.markWaiting(sessionId); + const nextStatus = working ? "working" : "waiting"; + if (nextStatus === publishedStatus) return; + publishedStatus = nextStatus; + if (working) runtime.markWorking(sessionId); + else runtime.markWaiting(sessionId); }; const unsubscribe = useChatStore.subscribe(check); queueMicrotask(check); @@ -1228,10 +1231,22 @@ class OpenAiRealtimeConversationRuntime { this.setSnapshot(OFF_SNAPSHOT); } + markWorking(sessionId: string): void { + if (this.snapshot.boundSessionId !== sessionId) return; + this.setSnapshot({ ...this.snapshot, state: "agent-working" }); + const runtimeSessionId = this.realtimeRuntimeSessionId; + if (runtimeSessionId) { + publishRealtimeStatus(runtimeSessionId, "working"); + } + } + markWaiting(sessionId: string): void { if (this.snapshot.boundSessionId !== sessionId) return; this.setSnapshot({ ...this.snapshot, state: "listening" }); - publishRealtimeStatus(sessionId, "waiting"); + const runtimeSessionId = this.realtimeRuntimeSessionId; + if (runtimeSessionId) { + publishRealtimeStatus(runtimeSessionId, "waiting"); + } } private deliverToMaster( @@ -1301,7 +1316,10 @@ class OpenAiRealtimeConversationRuntime { }; if (!continueAfterStop) { this.setSnapshot({ ...this.snapshot, state: "agent-working" }); - publishRealtimeStatus(sessionId, "working"); + const runtimeSessionId = this.realtimeRuntimeSessionId; + if (runtimeSessionId) { + publishRealtimeStatus(runtimeSessionId, "working"); + } } for (;;) { const opportunity = await waitForMasterDeliveryOpportunity( diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 1992f11cd..505837e04 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -226,7 +226,26 @@ describe("voice transcript delivery coordination", () => { expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( expect.objectContaining({ sessionId: "session-1", revision: 3 }), "waiting", - { mode: "continuous-while-working", volume: 0.4 }, + { mode: "working" }, + ); + + voiceApiMocks.updateStatusSounds.mockClear(); + useChatStore.getState().setActiveRunId("session-1", "run-2"); + expect(useVoiceConversationStore.getState().uiState).toBe("agent-working"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1", revision: 3 }), + "working", + { mode: "working" }, + ); + + voiceApiMocks.updateStatusSounds.mockClear(); + useChatStore.getState().setActiveRunId("session-1", null); + useChatStore.getState().setError("session-1", "run failed"); + expect(useVoiceConversationStore.getState().uiState).toBe("listening"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1", revision: 3 }), + "waiting", + { mode: "working" }, ); }); @@ -258,7 +277,8 @@ describe("voice transcript delivery coordination", () => { microphoneMuted: false, revision: 3, }, - uiState: "agent-working", + uiState: "agent-speaking", + activityFallbackState: "agent-working", hydrated: true, init: vi.fn().mockResolvedValue(undefined), }); @@ -273,13 +293,13 @@ describe("voice transcript delivery coordination", () => { }), ); - act(() => setStatusSoundPreference({ mode: "once", volume: 0.7 })); + act(() => setStatusSoundPreference({ mode: "working-and-waiting" })); await waitFor(() => expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( expect.objectContaining({ sessionId: "session-1", revision: 3 }), "working", - { mode: "once", volume: 0.7 }, + { mode: "working-and-waiting" }, ), ); unmount(); @@ -338,7 +358,7 @@ describe("voice transcript delivery coordination", () => { expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( expect.objectContaining({ sessionId: "session-1", revision: 1 }), "working", - { mode: "continuous-while-working", volume: 0.4 }, + { mode: "working" }, ); }); diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 23aa77d0a..546651cbe 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -7,6 +7,7 @@ import type { ChatInputVoiceConversation, } from "@/features/chat/types"; import { useChatStore } from "@/features/chat/stores/chatStore"; +import { isSessionRunning } from "@/features/chat/lib/sessionActivity"; import { createSystemNotificationMessage } from "@/shared/types/messages"; import { steerPromptInSession } from "@/features/chat/lib/steerCore"; import { @@ -434,38 +435,51 @@ export function publishChainedVoiceStatus( ); } +const chainedRunStatusObservers = new Map void>(); + export function resetVoiceUiWhenRunSettles( sessionId: string, deliveryRevision: number, ): void { + chainedRunStatusObservers.get(sessionId)?.(); let sawRun = false; + let publishedStatus: "working" | "waiting" = "working"; + const cleanup = () => { + unsubscribeChat(); + unsubscribeVoice(); + if (chainedRunStatusObservers.get(sessionId) === cleanup) { + chainedRunStatusObservers.delete(sessionId); + } + }; const check = () => { const voice = useVoiceConversationStore.getState(); if ( voice.status.lifecycle !== "running" || voice.status.sessionId !== sessionId ) { - unsubscribeChat(); - unsubscribeVoice(); + cleanup(); return; } const runtime = useChatStore.getState().getSessionRuntime(sessionId); - if (runtime.activeRunId !== null || runtime.chatState !== "idle") { - sawRun = true; - return; - } + const working = + runtime.activeRunId !== null || isSessionRunning(runtime.chatState); + if (working) sawRun = true; if (!sawRun) return; - unsubscribeChat(); - unsubscribeVoice(); + const nextStatus = working ? "working" : "waiting"; + if (nextStatus === publishedStatus) return; if (voice.status.revision >= deliveryRevision) { - voice.setUiState("listening"); - publishChainedVoiceStatus(sessionId, "waiting"); + publishedStatus = nextStatus; + voice.setUiState( + nextStatus === "working" ? "agent-working" : "listening", + ); + publishChainedVoiceStatus(sessionId, nextStatus); } }; const unsubscribeChat = useChatStore.subscribe(check); const unsubscribeVoice = useVoiceConversationStore.subscribe(check); + chainedRunStatusObservers.set(sessionId, cleanup); queueMicrotask(check); } @@ -478,7 +492,7 @@ function ensureVoiceEventDeliveryInitialized() { if (voice.status.lifecycle !== "running" || !activeSessionId) return; publishChainedVoiceStatus( activeSessionId, - voice.uiState === "agent-working" ? "working" : "waiting", + voice.activityFallbackState === "agent-working" ? "working" : "waiting", preference, ); }); diff --git a/src/features/voice-conversation/lib/statusSoundPreference.test.ts b/src/features/voice-conversation/lib/statusSoundPreference.test.ts index 969b0e7f3..0dd566781 100644 --- a/src/features/voice-conversation/lib/statusSoundPreference.test.ts +++ b/src/features/voice-conversation/lib/statusSoundPreference.test.ts @@ -10,39 +10,35 @@ describe("status sound preference", () => { beforeEach(() => window.localStorage.clear()); afterEach(() => vi.restoreAllMocks()); - it("defaults to working pulses and one waiting cue", () => { - expect(getDefaultStatusSoundPreference()).toEqual({ - mode: "continuous-while-working", - volume: 0.4, - }); + it("defaults to repeating only while working", () => { + expect(getDefaultStatusSoundPreference()).toEqual({ mode: "working" }); expect(getStatusSoundPreference()).toEqual( getDefaultStatusSoundPreference(), ); }); - it("persists mode and volume", () => { - setStatusSoundPreference({ mode: "once", volume: 0.65 }); - expect(getStatusSoundPreference()).toEqual({ mode: "once", volume: 0.65 }); + it("persists the working-and-waiting mode", () => { + setStatusSoundPreference({ mode: "working-and-waiting" }); + expect(getStatusSoundPreference()).toEqual({ + mode: "working-and-waiting", + }); }); it("normalizes malformed persisted values", () => { window.localStorage.setItem( "goose:voice-status-sound-preference", - JSON.stringify({ mode: "unexpected", volume: 4 }), + JSON.stringify({ mode: "unexpected" }), ); - expect(getStatusSoundPreference()).toEqual({ - mode: "continuous-while-working", - volume: 1, - }); + expect(getStatusSoundPreference()).toEqual({ mode: "working" }); }); it("notifies runtime subscribers with the applied preference", () => { const listener = vi.fn(); const unsubscribe = subscribeToStatusSoundPreference(listener); - setStatusSoundPreference({ mode: "continuous", volume: 0.7 }); + setStatusSoundPreference({ mode: "working-and-waiting" }); - expect(listener).toHaveBeenCalledWith({ mode: "continuous", volume: 0.7 }); + expect(listener).toHaveBeenCalledWith({ mode: "working-and-waiting" }); unsubscribe(); }); @@ -50,7 +46,9 @@ describe("status sound preference", () => { vi.spyOn(window.localStorage, "setItem").mockImplementation(() => { throw new Error("storage unavailable"); }); - setStatusSoundPreference({ mode: "off", volume: 0.2 }); - expect(getStatusSoundPreference()).toEqual({ mode: "off", volume: 0.2 }); + setStatusSoundPreference({ mode: "working-and-waiting" }); + expect(getStatusSoundPreference()).toEqual({ + mode: "working-and-waiting", + }); }); }); diff --git a/src/features/voice-conversation/lib/statusSoundPreference.ts b/src/features/voice-conversation/lib/statusSoundPreference.ts index 44a48e20c..d35535163 100644 --- a/src/features/voice-conversation/lib/statusSoundPreference.ts +++ b/src/features/voice-conversation/lib/statusSoundPreference.ts @@ -1,40 +1,36 @@ import { useCallback, useSyncExternalStore } from "react"; -export type StatusSoundMode = - | "continuous" - | "continuous-while-working" - | "once" - | "off"; +export type StatusSoundMode = "working" | "working-and-waiting"; export interface StatusSoundPreference { mode: StatusSoundMode; - volume: number; } const STORAGE_KEY = "goose:voice-status-sound-preference"; const CHANGED_EVENT = "goose:voice-status-sound-preference-changed"; const DEFAULT_PREFERENCE: StatusSoundPreference = { - mode: "continuous-while-working", - volume: 0.4, + mode: "working", }; const DEFAULT_SNAPSHOT = JSON.stringify(DEFAULT_PREFERENCE); let volatilePreference: StatusSoundPreference | undefined; function normalize(value: unknown): StatusSoundPreference { if (!value || typeof value !== "object") return DEFAULT_PREFERENCE; - const candidate = value as Partial; - const mode = - candidate.mode === "continuous" || - candidate.mode === "continuous-while-working" || - candidate.mode === "once" || - candidate.mode === "off" - ? candidate.mode - : DEFAULT_PREFERENCE.mode; - const volume = - typeof candidate.volume === "number" && Number.isFinite(candidate.volume) - ? Math.min(1, Math.max(0, candidate.volume)) - : DEFAULT_PREFERENCE.volume; - return { mode, volume }; + const candidate = value as { mode?: unknown }; + const mode = (() => { + switch (candidate.mode) { + case "working-and-waiting": + case "continuous": + return "working-and-waiting"; + case "working": + case "continuous-while-working": + case "once": + case "off": + default: + return DEFAULT_PREFERENCE.mode; + } + })(); + return { mode }; } export function getDefaultStatusSoundPreference(): StatusSoundPreference { diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 84d27a1f4..7524f5129 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -68,12 +68,7 @@ const preferenceMocks = vi.hoisted(() => ({ setRealtimePreference: vi.fn(), })); const statusSoundState = vi.hoisted(() => ({ - mode: "continuous-while-working" as - | "continuous" - | "continuous-while-working" - | "once" - | "off", - volume: 0.4, + mode: "working" as "working" | "working-and-waiting", })); const interruptionState = vi.hoisted(() => ({ mode: "automatic" as "automatic" | "allowInterruptions" | "preventFeedback", @@ -186,10 +181,7 @@ vi.mock("../lib/voiceConversationModePreference", () => ({ }), })); vi.mock("../lib/statusSoundPreference", () => ({ - getDefaultStatusSoundPreference: () => ({ - mode: "continuous-while-working", - volume: 0.4, - }), + getDefaultStatusSoundPreference: () => ({ mode: "working" }), useStatusSoundPreference: () => ({ ...statusSoundState, update: preferenceMocks.setStatusSounds, @@ -346,6 +338,21 @@ describe("VoiceSettings", () => { preferenceMocks.setRealtimePreference.mockClear(); }); + it("offers only the two continuous status sound modes without a volume control", async () => { + const user = userEvent.setup(); + renderWithProviders(); + + expect(screen.queryByText("Status sound volume")).not.toBeInTheDocument(); + await user.click(screen.getByRole("combobox", { name: "Status sounds" })); + expect( + screen.getByRole("option", { name: "While working" }), + ).toBeInTheDocument(); + expect( + screen.getByRole("option", { name: "While working and waiting" }), + ).toBeInTheDocument(); + expect(screen.getAllByRole("option")).toHaveLength(2); + }); + it("describes voice modes by who the user talks with", () => { renderWithProviders(); diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index 162a526ac..19b8cec64 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -9,7 +9,6 @@ import { Badge } from "@/shared/ui/badge"; import { ConfirmDialog } from "@/shared/ui/confirm-dialog"; import { RadioGroup, RadioGroupCard } from "@/shared/ui/radio-group"; import { SettingsRow } from "@/shared/ui/settings-row"; -import { Slider } from "@/shared/ui/slider"; import { Select, SelectContent, @@ -75,10 +74,8 @@ import { } from "../lib/openAiVoiceOptions"; const STATUS_SOUND_MODES: StatusSoundMode[] = [ - "continuous", - "continuous-while-working", - "once", - "off", + "working", + "working-and-waiting", ]; const INTERRUPTION_MODES: VoiceInterruptionMode[] = [ @@ -643,26 +640,6 @@ export function VoiceSettings() { )} - details={ - statusSounds.mode === "off" ? null : ( -

-
- {t("voice.statusSoundVolume")} - - {Math.round(statusSounds.volume * 100)}% - -
- statusSounds.update({ volume })} - aria-label={t("voice.statusSoundVolume")} - /> -
- ) - } /> {resetError ? ( diff --git a/src/shared/api/openaiRealtime.ts b/src/shared/api/openaiRealtime.ts index bb3552c31..c0076ae07 100644 --- a/src/shared/api/openaiRealtime.ts +++ b/src/shared/api/openaiRealtime.ts @@ -55,8 +55,7 @@ export function updateOpenAiRealtimeStatusSounds( sessionId: string, status: "working" | "waiting", settings: { - mode: "continuous" | "continuous-while-working" | "once" | "off"; - volume: number; + mode: "working" | "working-and-waiting"; }, ): Promise { return invoke("update_openai_realtime_status_sounds", { diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index 0ec2efc8a..ef09f30c2 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -1100,18 +1100,13 @@ "voice": "Voice", "voiceLabel": "Pocket TTS voice", "statusSounds": "Status sounds", - "statusSoundVolume": "Status sound volume", "statusSoundModes": { - "continuous": "Always repeat", - "continuous-while-working": "Repeat while working", - "once": "Play once", - "off": "Off" + "working": "While working", + "working-and-waiting": "While working and waiting" }, "statusSoundModeDescriptions": { - "continuous": "Plays the current working or waiting cue every five seconds.", - "continuous-while-working": "Repeats the working cue every five seconds, then plays the waiting cue once.", - "once": "Plays one cue whenever the agent changes between working and waiting.", - "off": "Disables agent status sounds." + "working": "Repeats the working sound while the agent is working.", + "working-and-waiting": "Repeats the working sound while the agent is working and the waiting sound while it waits." } } } diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index ce580648c..4c338769b 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -1099,18 +1099,13 @@ "voice": "Voz", "voiceLabel": "Voz de Pocket TTS", "statusSounds": "Sonidos de estado", - "statusSoundVolume": "Volumen de los sonidos de estado", "statusSoundModes": { - "continuous": "Repetir siempre", - "continuous-while-working": "Repetir mientras trabaja", - "once": "Reproducir una vez", - "off": "Desactivados" + "working": "Mientras trabaja", + "working-and-waiting": "Mientras trabaja y espera" }, "statusSoundModeDescriptions": { - "continuous": "Reproduce la señal actual de trabajo o espera cada cinco segundos.", - "continuous-while-working": "Repite la señal de trabajo cada cinco segundos y luego reproduce una vez la señal de espera.", - "once": "Reproduce una señal cuando el agente cambia entre trabajo y espera.", - "off": "Desactiva los sonidos de estado del agente." + "working": "Repite el sonido de trabajo mientras el agente está trabajando.", + "working-and-waiting": "Repite el sonido de trabajo mientras el agente trabaja y el sonido de espera mientras espera." } } } From cd5f802f7ca610131d85602e08c0519ad3385257 Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Thu, 10 Sep 2026 12:48:43 -0400 Subject: [PATCH 13/29] Derive status sound settings defaults Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-voice/src/status_sounds.rs | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 489a0826b..097a4f5f8 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -24,20 +24,12 @@ pub enum ConversationStatus { Waiting, } -#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct StatusSoundSettings { pub mode: StatusSoundMode, } -impl Default for StatusSoundSettings { - fn default() -> Self { - Self { - mode: StatusSoundMode::default(), - } - } -} - #[derive(Clone, Copy, Debug, PartialEq)] pub struct StatusSoundCue { pub status: ConversationStatus, From 449111c651c92179d9028c5c380f83e911ea410b Mon Sep 17 00:00:00 2001 From: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> Date: Thu, 10 Sep 2026 12:58:55 -0400 Subject: [PATCH 14/29] Resume status cues after voice activity Signed-off-by: Sol <49aa1f65411fd096d2e2ec144f1e7aa36fdc76d1b907cfdf7be000c66f9d3b8e@buzz.block.builderlab.xyz> --- src-tauri/crates/berd-voice/PROTOCOL.md | 2 +- .../crates/berd-voice/src/status_sounds.rs | 89 +++++++++++++++++-- .../api/voiceConversation.ts | 1 + src/shared/api/openaiRealtime.ts | 1 + 4 files changed, 86 insertions(+), 7 deletions(-) diff --git a/src-tauri/crates/berd-voice/PROTOCOL.md b/src-tauri/crates/berd-voice/PROTOCOL.md index 80190f763..2c6971081 100644 --- a/src-tauri/crates/berd-voice/PROTOCOL.md +++ b/src-tauri/crates/berd-voice/PROTOCOL.md @@ -205,7 +205,7 @@ Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. The parent cannot author speaking state or finalized input; those are derived only from PCM by the child runtime. -`StatusSoundSettings` has a `mode` of `working` or `working-and-waiting`. No cue +`StatusSoundSettings` has a `mode` of `working` or `working-and-waiting` and an optional per-session `volume` from 0 to 1 that defaults to `0.8`. No cue is emitted until the first `set_conversation_status` request. The runtime then ticks immediately and every five seconds. `working` repeats only the working cue and stays silent while waiting; `working-and-waiting` repeats the current working diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 097a4f5f8..765524f27 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -6,7 +6,7 @@ use std::{ use serde::{Deserialize, Serialize}; -const STATUS_SOUND_GAIN: f32 = 0.4; +pub const DEFAULT_STATUS_SOUND_VOLUME: f32 = 0.8; pub const STATUS_SOUND_INTERVAL: Duration = Duration::from_secs(5); #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] @@ -24,15 +24,40 @@ pub enum ConversationStatus { Waiting, } -#[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct StatusSoundSettings { pub mode: StatusSoundMode, + #[serde(default = "default_status_sound_volume")] + pub volume: f32, +} + +impl Default for StatusSoundSettings { + fn default() -> Self { + Self { + mode: StatusSoundMode::default(), + volume: DEFAULT_STATUS_SOUND_VOLUME, + } + } +} + +impl StatusSoundSettings { + pub fn validate(self) -> Result { + if !self.volume.is_finite() || !(0.0..=1.0).contains(&self.volume) { + return Err("status sound volume must be finite and between 0 and 1"); + } + Ok(self) + } +} + +const fn default_status_sound_volume() -> f32 { + DEFAULT_STATUS_SOUND_VOLUME } #[derive(Clone, Copy, Debug, PartialEq)] pub struct StatusSoundCue { pub status: ConversationStatus, + pub volume: f32, } /// Pure policy for deciding which cue, if any, a fixed-cadence runtime tick plays. @@ -58,7 +83,10 @@ impl StatusSoundStateMachine { { return None; } - Some(StatusSoundCue { status }) + Some(StatusSoundCue { + status, + volume: settings.volume, + }) } } @@ -69,6 +97,7 @@ pub struct StatusSoundRuntime { next_tick: Option, player: StatusSoundPlayer, output_device: Option, + conversation_active: bool, } impl Default for StatusSoundRuntime { @@ -78,6 +107,7 @@ impl Default for StatusSoundRuntime { next_tick: None, player: StatusSoundPlayer::new(), output_device: None, + conversation_active: false, } } } @@ -105,6 +135,11 @@ impl StatusSoundRuntime { pub fn poll(&mut self, conversation_active: bool) -> Result { if conversation_active { self.stop(); + } else if self.conversation_active { + self.next_tick = Some(Instant::now()); + } + self.conversation_active = conversation_active; + if conversation_active { self.player.reap(); return Ok(false); } @@ -174,6 +209,7 @@ impl ManagedStatusSoundRuntime { status: ConversationStatus, settings: StatusSoundSettings, ) -> Result<(), String> { + settings.validate().map_err(str::to_string)?; self.send(StatusSoundCommand::Update(status, settings)) } @@ -259,7 +295,7 @@ impl StatusSoundPlayer { let samples = asset .samples .iter() - .map(|sample| sample * STATUS_SOUND_GAIN) + .map(|sample| sample * cue.volume) .collect::>(); player.enqueue(&samples)?; self.active.push(ActiveStatusSound { @@ -316,7 +352,10 @@ mod tests { use super::*; fn settings(mode: StatusSoundMode) -> StatusSoundSettings { - StatusSoundSettings { mode } + StatusSoundSettings { + mode, + ..StatusSoundSettings::default() + } } #[test] @@ -367,6 +406,41 @@ mod tests { assert_eq!(StatusSoundStateMachine::default().tick(false), None); } + #[test] + fn resuming_after_conversation_audio_plays_without_waiting_for_old_cadence() { + let mut runtime = StatusSoundRuntime::default(); + runtime.update( + ConversationStatus::Working, + settings(StatusSoundMode::Working), + ); + assert!(!runtime.poll(true).unwrap()); + runtime.next_tick = Some(Instant::now() + Duration::from_secs(60)); + let _ = runtime.poll(false); + assert!(runtime.next_tick.unwrap() < Instant::now() + STATUS_SOUND_INTERVAL); + } + + #[test] + fn session_volume_defaults_to_point_eight_and_accepts_an_override() { + assert_eq!(StatusSoundSettings::default().volume, 0.8); + let explicit = StatusSoundSettings { + mode: StatusSoundMode::Working, + volume: 0.25, + }; + assert_eq!(explicit.validate().unwrap().volume, 0.25); + } + + #[test] + fn rejects_invalid_session_volume() { + for volume in [f32::NAN, f32::INFINITY, -0.1, 1.1] { + assert!(StatusSoundSettings { + mode: StatusSoundMode::Working, + volume, + } + .validate() + .is_err()); + } + } + #[test] fn duplicate_updates_preserve_the_existing_cadence() { let mut runtime = StatusSoundRuntime::default(); @@ -400,7 +474,10 @@ mod tests { for status in [ConversationStatus::Working, ConversationStatus::Waiting] { player .play( - StatusSoundCue { status }, + StatusSoundCue { + status, + volume: DEFAULT_STATUS_SOUND_VOLUME, + }, None, ) .unwrap(); diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index 59d1394a7..916ff2668 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -591,6 +591,7 @@ export function rejectVoiceConversationTranscript( export interface VoiceStatusSoundSettings { mode: "working" | "working-and-waiting"; + volume?: number; } export async function updateVoiceConversationStatusSounds( diff --git a/src/shared/api/openaiRealtime.ts b/src/shared/api/openaiRealtime.ts index c0076ae07..cd9df8083 100644 --- a/src/shared/api/openaiRealtime.ts +++ b/src/shared/api/openaiRealtime.ts @@ -56,6 +56,7 @@ export function updateOpenAiRealtimeStatusSounds( status: "working" | "waiting", settings: { mode: "working" | "working-and-waiting"; + volume?: number; }, ): Promise { return invoke("update_openai_realtime_status_sounds", { From aa0d94303d2f2e3caec10667e1dfabb23cfa1293 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 09:41:44 -0400 Subject: [PATCH 15/29] docs(voice): correct status sound default volume --- src-tauri/crates/berd-voice/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/crates/berd-voice/README.md b/src-tauri/crates/berd-voice/README.md index 12da7725a..f38416d28 100644 --- a/src-tauri/crates/berd-voice/README.md +++ b/src-tauri/crates/berd-voice/README.md @@ -21,7 +21,7 @@ device; the existing Berd Siri player and the CLI use the same decoder. in [PROTOCOL.md](PROTOCOL.md). The host supplies persisted status-sound settings and semantic `working` / `waiting` updates through that protocol; the runtime owns the five-second cadence, speech suppression, and macOS Pop/Purr playback. -The default mode is `working`; status sounds use a fixed gain of `0.4`. +The default mode is `working`; status sounds default to a gain of `0.8`. Siri TTS and macOS speech recognition are the defaults: From 43cb03719b35fbacec50186727f27700e59de123 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 09:46:30 -0400 Subject: [PATCH 16/29] fix(voice): preserve legacy status sound modes --- .../lib/statusSoundPreference.test.ts | 13 ++++++++++ .../lib/statusSoundPreference.ts | 25 +++++++++---------- 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/src/features/voice-conversation/lib/statusSoundPreference.test.ts b/src/features/voice-conversation/lib/statusSoundPreference.test.ts index 0dd566781..9d45e52e2 100644 --- a/src/features/voice-conversation/lib/statusSoundPreference.test.ts +++ b/src/features/voice-conversation/lib/statusSoundPreference.test.ts @@ -32,6 +32,19 @@ describe("status sound preference", () => { expect(getStatusSoundPreference()).toEqual({ mode: "working" }); }); + it.each([ + ["continuous", "working-and-waiting"], + ["continuous-while-working", "working"], + ["once", "working"], + ["off", "working"], + ] as const)("migrates the legacy %s mode to %s", (legacy, expected) => { + window.localStorage.setItem( + "goose:voice-status-sound-preference", + JSON.stringify({ mode: legacy }), + ); + expect(getStatusSoundPreference()).toEqual({ mode: expected }); + }); + it("notifies runtime subscribers with the applied preference", () => { const listener = vi.fn(); const unsubscribe = subscribeToStatusSoundPreference(listener); diff --git a/src/features/voice-conversation/lib/statusSoundPreference.ts b/src/features/voice-conversation/lib/statusSoundPreference.ts index d35535163..6a4cda7f1 100644 --- a/src/features/voice-conversation/lib/statusSoundPreference.ts +++ b/src/features/voice-conversation/lib/statusSoundPreference.ts @@ -11,25 +11,24 @@ const CHANGED_EVENT = "goose:voice-status-sound-preference-changed"; const DEFAULT_PREFERENCE: StatusSoundPreference = { mode: "working", }; +const NORMALIZED_MODES: Record = { + continuous: "working-and-waiting", + "continuous-while-working": "working", + off: "working", + once: "working", + working: "working", + "working-and-waiting": "working-and-waiting", +}; const DEFAULT_SNAPSHOT = JSON.stringify(DEFAULT_PREFERENCE); let volatilePreference: StatusSoundPreference | undefined; function normalize(value: unknown): StatusSoundPreference { if (!value || typeof value !== "object") return DEFAULT_PREFERENCE; const candidate = value as { mode?: unknown }; - const mode = (() => { - switch (candidate.mode) { - case "working-and-waiting": - case "continuous": - return "working-and-waiting"; - case "working": - case "continuous-while-working": - case "once": - case "off": - default: - return DEFAULT_PREFERENCE.mode; - } - })(); + const mode = + typeof candidate.mode === "string" + ? (NORMALIZED_MODES[candidate.mode] ?? DEFAULT_PREFERENCE.mode) + : DEFAULT_PREFERENCE.mode; return { mode }; } From df408fc8b867a965eed578ac189ab89c162ff7c6 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 09:53:47 -0400 Subject: [PATCH 17/29] fix(voice): stabilize status sound formatting --- .../crates/berd-voice/src/status_sounds.rs | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 765524f27..31733ae43 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -77,10 +77,9 @@ impl StatusSoundStateMachine { pub fn tick(&mut self, conversation_active: bool) -> Option { let (status, settings) = self.current?; - if conversation_active - || (settings.mode == StatusSoundMode::Working - && status == ConversationStatus::Waiting) - { + let working_only = settings.mode == StatusSoundMode::Working; + let waiting = status == ConversationStatus::Waiting; + if conversation_active || (working_only && waiting) { return None; } Some(StatusSoundCue { @@ -358,6 +357,11 @@ mod tests { } } + fn assert_status(machine: &mut StatusSoundStateMachine, expected: ConversationStatus) { + let actual = machine.tick(false).map(|cue| cue.status); + assert_eq!(actual, Some(expected)); + } + #[test] fn defaults_to_working_only() { assert_eq!( @@ -371,11 +375,11 @@ mod tests { let mut machine = StatusSoundStateMachine::default(); let settings = settings(StatusSoundMode::WorkingAndWaiting); machine.update(ConversationStatus::Working, settings); - assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Working); - assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Working); + assert_status(&mut machine, ConversationStatus::Working); + assert_status(&mut machine, ConversationStatus::Working); machine.update(ConversationStatus::Waiting, settings); - assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Waiting); - assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Waiting); + assert_status(&mut machine, ConversationStatus::Waiting); + assert_status(&mut machine, ConversationStatus::Waiting); } #[test] @@ -383,8 +387,8 @@ mod tests { let mut machine = StatusSoundStateMachine::default(); let settings = settings(StatusSoundMode::Working); machine.update(ConversationStatus::Working, settings); - assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Working); - assert_eq!(machine.tick(false).unwrap().status, ConversationStatus::Working); + assert_status(&mut machine, ConversationStatus::Working); + assert_status(&mut machine, ConversationStatus::Working); machine.update(ConversationStatus::Waiting, settings); assert_eq!(machine.tick(false), None); assert_eq!(machine.tick(false), None); From eda164f27f2e0e48d0699a09eecad0e5f004edcb Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 11:08:05 -0400 Subject: [PATCH 18/29] fix(voice): separate run status from playback suppression --- src-tauri/src/commands/native_voice.rs | 112 +++++++++++------- .../useVoiceConversationController.test.ts | 18 ++- .../hooks/useVoiceConversationController.ts | 53 +++++++-- 3 files changed, 128 insertions(+), 55 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 1c0e71488..398713ac2 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -255,8 +255,7 @@ struct Runtime { native_microphone_mute_control: bool, status_sounds: Option, status_sound_user_speaking: bool, - status_sound_recognition_pending: bool, - status_sound_assistant_speaking: bool, + status_sound_playbacks: usize, admission: Option>, voice_input_quarantined: bool, } @@ -570,6 +569,32 @@ pub struct NativeVoiceState { #[must_use = "assistant speech policy ends when the guard is dropped"] pub(crate) struct AssistantSpeechGuard { _activity: Option, + runtime: Arc>, +} + +impl Runtime { + fn status_sound_suppressed(&self) -> bool { + self.status_sound_user_speaking || self.status_sound_playbacks > 0 + } + + fn publish_status_sound_suppression(&self) { + if let Some(status_sounds) = self.status_sounds.as_ref() { + if let Err(error) = + status_sounds.set_conversation_active(self.status_sound_suppressed()) + { + log::warn!("Could not update status sound activity: {error}"); + } + } + } +} + +impl Drop for AssistantSpeechGuard { + fn drop(&mut self) { + if let Ok(mut runtime) = self.runtime.lock() { + runtime.status_sound_playbacks -= 1; + runtime.publish_status_sound_suppression(); + } + } } impl NativeVoiceState { @@ -794,8 +819,13 @@ impl NativeVoiceState { .input_controls .begin_assistant_activity(sensitivity.vad_threshold(), input_during_tts) .ok(); + if let Ok(mut runtime) = self.runtime.lock() { + runtime.status_sound_playbacks += 1; + runtime.publish_status_sound_suppression(); + } AssistantSpeechGuard { _activity: activity, + runtime: Arc::clone(&self.runtime), } } @@ -1022,7 +1052,7 @@ impl NativeVoiceState { speaking: bool, ) -> Result<(), String> { let (owner_window_label, revision) = { - let mut runtime = self + let runtime = self .runtime .lock() .map_err(|_| "native voice state lock was poisoned".to_string())?; @@ -1041,13 +1071,6 @@ impl NativeVoiceState { "Only the voice conversation owner can report assistant activity.".into(), ); } - runtime.status_sound_assistant_speaking = speaking; - let conversation_active = runtime.status_sound_user_speaking - || runtime.status_sound_recognition_pending - || runtime.status_sound_assistant_speaking; - if let Some(status_sounds) = runtime.status_sounds.as_ref() { - status_sounds.set_conversation_active(conversation_active)?; - } (owner_window_label, runtime.revision) }; let event = NativeVoiceEvent::Activity { @@ -1070,8 +1093,7 @@ impl NativeVoiceState { &self, session_id: &str, revision: u64, - user_speaking: Option, - recognition_pending: Option, + user_speaking: bool, ) { let Ok(mut runtime) = self.runtime.lock() else { return; @@ -1079,20 +1101,8 @@ impl NativeVoiceState { if runtime.session_id.as_deref() != Some(session_id) || runtime.revision != revision { return; } - if let Some(speaking) = user_speaking { - runtime.status_sound_user_speaking = speaking; - } - if let Some(pending) = recognition_pending { - runtime.status_sound_recognition_pending = pending; - } - let conversation_active = runtime.status_sound_user_speaking - || runtime.status_sound_recognition_pending - || runtime.status_sound_assistant_speaking; - if let Some(status_sounds) = runtime.status_sounds.as_ref() { - if let Err(error) = status_sounds.set_conversation_active(conversation_active) { - log::warn!("Could not update status sound activity: {error}"); - } - } + runtime.status_sound_user_speaking = user_speaking; + runtime.publish_status_sound_suppression(); } fn take_stop_snapshot( @@ -1641,8 +1651,7 @@ pub async fn start_native_voice_conversation( .map_err(|error| log::warn!("Status sounds unavailable: {error}")) .ok(); runtime.status_sound_user_speaking = false; - runtime.status_sound_recognition_pending = false; - runtime.status_sound_assistant_speaking = false; + runtime.publish_status_sound_suppression(); runtime.admission = Some(Arc::new(BerdAdmissionCoordinator::default())); runtime.controls_ready = false; // Voice always starts from its owning session, where the in-session @@ -1759,12 +1768,7 @@ pub async fn start_native_voice_conversation( } berd_voice::input::VoiceInputEvent::SpeakingChanged(speaking) => { admission.set_user_speaking(speaking); - event_state.set_status_sound_input_activity( - &session_id, - revision, - Some(speaking), - None, - ); + event_state.set_status_sound_input_activity(&session_id, revision, speaking); let event = NativeVoiceEvent::Activity { session_id: session_id.clone(), activity: if speaking { @@ -1779,12 +1783,6 @@ pub async fn start_native_voice_conversation( } berd_voice::input::VoiceInputEvent::RecognitionPendingChanged(pending) => { admission.set_recognition_pending(pending); - event_state.set_status_sound_input_activity( - &session_id, - revision, - None, - Some(pending), - ); // The runtime owns recognition-pending sequencing. Berd's // renderer does not project that state yet. } @@ -3363,6 +3361,40 @@ mod tests { assert!(!caller_owns_target("voice-buddy", None, true,)); } + #[test] + fn status_sound_suppression_ends_between_speech_segments() { + let state = NativeVoiceState::default(); + for _ in 0..2 { + let speech = state.begin_assistant_speech( + InterruptionSensitivity::Balanced, + berd_voice::input::InputDuringTtsPolicy::AllowBargeIn, + ); + assert!(state.runtime.lock().unwrap().status_sound_suppressed()); + drop(speech); + assert!(!state.runtime.lock().unwrap().status_sound_suppressed()); + } + } + + #[test] + fn status_sounds_wait_for_all_audio_to_stop() { + let state = NativeVoiceState::default(); + let first = state.begin_assistant_speech( + InterruptionSensitivity::Balanced, + berd_voice::input::InputDuringTtsPolicy::AllowBargeIn, + ); + let second = state.begin_assistant_speech( + InterruptionSensitivity::Balanced, + berd_voice::input::InputDuringTtsPolicy::AllowBargeIn, + ); + drop(first); + assert!(state.runtime.lock().unwrap().status_sound_suppressed()); + state.runtime.lock().unwrap().status_sound_user_speaking = true; + drop(second); + assert!(state.runtime.lock().unwrap().status_sound_suppressed()); + state.runtime.lock().unwrap().status_sound_user_speaking = false; + assert!(!state.runtime.lock().unwrap().status_sound_suppressed()); + } + #[test] fn assistant_suppression_uses_the_shared_input_controls() { let state = NativeVoiceState::default(); diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 505837e04..072d7d113 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -67,6 +67,7 @@ import { createVoiceTranscriptDeliveryQueue, hasDeliveredVoiceTranscript, observeVoiceConversationControlVisibility, + observeChainedVoiceStatus, replaceActiveVoiceConversation, resetVoiceUiWhenRunSettles, resolveActiveVoiceButtonAction, @@ -215,11 +216,16 @@ describe("voice transcript delivery coordination", () => { activityFallbackState: "agent-working", }); + const stopObserving = observeChainedVoiceStatus(); resetVoiceUiWhenRunSettles("session-1", 3); await Promise.resolve(); expect(useVoiceConversationStore.getState().uiState).toBe("agent-working"); useChatStore.getState().setActiveRunId("session-1", "run-1"); + voiceApiMocks.updateStatusSounds.mockClear(); + useVoiceConversationStore.getState().setUiState("agent-speaking"); + useVoiceConversationStore.getState().setUiState("listening"); + expect(voiceApiMocks.updateStatusSounds).not.toHaveBeenCalled(); useChatStore.getState().setActiveRunId("session-1", null); expect(useVoiceConversationStore.getState().uiState).toBe("listening"); @@ -247,6 +253,7 @@ describe("voice transcript delivery coordination", () => { "waiting", { mode: "working" }, ); + stopObserving(); }); beforeEach(() => { @@ -293,7 +300,11 @@ describe("voice transcript delivery coordination", () => { }), ); - act(() => setStatusSoundPreference({ mode: "working-and-waiting" })); + act(() => { + useChatStore.getState().setActiveRunId("session-1", "run-1"); + useVoiceConversationStore.getState().setUiState("listening"); + setStatusSoundPreference({ mode: "working-and-waiting" }); + }); await waitFor(() => expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( @@ -306,7 +317,10 @@ describe("voice transcript delivery coordination", () => { }); it("delivers a queued transcript after its chat becomes temporarily ineligible", async () => { - const onSend = vi.fn().mockResolvedValue(true); + const onSend = vi.fn().mockImplementation(async () => { + useChatStore.getState().setActiveRunId("session-1", "run-1"); + return true; + }); useVoiceConversationStore.setState({ status: { available: true, diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 546651cbe..b09bb5dc4 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -437,6 +437,45 @@ export function publishChainedVoiceStatus( const chainedRunStatusObservers = new Map void>(); +export function observeChainedVoiceStatus(): () => void { + let published: string | null = null; + const check = () => { + const { status } = useVoiceConversationStore.getState(); + if ( + status.lifecycle !== "running" || + !status.sessionId || + status.ownerWindowLabel !== getCurrentWindow().label + ) { + published = null; + return; + } + const runtime = useChatStore.getState().getSessionRuntime(status.sessionId); + const conversationStatus = + runtime.activeRunId !== null || isSessionRunning(runtime.chatState) + ? "working" + : "waiting"; + const settings = getStatusSoundPreference(); + const next = JSON.stringify([ + status.sessionId, + status.revision, + conversationStatus, + settings, + ]); + if (next === published) return; + published = next; + publishChainedVoiceStatus(status.sessionId, conversationStatus, settings); + }; + const unsubscribeChat = useChatStore.subscribe(check); + const unsubscribeVoice = useVoiceConversationStore.subscribe(check); + const unsubscribePreference = subscribeToStatusSoundPreference(check); + check(); + return () => { + unsubscribeChat(); + unsubscribeVoice(); + unsubscribePreference(); + }; +} + export function resetVoiceUiWhenRunSettles( sessionId: string, deliveryRevision: number, @@ -474,7 +513,6 @@ export function resetVoiceUiWhenRunSettles( voice.setUiState( nextStatus === "working" ? "agent-working" : "listening", ); - publishChainedVoiceStatus(sessionId, nextStatus); } }; const unsubscribeChat = useChatStore.subscribe(check); @@ -486,16 +524,7 @@ export function resetVoiceUiWhenRunSettles( function ensureVoiceEventDeliveryInitialized() { if (deliveryInitialized) return; deliveryInitialized = true; - subscribeToStatusSoundPreference((preference) => { - const voice = useVoiceConversationStore.getState(); - const activeSessionId = voice.status.sessionId; - if (voice.status.lifecycle !== "running" || !activeSessionId) return; - publishChainedVoiceStatus( - activeSessionId, - voice.activityFallbackState === "agent-working" ? "working" : "waiting", - preference, - ); - }); + observeChainedVoiceStatus(); subscribeToVoiceConversationEvents(async (event) => { if (event.type === "cleanShutdown" || event.type === "controlsDismissed") { return; @@ -581,7 +610,6 @@ function ensureVoiceEventDeliveryInitialized() { displayText: event.text, }; store.setUiState("agent-working"); - publishChainedVoiceStatus(event.sessionId, "working"); const delivered = opportunity === "steer" ? await steerPromptInSession( @@ -1038,7 +1066,6 @@ export function useVoiceConversationController({ return "not-completed"; } startAssistantSpeech(assistantSpeechHistory); - publishChainedVoiceStatus(sessionId, "waiting"); return "completed"; } catch (startError) { const backendStatus = useVoiceConversationStore.getState().status; From 52c99ae2603f53226f283642463535a5e00820da Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 11:24:10 -0400 Subject: [PATCH 19/29] fix(voice): validate session status sound volume --- src-tauri/crates/berd-voice/src/main.rs | 32 +++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index d9749e507..4bd0b14a5 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -6601,6 +6601,9 @@ fn validate_request(request: SessionRequest) -> Result { return Err("request id must be positive".into()); } match &request { + SessionRequest::SetConversationStatus { settings, .. } => { + settings.validate().map_err(str::to_string)?; + } SessionRequest::PrepareSpeak { text, .. } if text.len() > MAX_SPEAK_TEXT_BYTES => { return Err("speak text exceeds 16 KiB".into()) } @@ -9591,6 +9594,35 @@ mod tests { assert!(control_receiver.try_recv().is_err()); } + #[test] + fn status_sound_requests_validate_volume() { + for volume in [-1.0, 2.0, f32::NAN, f32::INFINITY] { + let request = SessionRequest::SetConversationStatus { + id: 1, + status: berd_voice::ConversationStatus::Working, + settings: berd_voice::StatusSoundSettings { + volume, + ..Default::default() + }, + }; + assert_eq!( + validate_request(request).unwrap_err(), + "status sound volume must be finite and between 0 and 1" + ); + } + for volume in [0.0, 0.8, 1.0] { + assert!(validate_request(SessionRequest::SetConversationStatus { + id: 1, + status: berd_voice::ConversationStatus::Working, + settings: berd_voice::StatusSoundSettings { + volume, + ..Default::default() + }, + }) + .is_ok()); + } + } + #[test] fn input_policy_update_requires_a_positive_expected_revision() { let request = SessionRequest::SetInputDuringTts { From 4e0d454d2b7d30322d59ec33676ab15b861f1f50 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 11:27:24 -0400 Subject: [PATCH 20/29] fix(voice): retain status cue suppression transitions --- src-tauri/crates/berd-voice/src/main.rs | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index 4bd0b14a5..e9c4230bd 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -1350,12 +1350,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String core.recognition_pending(), active.is_some(), ); - let status_sound_result = if conversation_active { - status_sound_runtime.stop(); - Ok(false) - } else { - status_sound_runtime.poll(false) - }; + let status_sound_result = status_sound_runtime.poll(conversation_active); if let Err(message) = status_sound_result { eprintln!("status sound playback disabled: {message}"); } @@ -3353,12 +3348,7 @@ fn run_expert_spokesperson_session( turn_gate.input_blocks_output(), active.is_some(), ); - let status_sound_result = if conversation_active { - status_sound_runtime.stop(); - Ok(false) - } else { - status_sound_runtime.poll(false) - }; + let status_sound_result = status_sound_runtime.poll(conversation_active); if let Err(message) = status_sound_result { eprintln!("status sound playback disabled: {message}"); } From 81c5a653c3ac6dd4a982da3c3f6764a65174bec9 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 11:30:12 -0400 Subject: [PATCH 21/29] fix(voice): join managed status sound workers on shutdown --- .../crates/berd-voice/src/status_sounds.rs | 83 +++++++++++++++++-- src-tauri/src/commands/native_voice.rs | 8 +- src-tauri/src/commands/openai_realtime.rs | 12 ++- 3 files changed, 93 insertions(+), 10 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 31733ae43..62286e85d 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -1,5 +1,5 @@ use std::{ - sync::mpsc::{self, RecvTimeoutError, Sender}, + sync::{mpsc::{self, RecvTimeoutError, Sender}, Arc, Mutex}, thread, time::{Duration, Instant}, }; @@ -164,7 +164,12 @@ enum StatusSoundCommand { /// Thread-safe status-sound service for hosts that do not own a polling loop. #[derive(Clone)] pub struct ManagedStatusSoundRuntime { + inner: Arc, +} + +struct StatusSoundWorker { commands: Sender, + worker: Mutex>>, } impl ManagedStatusSoundRuntime { @@ -199,8 +204,12 @@ impl ManagedStatusSoundRuntime { } }) .map_err(|error| format!("Could not start status sound runtime: {error}"))?; - drop(worker); - Ok(Self { commands }) + Ok(Self { + inner: Arc::new(StatusSoundWorker { + commands, + worker: Mutex::new(Some(worker)), + }), + }) } pub fn update( @@ -221,16 +230,50 @@ impl ManagedStatusSoundRuntime { } fn send(&self, command: StatusSoundCommand) -> Result<(), String> { - self.commands + self.inner.commands .send(command) .map_err(|_| "Status sound runtime is unavailable".to_string()) } - pub fn finish(&self) { + pub fn finish(&self) -> Result<(), String> { + let _ = self.inner.commands.send(StatusSoundCommand::Shutdown); + let worker = self.inner.worker.lock() + .map_err(|_| "Status sound worker join state is unavailable")? + .take(); + if let Some(worker) = worker { + let deadline = Instant::now() + Duration::from_secs(2); + while !worker.is_finished() { + if Instant::now() >= deadline { + reap_status_sound_worker(worker); + return Err("Status sound worker shutdown timed out".into()); + } + thread::sleep(Duration::from_millis(10)); + } + worker.join().map_err(|_| "Status sound worker panicked".to_string())?; + } + Ok(()) + } +} + +impl Drop for StatusSoundWorker { + fn drop(&mut self) { let _ = self.commands.send(StatusSoundCommand::Shutdown); + if let Ok(worker) = self.worker.get_mut() { + if let Some(worker) = worker.take() { + reap_status_sound_worker(worker); + } + } } } +fn reap_status_sound_worker(worker: thread::JoinHandle<()>) { + let _ = thread::Builder::new() + .name("berd-status-sound-reaper".into()) + .spawn(move || { + let _ = worker.join(); + }); +} + #[cfg(not(target_os = "macos"))] struct StatusSoundPlayer; @@ -362,6 +405,36 @@ mod tests { assert_eq!(actual, Some(expected)); } + #[test] + fn managed_worker_finish_joins_and_is_idempotent_across_clones() { + let runtime = ManagedStatusSoundRuntime::spawn(None).unwrap(); + let other = runtime.clone(); + drop(runtime); + other.set_conversation_active(true).unwrap(); + other.finish().unwrap(); + assert!(other.inner.worker.lock().unwrap().is_none()); + assert!(other.set_conversation_active(false).is_err()); + other.finish().unwrap(); + } + + #[test] + fn last_owner_drop_stops_and_reaps_the_worker() { + let (commands, receiver) = mpsc::channel(); + let (stopped, completed) = mpsc::channel(); + let worker = thread::spawn(move || { + assert!(matches!(receiver.recv(), Ok(StatusSoundCommand::Shutdown))); + stopped.send(()).unwrap(); + }); + let runtime = ManagedStatusSoundRuntime { + inner: Arc::new(StatusSoundWorker { + commands, + worker: Mutex::new(Some(worker)), + }), + }; + drop(runtime); + completed.recv_timeout(Duration::from_secs(2)).unwrap(); + } + #[test] fn defaults_to_working_only() { assert_eq!( diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 398713ac2..9f6e30990 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -1853,7 +1853,9 @@ pub async fn start_native_voice_conversation( admission.close(); } if let Some(status_sounds) = current.status_sounds.take() { - status_sounds.finish(); + if let Err(error) = status_sounds.finish() { + log::warn!("Status sound shutdown failed: {error}"); + } } current.session_id = None; current.lifecycle_id = None; @@ -2300,7 +2302,9 @@ impl NativeVoiceState { return Ok(None); }; if let Some(status_sounds) = status_sounds { - status_sounds.finish(); + if let Err(error) = status_sounds.finish() { + log::warn!("Status sound shutdown failed: {error}"); + } } // Keep the lifecycle current through the bounded shutdown window so a // cooperative worker can flush its final utterance durably. A worker diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index b91778f74..294e5aae3 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -247,7 +247,9 @@ pub async fn stop_openai_realtime_spokesperson_runtime( .map(|entry| { ensure_runtime_owner(&entry.owner_window, webview_window.label())?; if let Some(status_sounds) = entry.status_sounds.as_ref() { - status_sounds.finish(); + if let Err(error) = status_sounds.finish() { + log::warn!("Status sound shutdown failed: {error}"); + } } Ok::<_, String>(Arc::clone(&entry.runtime)) }) @@ -279,7 +281,9 @@ pub async fn release_openai_realtime_spokesperson_runtime( }; if let Some(entry) = entry { if let Some(status_sounds) = entry.status_sounds.as_ref() { - status_sounds.finish(); + if let Err(error) = status_sounds.finish() { + log::warn!("Status sound shutdown failed: {error}"); + } } tauri::async_runtime::spawn_blocking(move || entry.runtime.finish()) .await @@ -302,7 +306,9 @@ pub fn handle_owner_window_destroyed(app: &AppHandle, window_label: &str) { .filter_map(|session_id| sessions.remove(&session_id)) .map(|entry| { if let Some(status_sounds) = entry.status_sounds.as_ref() { - status_sounds.finish(); + if let Err(error) = status_sounds.finish() { + log::warn!("Status sound shutdown failed: {error}"); + } } entry.runtime }) From c0a817e7c56192a96019ac5ab9528fa2f02b9caf Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 11:36:08 -0400 Subject: [PATCH 22/29] fix(voice): suppress speaker cue feedback into native input --- .../crates/berd-voice/src/status_sounds.rs | 49 +++++++++++++++++-- src-tauri/src/commands/native_voice.rs | 11 ++++- src-tauri/src/commands/pocket_voice.rs | 2 +- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 62286e85d..58b173415 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -97,6 +97,7 @@ pub struct StatusSoundRuntime { player: StatusSoundPlayer, output_device: Option, conversation_active: bool, + input_controls: Option, } impl Default for StatusSoundRuntime { @@ -107,11 +108,16 @@ impl Default for StatusSoundRuntime { player: StatusSoundPlayer::new(), output_device: None, conversation_active: false, + input_controls: None, } } } impl StatusSoundRuntime { + pub fn set_input_controls(&mut self, controls: Option) { + self.input_controls = controls; + } + pub fn set_output_device(&mut self, output_device: Option) { if self.output_device != output_device { self.player.stop(); @@ -147,7 +153,7 @@ impl StatusSoundRuntime { if self.next_tick.is_some_and(|deadline| now >= deadline) { self.next_tick = Some(now + STATUS_SOUND_INTERVAL); if let Some(cue) = self.machine.tick(conversation_active) { - self.player.play(cue, self.output_device.as_deref())?; + self.player.play(cue, self.output_device.as_deref(), self.input_controls.as_ref())?; } } Ok(self.player.is_active()) @@ -174,6 +180,13 @@ struct StatusSoundWorker { impl ManagedStatusSoundRuntime { pub fn spawn(output_device: Option) -> Result { + Self::spawn_with_input_controls(output_device, None) + } + + pub fn spawn_with_input_controls( + output_device: Option, + input_controls: Option, + ) -> Result { let (commands, receiver) = mpsc::channel(); let worker = thread::Builder::new() .name("berd-status-sounds".into()) @@ -181,6 +194,7 @@ impl ManagedStatusSoundRuntime { let mut runtime = StatusSoundRuntime::default(); let mut conversation_active = false; runtime.set_output_device(output_device); + runtime.set_input_controls(input_controls); loop { match receiver.recv_timeout(Duration::from_millis(10)) { Ok(StatusSoundCommand::Update(status, settings)) => { @@ -283,7 +297,7 @@ impl StatusSoundPlayer { Self } - fn play(&mut self, _cue: StatusSoundCue, _output_device: Option<&str>) -> Result<(), String> { + fn play(&mut self, _cue: StatusSoundCue, _output_device: Option<&str>, _input_controls: Option<&crate::input::VoiceInputControls>) -> Result<(), String> { Err("status sound playback is only available on macOS".into()) } @@ -303,6 +317,7 @@ const STATUS_SOUND_OUTPUT_TAIL: Duration = Duration::from_millis(100); struct ActiveStatusSound { player: crate::macos_audio_output::PocketAudioPlayer, output_tail_deadline: Option, + _input_activity: Option, } #[cfg(target_os = "macos")] @@ -322,7 +337,7 @@ impl StatusSoundPlayer { } } - fn play(&mut self, cue: StatusSoundCue, output_device: Option<&str>) -> Result<(), String> { + fn play(&mut self, cue: StatusSoundCue, output_device: Option<&str>, input_controls: Option<&crate::input::VoiceInputControls>) -> Result<(), String> { let asset = match cue.status { ConversationStatus::Working => &self.working, ConversationStatus::Waiting => &self.waiting, @@ -339,10 +354,14 @@ impl StatusSoundPlayer { .iter() .map(|sample| sample * cue.volume) .collect::>(); + let input_activity = input_controls.map(|controls| { + controls.begin_assistant_activity(0.65, crate::input::InputDuringTtsPolicy::SuppressInput) + }).transpose()?; player.enqueue(&samples)?; self.active.push(ActiveStatusSound { player, output_tail_deadline: None, + _input_activity: input_activity, }); Ok(()) } @@ -548,6 +567,7 @@ mod tests { #[ignore = "opens the default CoreAudio output and plays the macOS Pop and Purr cues"] fn macos_player_decodes_and_queues_both_status_cues() { let mut player = StatusSoundPlayer::new(); + let controls = crate::input::VoiceInputControls::default(); for status in [ConversationStatus::Working, ConversationStatus::Waiting] { player .play( @@ -556,10 +576,33 @@ mod tests { volume: DEFAULT_STATUS_SOUND_VOLUME, }, None, + Some(&controls), ) .unwrap(); } assert_eq!(player.active.len(), 2); + assert!(controls.is_muted()); player.stop(); + assert!(!controls.is_muted()); + } + + #[cfg(target_os = "macos")] + #[test] + #[ignore = "opens CoreAudio output and plays a status cue"] + fn macos_cue_suppresses_input_until_audio_and_tail_drain() { + let controls = crate::input::VoiceInputControls::default(); + let mut player = StatusSoundPlayer::new(); + player.play(StatusSoundCue { + status: ConversationStatus::Working, + volume: DEFAULT_STATUS_SOUND_VOLUME, + }, None, Some(&controls)).unwrap(); + let deadline = Instant::now() + Duration::from_secs(5); + while player.is_active() { + assert!(controls.is_muted()); + assert!(Instant::now() < deadline, "status cue did not drain"); + player.reap(); + thread::sleep(Duration::from_millis(10)); + } + assert!(!controls.is_muted()); } } diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index 9f6e30990..90ed055e1 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -1645,8 +1645,15 @@ pub async fn start_native_voice_conversation( window_label: window_label.clone(), }); runtime.pipeline = pipeline.take(); - runtime.status_sounds = berd_voice::ManagedStatusSoundRuntime::spawn( - super::pocket_voice::selected_output_device(), + let output_device = super::pocket_voice::selected_output_device(); + let effective_output_device = + super::pocket_voice::effective_output_device_name(output_device.as_deref()); + let cue_input_controls = + super::pocket_voice::output_device_uses_speakers(effective_output_device.as_deref()) + .then(|| state.input_controls.clone()); + runtime.status_sounds = berd_voice::ManagedStatusSoundRuntime::spawn_with_input_controls( + output_device, + cue_input_controls, ) .map_err(|error| log::warn!("Status sounds unavailable: {error}")) .ok(); diff --git a/src-tauri/src/commands/pocket_voice.rs b/src-tauri/src/commands/pocket_voice.rs index b9e1fbab9..7ac82dc90 100644 --- a/src-tauri/src/commands/pocket_voice.rs +++ b/src-tauri/src/commands/pocket_voice.rs @@ -422,7 +422,7 @@ pub(crate) fn effective_output_device_name(configured: Option<&str>) -> Option) -> Option { +pub(crate) fn effective_output_device_name(configured: Option<&str>) -> Option { configured.map(ToOwned::to_owned) } From aacad55d2c03e7efb250a8c67a2e71c877dd34bd Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 11:38:46 -0400 Subject: [PATCH 23/29] style(voice): format status sound crate for cross-platform checks --- .../crates/berd-voice/src/status_sounds.rs | 62 ++++++++++++++----- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 58b173415..53f2c4a12 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -1,5 +1,8 @@ use std::{ - sync::{mpsc::{self, RecvTimeoutError, Sender}, Arc, Mutex}, + sync::{ + mpsc::{self, RecvTimeoutError, Sender}, + Arc, Mutex, + }, thread, time::{Duration, Instant}, }; @@ -153,7 +156,11 @@ impl StatusSoundRuntime { if self.next_tick.is_some_and(|deadline| now >= deadline) { self.next_tick = Some(now + STATUS_SOUND_INTERVAL); if let Some(cue) = self.machine.tick(conversation_active) { - self.player.play(cue, self.output_device.as_deref(), self.input_controls.as_ref())?; + self.player.play( + cue, + self.output_device.as_deref(), + self.input_controls.as_ref(), + )?; } } Ok(self.player.is_active()) @@ -244,14 +251,18 @@ impl ManagedStatusSoundRuntime { } fn send(&self, command: StatusSoundCommand) -> Result<(), String> { - self.inner.commands + self.inner + .commands .send(command) .map_err(|_| "Status sound runtime is unavailable".to_string()) } pub fn finish(&self) -> Result<(), String> { let _ = self.inner.commands.send(StatusSoundCommand::Shutdown); - let worker = self.inner.worker.lock() + let worker = self + .inner + .worker + .lock() .map_err(|_| "Status sound worker join state is unavailable")? .take(); if let Some(worker) = worker { @@ -263,7 +274,9 @@ impl ManagedStatusSoundRuntime { } thread::sleep(Duration::from_millis(10)); } - worker.join().map_err(|_| "Status sound worker panicked".to_string())?; + worker + .join() + .map_err(|_| "Status sound worker panicked".to_string())?; } Ok(()) } @@ -297,7 +310,12 @@ impl StatusSoundPlayer { Self } - fn play(&mut self, _cue: StatusSoundCue, _output_device: Option<&str>, _input_controls: Option<&crate::input::VoiceInputControls>) -> Result<(), String> { + fn play( + &mut self, + _cue: StatusSoundCue, + _output_device: Option<&str>, + _input_controls: Option<&crate::input::VoiceInputControls>, + ) -> Result<(), String> { Err("status sound playback is only available on macOS".into()) } @@ -337,7 +355,12 @@ impl StatusSoundPlayer { } } - fn play(&mut self, cue: StatusSoundCue, output_device: Option<&str>, input_controls: Option<&crate::input::VoiceInputControls>) -> Result<(), String> { + fn play( + &mut self, + cue: StatusSoundCue, + output_device: Option<&str>, + input_controls: Option<&crate::input::VoiceInputControls>, + ) -> Result<(), String> { let asset = match cue.status { ConversationStatus::Working => &self.working, ConversationStatus::Waiting => &self.waiting, @@ -354,9 +377,14 @@ impl StatusSoundPlayer { .iter() .map(|sample| sample * cue.volume) .collect::>(); - let input_activity = input_controls.map(|controls| { - controls.begin_assistant_activity(0.65, crate::input::InputDuringTtsPolicy::SuppressInput) - }).transpose()?; + let input_activity = input_controls + .map(|controls| { + controls.begin_assistant_activity( + 0.65, + crate::input::InputDuringTtsPolicy::SuppressInput, + ) + }) + .transpose()?; player.enqueue(&samples)?; self.active.push(ActiveStatusSound { player, @@ -592,10 +620,16 @@ mod tests { fn macos_cue_suppresses_input_until_audio_and_tail_drain() { let controls = crate::input::VoiceInputControls::default(); let mut player = StatusSoundPlayer::new(); - player.play(StatusSoundCue { - status: ConversationStatus::Working, - volume: DEFAULT_STATUS_SOUND_VOLUME, - }, None, Some(&controls)).unwrap(); + player + .play( + StatusSoundCue { + status: ConversationStatus::Working, + volume: DEFAULT_STATUS_SOUND_VOLUME, + }, + None, + Some(&controls), + ) + .unwrap(); let deadline = Instant::now() + Duration::from_secs(5); while player.is_active() { assert!(controls.is_muted()); From 7b2100176ca082e59bcddb7f215b4e14401b64cd Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 12:33:55 -0400 Subject: [PATCH 24/29] feat(voice): add off mode and delay status cues to window end --- src-tauri/crates/berd-voice/README.md | 2 +- .../crates/berd-voice/src/status_sounds.rs | 48 ++++++++++++++++--- .../api/voiceConversation.ts | 2 +- .../lib/statusSoundPreference.test.ts | 6 ++- .../lib/statusSoundPreference.ts | 4 +- .../ui/VoiceSettings.test.tsx | 7 +-- .../voice-conversation/ui/VoiceSettings.tsx | 1 + src/shared/api/openaiRealtime.ts | 2 +- src/shared/i18n/locales/en/settings.json | 2 + src/shared/i18n/locales/es/settings.json | 2 + 10 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src-tauri/crates/berd-voice/README.md b/src-tauri/crates/berd-voice/README.md index f38416d28..fe8da8546 100644 --- a/src-tauri/crates/berd-voice/README.md +++ b/src-tauri/crates/berd-voice/README.md @@ -21,7 +21,7 @@ device; the existing Berd Siri player and the CLI use the same decoder. in [PROTOCOL.md](PROTOCOL.md). The host supplies persisted status-sound settings and semantic `working` / `waiting` updates through that protocol; the runtime owns the five-second cadence, speech suppression, and macOS Pop/Purr playback. -The default mode is `working`; status sounds default to a gain of `0.8`. +The default mode is `working`; `working-and-waiting` also plays idle cues, and `off` disables both. Status sounds default to a gain of `0.8`. Cues play at the end of each five-second window, including the first window and the window after conversation audio ends. Siri TTS and macOS speech recognition are the defaults: diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index 53f2c4a12..ded5b06d0 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -15,6 +15,7 @@ pub const STATUS_SOUND_INTERVAL: Duration = Duration::from_secs(5); #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub enum StatusSoundMode { + Off, WorkingAndWaiting, #[default] Working, @@ -82,7 +83,8 @@ impl StatusSoundStateMachine { let (status, settings) = self.current?; let working_only = settings.mode == StatusSoundMode::Working; let waiting = status == ConversationStatus::Waiting; - if conversation_active || (working_only && waiting) { + if settings.mode == StatusSoundMode::Off || conversation_active || (working_only && waiting) + { return None; } Some(StatusSoundCue { @@ -132,7 +134,7 @@ impl StatusSoundRuntime { let changed = self.machine.update(status, settings); if changed { self.player.stop(); - self.next_tick = Some(Instant::now()); + self.next_tick = Some(Instant::now() + STATUS_SOUND_INTERVAL); } } @@ -144,7 +146,7 @@ impl StatusSoundRuntime { if conversation_active { self.stop(); } else if self.conversation_active { - self.next_tick = Some(Instant::now()); + self.next_tick = Some(Instant::now() + STATUS_SOUND_INTERVAL); } self.conversation_active = conversation_active; if conversation_active { @@ -482,6 +484,37 @@ mod tests { completed.recv_timeout(Duration::from_secs(2)).unwrap(); } + #[test] + fn off_suppresses_both_statuses_and_round_trips_on_the_wire() { + let mode: StatusSoundMode = serde_json::from_str("\"off\"").unwrap(); + assert_eq!(mode, StatusSoundMode::Off); + assert_eq!(serde_json::to_string(&mode).unwrap(), "\"off\""); + let mut machine = StatusSoundStateMachine::default(); + for status in [ConversationStatus::Working, ConversationStatus::Waiting] { + machine.update(status, settings(mode)); + assert_eq!(machine.tick(false), None); + assert_eq!(machine.tick(true), None); + } + } + + #[test] + fn first_cue_waits_a_full_window_and_quick_runs_stay_silent() { + let mut runtime = StatusSoundRuntime::default(); + let started_at = Instant::now(); + runtime.update( + ConversationStatus::Working, + settings(StatusSoundMode::Working), + ); + assert!(runtime.next_tick.unwrap() >= started_at + STATUS_SOUND_INTERVAL); + assert!(!runtime.poll(false).unwrap()); + runtime.update( + ConversationStatus::Waiting, + settings(StatusSoundMode::Working), + ); + assert!(!runtime.poll(false).unwrap()); + assert_eq!(runtime.machine.tick(false), None); + } + #[test] fn defaults_to_working_only() { assert_eq!( @@ -531,7 +564,7 @@ mod tests { } #[test] - fn resuming_after_conversation_audio_plays_without_waiting_for_old_cadence() { + fn resuming_after_conversation_audio_starts_a_fresh_full_window() { let mut runtime = StatusSoundRuntime::default(); runtime.update( ConversationStatus::Working, @@ -539,7 +572,9 @@ mod tests { ); assert!(!runtime.poll(true).unwrap()); runtime.next_tick = Some(Instant::now() + Duration::from_secs(60)); - let _ = runtime.poll(false); + let resumed_at = Instant::now(); + assert!(!runtime.poll(false).unwrap()); + assert!(runtime.next_tick.unwrap() >= resumed_at + STATUS_SOUND_INTERVAL); assert!(runtime.next_tick.unwrap() < Instant::now() + STATUS_SOUND_INTERVAL); } @@ -587,7 +622,8 @@ mod tests { ConversationStatus::Waiting, settings(StatusSoundMode::WorkingAndWaiting), ); - assert!(runtime.next_tick.unwrap() < Instant::now() + Duration::from_secs(1)); + assert!(runtime.next_tick.unwrap() > Instant::now() + Duration::from_secs(4)); + assert!(runtime.next_tick.unwrap() <= Instant::now() + STATUS_SOUND_INTERVAL); } #[cfg(target_os = "macos")] diff --git a/src/features/voice-conversation/api/voiceConversation.ts b/src/features/voice-conversation/api/voiceConversation.ts index 916ff2668..a80850283 100644 --- a/src/features/voice-conversation/api/voiceConversation.ts +++ b/src/features/voice-conversation/api/voiceConversation.ts @@ -590,7 +590,7 @@ export function rejectVoiceConversationTranscript( } export interface VoiceStatusSoundSettings { - mode: "working" | "working-and-waiting"; + mode: "off" | "working" | "working-and-waiting"; volume?: number; } diff --git a/src/features/voice-conversation/lib/statusSoundPreference.test.ts b/src/features/voice-conversation/lib/statusSoundPreference.test.ts index 9d45e52e2..d01185eda 100644 --- a/src/features/voice-conversation/lib/statusSoundPreference.test.ts +++ b/src/features/voice-conversation/lib/statusSoundPreference.test.ts @@ -24,6 +24,11 @@ describe("status sound preference", () => { }); }); + it("persists off without re-enabling sounds", () => { + setStatusSoundPreference({ mode: "off" }); + expect(getStatusSoundPreference()).toEqual({ mode: "off" }); + }); + it("normalizes malformed persisted values", () => { window.localStorage.setItem( "goose:voice-status-sound-preference", @@ -36,7 +41,6 @@ describe("status sound preference", () => { ["continuous", "working-and-waiting"], ["continuous-while-working", "working"], ["once", "working"], - ["off", "working"], ] as const)("migrates the legacy %s mode to %s", (legacy, expected) => { window.localStorage.setItem( "goose:voice-status-sound-preference", diff --git a/src/features/voice-conversation/lib/statusSoundPreference.ts b/src/features/voice-conversation/lib/statusSoundPreference.ts index 6a4cda7f1..22cf70728 100644 --- a/src/features/voice-conversation/lib/statusSoundPreference.ts +++ b/src/features/voice-conversation/lib/statusSoundPreference.ts @@ -1,6 +1,6 @@ import { useCallback, useSyncExternalStore } from "react"; -export type StatusSoundMode = "working" | "working-and-waiting"; +export type StatusSoundMode = "off" | "working" | "working-and-waiting"; export interface StatusSoundPreference { mode: StatusSoundMode; @@ -14,7 +14,7 @@ const DEFAULT_PREFERENCE: StatusSoundPreference = { const NORMALIZED_MODES: Record = { continuous: "working-and-waiting", "continuous-while-working": "working", - off: "working", + off: "off", once: "working", working: "working", "working-and-waiting": "working-and-waiting", diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 7524f5129..f26a185ff 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -68,7 +68,7 @@ const preferenceMocks = vi.hoisted(() => ({ setRealtimePreference: vi.fn(), })); const statusSoundState = vi.hoisted(() => ({ - mode: "working" as "working" | "working-and-waiting", + mode: "working" as "off" | "working" | "working-and-waiting", })); const interruptionState = vi.hoisted(() => ({ mode: "automatic" as "automatic" | "allowInterruptions" | "preventFeedback", @@ -338,7 +338,7 @@ describe("VoiceSettings", () => { preferenceMocks.setRealtimePreference.mockClear(); }); - it("offers only the two continuous status sound modes without a volume control", async () => { + it("offers off and both repeating status sound modes without a volume control", async () => { const user = userEvent.setup(); renderWithProviders(); @@ -350,7 +350,8 @@ describe("VoiceSettings", () => { expect( screen.getByRole("option", { name: "While working and waiting" }), ).toBeInTheDocument(); - expect(screen.getAllByRole("option")).toHaveLength(2); + expect(screen.getByRole("option", { name: "Off" })).toBeInTheDocument(); + expect(screen.getAllByRole("option")).toHaveLength(3); }); it("describes voice modes by who the user talks with", () => { diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index 19b8cec64..eea396b17 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -74,6 +74,7 @@ import { } from "../lib/openAiVoiceOptions"; const STATUS_SOUND_MODES: StatusSoundMode[] = [ + "off", "working", "working-and-waiting", ]; diff --git a/src/shared/api/openaiRealtime.ts b/src/shared/api/openaiRealtime.ts index cd9df8083..356a01cc2 100644 --- a/src/shared/api/openaiRealtime.ts +++ b/src/shared/api/openaiRealtime.ts @@ -55,7 +55,7 @@ export function updateOpenAiRealtimeStatusSounds( sessionId: string, status: "working" | "waiting", settings: { - mode: "working" | "working-and-waiting"; + mode: "off" | "working" | "working-and-waiting"; volume?: number; }, ): Promise { diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index ef09f30c2..7658e72ee 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -1101,10 +1101,12 @@ "voiceLabel": "Pocket TTS voice", "statusSounds": "Status sounds", "statusSoundModes": { + "off": "Off", "working": "While working", "working-and-waiting": "While working and waiting" }, "statusSoundModeDescriptions": { + "off": "Do not play status sounds.", "working": "Repeats the working sound while the agent is working.", "working-and-waiting": "Repeats the working sound while the agent is working and the waiting sound while it waits." } diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index 4c338769b..adbace8e3 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -1100,10 +1100,12 @@ "voiceLabel": "Voz de Pocket TTS", "statusSounds": "Sonidos de estado", "statusSoundModes": { + "off": "Desactivados", "working": "Mientras trabaja", "working-and-waiting": "Mientras trabaja y espera" }, "statusSoundModeDescriptions": { + "off": "No reproducir sonidos de estado.", "working": "Repite el sonido de trabajo mientras el agente está trabajando.", "working-and-waiting": "Repite el sonido de trabajo mientras el agente trabaja y el sonido de espera mientras espera." } From fbe8bc66a06cdf795b9c7ad580d3818906ffa643 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 12:43:07 -0400 Subject: [PATCH 25/29] docs(voice): align status cue protocol with quiet windows --- src-tauri/crates/berd-voice/PROTOCOL.md | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src-tauri/crates/berd-voice/PROTOCOL.md b/src-tauri/crates/berd-voice/PROTOCOL.md index 2c6971081..880cb9b88 100644 --- a/src-tauri/crates/berd-voice/PROTOCOL.md +++ b/src-tauri/crates/berd-voice/PROTOCOL.md @@ -205,12 +205,7 @@ Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. The parent cannot author speaking state or finalized input; those are derived only from PCM by the child runtime. -`StatusSoundSettings` has a `mode` of `working` or `working-and-waiting` and an optional per-session `volume` from 0 to 1 that defaults to `0.8`. No cue -is emitted until the first `set_conversation_status` request. The runtime then -ticks immediately and every five seconds. `working` repeats only the working cue -and stays silent while waiting; `working-and-waiting` repeats the current working -or waiting cue. Active user input, pending recognition, or assistant output suppresses a tick without consuming its pending cue. On macOS, working uses the system Pop sound and waiting uses Purr through the native PCM player. The applied -request is acknowledged with: +`StatusSoundSettings` has a `mode` of `off`, `working`, or `working-and-waiting` and an optional per-session `volume` from 0 to 1 that defaults to `0.8`. No cue is emitted until the first `set_conversation_status` request. Cues play at the end of each five-second window, including the first window and the window after conversation audio ends. `off` disables both cues; `working` repeats only the working cue and stays silent while waiting; `working-and-waiting` repeats the current working or waiting cue. Active user speech or assistant output suppresses playback without changing the working/waiting state. Pending recognition alone does not suppress playback. On macOS, working uses the system Pop sound and waiting uses Purr through the selected output device. The applied request is acknowledged with: ```text {"type":"conversation_status_applied","id":u64,"status":"working"|"waiting","settings":StatusSoundSettings} From 259f1413cf438a7e0d29aef713c7b16d67473233 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 12:51:05 -0400 Subject: [PATCH 26/29] fix(voice): keep CLI status cues active during recognition --- src-tauri/crates/berd-voice/src/main.rs | 40 ++++++++++++------- .../berd-voice/src/realtime_host_lifecycle.rs | 4 ++ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src-tauri/crates/berd-voice/src/main.rs b/src-tauri/crates/berd-voice/src/main.rs index e9c4230bd..59d17cdba 100644 --- a/src-tauri/crates/berd-voice/src/main.rs +++ b/src-tauri/crates/berd-voice/src/main.rs @@ -1237,18 +1237,17 @@ fn run_management_command(command: ManagementCommand) -> Result<(), ManagementFa } fn standard_session_status_cues_suppressed( - user_speaking: bool, - recognition_pending: bool, + core: &SessionCore, assistant_output_active: bool, ) -> bool { - user_speaking || recognition_pending || assistant_output_active + core.user_speaking() || assistant_output_active } fn expert_session_status_cues_suppressed( - input_blocks_output: bool, + turn_gate: &ExpertTurnGate, assistant_output_active: bool, ) -> bool { - input_blocks_output || assistant_output_active + turn_gate.lifecycle.user_speaking() || assistant_output_active } fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String> { @@ -1345,11 +1344,7 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String }, )?; } - let conversation_active = standard_session_status_cues_suppressed( - core.user_speaking(), - core.recognition_pending(), - active.is_some(), - ); + let conversation_active = standard_session_status_cues_suppressed(&core, active.is_some()); let status_sound_result = status_sound_runtime.poll(conversation_active); if let Err(message) = status_sound_result { eprintln!("status sound playback disabled: {message}"); @@ -3344,10 +3339,8 @@ fn run_expert_spokesperson_session( )?; } } - let conversation_active = expert_session_status_cues_suppressed( - turn_gate.input_blocks_output(), - active.is_some(), - ); + let conversation_active = + expert_session_status_cues_suppressed(&turn_gate, active.is_some()); let status_sound_result = status_sound_runtime.poll(conversation_active); if let Err(message) = status_sound_result { eprintln!("status sound playback disabled: {message}"); @@ -6825,6 +6818,25 @@ mod tests { use std::os::unix::net::UnixStream; use std::sync::Mutex; + #[test] + fn status_cues_ignore_pending_recognition_but_suppress_actual_audio() { + let mut core = SessionCore::default(); + core.set_recognition_pending(true); + assert!(!standard_session_status_cues_suppressed(&core, false)); + assert!(standard_session_status_cues_suppressed(&core, true)); + core.set_user_speaking(true); + assert!(standard_session_status_cues_suppressed(&core, false)); + core.set_user_speaking(false); + assert!(!standard_session_status_cues_suppressed(&core, false)); + + let mut gate = ExpertTurnGate::default(); + gate.begin_user_speaking("pending-transcript".into()); + assert!(expert_session_status_cues_suppressed(&gate, false)); + gate.finish_user_speaking(); + assert!(gate.input_blocks_output()); + assert!(!expert_session_status_cues_suppressed(&gate, false)); + assert!(expert_session_status_cues_suppressed(&gate, true)); + } fn synthesis_config(tts: SynthesisTtsConfig, output: PathBuf) -> SynthesisConfig { SynthesisConfig { tts, diff --git a/src-tauri/crates/berd-voice/src/realtime_host_lifecycle.rs b/src-tauri/crates/berd-voice/src/realtime_host_lifecycle.rs index 3085d3c85..49e25ed47 100644 --- a/src-tauri/crates/berd-voice/src/realtime_host_lifecycle.rs +++ b/src-tauri/crates/berd-voice/src/realtime_host_lifecycle.rs @@ -86,6 +86,10 @@ impl RealtimeHostLifecycle { self.activity.input_blocks_output() } + pub fn user_speaking(&self) -> bool { + self.activity.user_speaking + } + pub fn is_busy(&self, work: RealtimeHostWork) -> bool { self.activity.is_busy(work) } From d8879d9bb557056f881b4e74f710ed5c0515d0c0 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 14:16:39 -0400 Subject: [PATCH 27/29] fix(voice): start working cues on the first live tool call --- src-tauri/crates/berd-voice/PROTOCOL.md | 2 +- src-tauri/crates/berd-voice/README.md | 2 +- .../crates/berd-voice/src/status_sounds.rs | 19 +++---- .../__tests__/acpNotificationHandler.test.ts | 19 +++++++ .../chat/acp/acpNotificationHandler.ts | 1 + .../chat/stores/__tests__/chatStore.test.ts | 20 ++++++++ src/features/chat/stores/chatStore.ts | 51 +++++++++++++++---- .../useVoiceConversationController.test.ts | 8 ++- .../hooks/useVoiceConversationController.ts | 3 +- src/shared/types/chat.ts | 2 + 10 files changed, 103 insertions(+), 24 deletions(-) diff --git a/src-tauri/crates/berd-voice/PROTOCOL.md b/src-tauri/crates/berd-voice/PROTOCOL.md index 880cb9b88..8306ff893 100644 --- a/src-tauri/crates/berd-voice/PROTOCOL.md +++ b/src-tauri/crates/berd-voice/PROTOCOL.md @@ -205,7 +205,7 @@ Unknown fields are rejected. IDs are positive. Speak text is at most 16 KiB. The parent cannot author speaking state or finalized input; those are derived only from PCM by the child runtime. -`StatusSoundSettings` has a `mode` of `off`, `working`, or `working-and-waiting` and an optional per-session `volume` from 0 to 1 that defaults to `0.8`. No cue is emitted until the first `set_conversation_status` request. Cues play at the end of each five-second window, including the first window and the window after conversation audio ends. `off` disables both cues; `working` repeats only the working cue and stays silent while waiting; `working-and-waiting` repeats the current working or waiting cue. Active user speech or assistant output suppresses playback without changing the working/waiting state. Pending recognition alone does not suppress playback. On macOS, working uses the system Pop sound and waiting uses Purr through the selected output device. The applied request is acknowledged with: +`StatusSoundSettings` has a `mode` of `off`, `working`, or `working-and-waiting` and an optional per-session `volume` from 0 to 1 that defaults to `0.8`. No cue is emitted until the first `set_conversation_status` request. Cues play immediately on status transitions and repeat every five seconds; repeated identical updates preserve the cadence. After conversation audio ends, the current cue resumes immediately. `off` disables both cues; `working` repeats only the working cue and stays silent while waiting; `working-and-waiting` repeats the current working or waiting cue. Active user speech or assistant output suppresses playback without changing the working/waiting state. Pending recognition alone does not suppress playback. On macOS, working uses the system Pop sound and waiting uses Purr through the selected output device. The applied request is acknowledged with: ```text {"type":"conversation_status_applied","id":u64,"status":"working"|"waiting","settings":StatusSoundSettings} diff --git a/src-tauri/crates/berd-voice/README.md b/src-tauri/crates/berd-voice/README.md index fe8da8546..0035f54d1 100644 --- a/src-tauri/crates/berd-voice/README.md +++ b/src-tauri/crates/berd-voice/README.md @@ -21,7 +21,7 @@ device; the existing Berd Siri player and the CLI use the same decoder. in [PROTOCOL.md](PROTOCOL.md). The host supplies persisted status-sound settings and semantic `working` / `waiting` updates through that protocol; the runtime owns the five-second cadence, speech suppression, and macOS Pop/Purr playback. -The default mode is `working`; `working-and-waiting` also plays idle cues, and `off` disables both. Status sounds default to a gain of `0.8`. Cues play at the end of each five-second window, including the first window and the window after conversation audio ends. +The default mode is `working`; `working-and-waiting` also plays idle cues, and `off` disables both. Status sounds default to a gain of `0.8`. Cues play immediately on a status transition and repeat every five seconds. When conversation audio ends, the current cue resumes immediately. Repeated identical status updates preserve the cadence. The host determines when work begins; the chained client signals working on its first live tool call and stays working until the run ends. Siri TTS and macOS speech recognition are the defaults: diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index ded5b06d0..ab44a7efa 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -134,7 +134,7 @@ impl StatusSoundRuntime { let changed = self.machine.update(status, settings); if changed { self.player.stop(); - self.next_tick = Some(Instant::now() + STATUS_SOUND_INTERVAL); + self.next_tick = Some(Instant::now()); } } @@ -146,7 +146,7 @@ impl StatusSoundRuntime { if conversation_active { self.stop(); } else if self.conversation_active { - self.next_tick = Some(Instant::now() + STATUS_SOUND_INTERVAL); + self.next_tick = Some(Instant::now()); } self.conversation_active = conversation_active; if conversation_active { @@ -498,15 +498,15 @@ mod tests { } #[test] - fn first_cue_waits_a_full_window_and_quick_runs_stay_silent() { + fn first_status_update_schedules_an_immediate_cue() { let mut runtime = StatusSoundRuntime::default(); let started_at = Instant::now(); runtime.update( ConversationStatus::Working, settings(StatusSoundMode::Working), ); - assert!(runtime.next_tick.unwrap() >= started_at + STATUS_SOUND_INTERVAL); - assert!(!runtime.poll(false).unwrap()); + assert!(runtime.next_tick.unwrap() >= started_at); + assert!(runtime.next_tick.unwrap() <= Instant::now()); runtime.update( ConversationStatus::Waiting, settings(StatusSoundMode::Working), @@ -564,7 +564,7 @@ mod tests { } #[test] - fn resuming_after_conversation_audio_starts_a_fresh_full_window() { + fn resuming_after_conversation_audio_plays_without_waiting_for_old_cadence() { let mut runtime = StatusSoundRuntime::default(); runtime.update( ConversationStatus::Working, @@ -572,9 +572,7 @@ mod tests { ); assert!(!runtime.poll(true).unwrap()); runtime.next_tick = Some(Instant::now() + Duration::from_secs(60)); - let resumed_at = Instant::now(); - assert!(!runtime.poll(false).unwrap()); - assert!(runtime.next_tick.unwrap() >= resumed_at + STATUS_SOUND_INTERVAL); + let _ = runtime.poll(false); assert!(runtime.next_tick.unwrap() < Instant::now() + STATUS_SOUND_INTERVAL); } @@ -622,8 +620,7 @@ mod tests { ConversationStatus::Waiting, settings(StatusSoundMode::WorkingAndWaiting), ); - assert!(runtime.next_tick.unwrap() > Instant::now() + Duration::from_secs(4)); - assert!(runtime.next_tick.unwrap() <= Instant::now() + STATUS_SOUND_INTERVAL); + assert!(runtime.next_tick.unwrap() <= Instant::now()); } #[cfg(target_os = "macos")] diff --git a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts index 1ce646783..b5de9856a 100644 --- a/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts +++ b/src/features/chat/acp/__tests__/acpNotificationHandler.test.ts @@ -173,6 +173,21 @@ describe("acpNotificationHandler", () => { ).not.toHaveBeenCalled(); }); + it("does not mark replayed tool calls as live run activity", async () => { + markSessionReplayLoading(); + await handleSessionNotification({ + sessionId: "acp-session", + update: { + sessionUpdate: "tool_call", + toolCallId: "old-tool", + title: "shell", + }, + } as never); + expect( + useChatStore.getState().getSessionRuntime("acp-session").hasToolCallInRun, + ).toBe(false); + }); + it("keeps tool calls that arrive before the first text chunk on the pending assistant message", async () => { registerPreparedSession("acp-session", "goose", "/Users/aharvard"); setActiveMessageId("acp-session", "assistant-1"); @@ -186,6 +201,10 @@ describe("acpNotificationHandler", () => { }, } as never); + expect( + useChatStore.getState().getSessionRuntime("acp-session").hasToolCallInRun, + ).toBe(true); + await handleSessionNotification({ sessionId: "acp-session", update: { diff --git a/src/features/chat/acp/acpNotificationHandler.ts b/src/features/chat/acp/acpNotificationHandler.ts index e2da1f650..b860a4c02 100644 --- a/src/features/chat/acp/acpNotificationHandler.ts +++ b/src/features/chat/acp/acpNotificationHandler.ts @@ -641,6 +641,7 @@ function handleLive(sessionId: string, update: SessionUpdate): void { case "tool_call": { flushBufferedStreamingUpdatesForSession(sessionId); + store.markToolCallInRun(sessionId); const messageId = ensureLiveAssistantMessage(sessionId); const identity = getToolCallIdentity(update); const chainSummary = getToolChainSummary(update); diff --git a/src/features/chat/stores/__tests__/chatStore.test.ts b/src/features/chat/stores/__tests__/chatStore.test.ts index 796c0a23f..dacdd1b85 100644 --- a/src/features/chat/stores/__tests__/chatStore.test.ts +++ b/src/features/chat/stores/__tests__/chatStore.test.ts @@ -120,6 +120,26 @@ describe("chatStore", () => { expect(messagesBySession.s11).toHaveLength(1); }); + it("latches live tool activity once and clears it between runs", () => { + const store = useChatStore.getState(); + store.setChatState("tools", "thinking"); + store.setActiveRunId("tools", "run-1"); + expect(store.getSessionRuntime("tools").hasToolCallInRun).toBe(false); + store.markToolCallInRun("tools"); + const working = store.getSessionRuntime("tools"); + store.markToolCallInRun("tools"); + expect(store.getSessionRuntime("tools")).toBe(working); + store.setChatState("tools", "streaming"); + expect(store.getSessionRuntime("tools").hasToolCallInRun).toBe(true); + store.setChatState("tools", "idle"); + store.settleActiveRun("tools"); + store.setActiveRunId("tools", "run-2"); + expect(store.getSessionRuntime("tools").hasToolCallInRun).toBe(false); + store.markToolCallInRun("tools"); + store.setActiveRunId("tools", "run-3"); + expect(store.getSessionRuntime("tools").hasToolCallInRun).toBe(false); + }); + it("does not evict inactive messages for a running session", () => { useChatStore.getState().setActiveSession("running"); useChatStore diff --git a/src/features/chat/stores/chatStore.ts b/src/features/chat/stores/chatStore.ts index 9e6f2a588..33ffb53c8 100644 --- a/src/features/chat/stores/chatStore.ts +++ b/src/features/chat/stores/chatStore.ts @@ -455,6 +455,7 @@ interface ChatStoreActions { clearSettledStreamingMessage: (sessionId: string) => boolean; settleActiveRun: (sessionId: string) => void; setActiveRunId: (sessionId: string, runId: string | null) => void; + markToolCallInRun: (sessionId: string) => void; setRunCancellationPending: (sessionId: string, pending: boolean) => void; setPendingInterventionBoundary: ( sessionId: string, @@ -924,6 +925,7 @@ const createChatStore: StateCreator< const shouldClearStreamTracking = !isSessionRunning(current.chatState); if ( current.activeRunId === null && + !current.hasToolCallInRun && !current.isRunCancellationPending && (!shouldClearStreamTracking || (current.streamingMessageId === null && @@ -942,6 +944,7 @@ const createChatStore: StateCreator< ...(shouldClearStreamTracking ? { streamingMessageId: null, + hasToolCallInRun: false, pendingInterventionBoundary: null, } : {}), @@ -950,17 +953,41 @@ const createChatStore: StateCreator< }; }), + markToolCallInRun: (sessionId) => + set((state) => { + const current = + state.sessionStateById[sessionId] ?? createInitialSessionRuntime(); + if (current.hasToolCallInRun) return state; + return { + sessionStateById: { + ...state.sessionStateById, + [sessionId]: { ...current, hasToolCallInRun: true }, + }, + }; + }), + setActiveRunId: (sessionId, activeRunId) => - set((state) => ({ - sessionStateById: { - ...state.sessionStateById, - [sessionId]: { - ...(state.sessionStateById[sessionId] ?? - createInitialSessionRuntime()), - activeRunId, + set((state) => { + const current = + state.sessionStateById[sessionId] ?? createInitialSessionRuntime(); + const replacedRun = + current.activeRunId !== null && + activeRunId !== null && + current.activeRunId !== activeRunId; + const settled = + activeRunId === null && !isSessionRunning(current.chatState); + return { + sessionStateById: { + ...state.sessionStateById, + [sessionId]: { + ...current, + activeRunId, + hasToolCallInRun: + replacedRun || settled ? false : current.hasToolCallInRun, + }, }, - }, - })), + }; + }), setRunCancellationPending: (sessionId, isRunCancellationPending) => set((state) => ({ @@ -1302,12 +1329,17 @@ const createChatStore: StateCreator< const current = state.sessionStateById[sessionId] ?? createInitialSessionRuntime(); const isIdle = chatState === "idle"; + const hasToolCallInRun = + !isSessionRunning(chatState) && current.activeRunId === null + ? false + : current.hasToolCallInRun; const streamingMessageId = isIdle ? null : current.streamingMessageId; const pendingInterventionBoundary = isIdle ? null : current.pendingInterventionBoundary; if ( current.chatState === chatState && + current.hasToolCallInRun === hasToolCallInRun && current.streamingMessageId === streamingMessageId && current.pendingInterventionBoundary === pendingInterventionBoundary ) { @@ -1320,6 +1352,7 @@ const createChatStore: StateCreator< [sessionId]: { ...current, chatState, + hasToolCallInRun, streamingMessageId, pendingInterventionBoundary, }, diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 072d7d113..050ac3e45 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -222,6 +222,7 @@ describe("voice transcript delivery coordination", () => { expect(useVoiceConversationStore.getState().uiState).toBe("agent-working"); useChatStore.getState().setActiveRunId("session-1", "run-1"); + useChatStore.getState().markToolCallInRun("session-1"); voiceApiMocks.updateStatusSounds.mockClear(); useVoiceConversationStore.getState().setUiState("agent-speaking"); useVoiceConversationStore.getState().setUiState("listening"); @@ -238,6 +239,8 @@ describe("voice transcript delivery coordination", () => { voiceApiMocks.updateStatusSounds.mockClear(); useChatStore.getState().setActiveRunId("session-1", "run-2"); expect(useVoiceConversationStore.getState().uiState).toBe("agent-working"); + expect(voiceApiMocks.updateStatusSounds).not.toHaveBeenCalled(); + useChatStore.getState().markToolCallInRun("session-1"); expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( expect.objectContaining({ sessionId: "session-1", revision: 3 }), "working", @@ -245,6 +248,8 @@ describe("voice transcript delivery coordination", () => { ); voiceApiMocks.updateStatusSounds.mockClear(); + useChatStore.getState().markToolCallInRun("session-1"); + expect(voiceApiMocks.updateStatusSounds).not.toHaveBeenCalled(); useChatStore.getState().setActiveRunId("session-1", null); useChatStore.getState().setError("session-1", "run failed"); expect(useVoiceConversationStore.getState().uiState).toBe("listening"); @@ -302,6 +307,7 @@ describe("voice transcript delivery coordination", () => { act(() => { useChatStore.getState().setActiveRunId("session-1", "run-1"); + useChatStore.getState().markToolCallInRun("session-1"); useVoiceConversationStore.getState().setUiState("listening"); setStatusSoundPreference({ mode: "working-and-waiting" }); }); @@ -371,7 +377,7 @@ describe("voice transcript delivery coordination", () => { ); expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( expect.objectContaining({ sessionId: "session-1", revision: 1 }), - "working", + "waiting", { mode: "working" }, ); }); diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index b09bb5dc4..6d113996d 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -451,7 +451,8 @@ export function observeChainedVoiceStatus(): () => void { } const runtime = useChatStore.getState().getSessionRuntime(status.sessionId); const conversationStatus = - runtime.activeRunId !== null || isSessionRunning(runtime.chatState) + runtime.hasToolCallInRun && + (runtime.activeRunId !== null || isSessionRunning(runtime.chatState)) ? "working" : "waiting"; const settings = getStatusSoundPreference(); diff --git a/src/shared/types/chat.ts b/src/shared/types/chat.ts index deacfff3d..1bb45851c 100644 --- a/src/shared/types/chat.ts +++ b/src/shared/types/chat.ts @@ -38,6 +38,7 @@ export interface SessionChatRuntime { hasUsageSnapshot: boolean; streamingMessageId: string | null; activeRunId: string | null; + hasToolCallInRun: boolean; isRunCancellationPending: boolean; pendingInterventionBoundary: { interventionMessageId: string; @@ -53,6 +54,7 @@ export const INITIAL_SESSION_CHAT_RUNTIME: SessionChatRuntime = { hasUsageSnapshot: false, streamingMessageId: null, activeRunId: null, + hasToolCallInRun: false, isRunCancellationPending: false, pendingInterventionBoundary: null, pendingAssistantProviderId: null, From a2228b4b709c1b829f328532810b1623784a21e8 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 14:50:34 -0400 Subject: [PATCH 28/29] fix(voice): silence waiting cues while a prompt awaits tools --- .../useVoiceConversationController.test.ts | 43 ++++++++++++++++++- .../hooks/useVoiceConversationController.ts | 12 +++--- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts index 050ac3e45..6752807b8 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.test.ts @@ -201,6 +201,43 @@ describe("voice transcript delivery coordination", () => { ).toBe(false); }); + it("silences waiting during a tool-free run without changing the preference", () => { + setStatusSoundPreference({ mode: "working-and-waiting" }); + useVoiceConversationStore.setState({ + status: { + available: true, + unavailableReason: null, + lifecycle: "running", + sessionId: "session-1", + ownerWindowLabel: "main", + microphoneMuted: false, + revision: 3, + }, + }); + const stopObserving = observeChainedVoiceStatus(); + const store = useChatStore.getState(); + store.setChatState("session-1", "thinking"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenLastCalledWith( + expect.anything(), + "waiting", + { mode: "off" }, + ); + store.setChatState("session-1", "idle"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenLastCalledWith( + expect.anything(), + "waiting", + { mode: "working-and-waiting" }, + ); + store.setActiveRunId("session-1", "run-1"); + store.markToolCallInRun("session-1"); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenLastCalledWith( + expect.anything(), + "working", + { mode: "working-and-waiting" }, + ); + stopObserving(); + }); + it("keeps working state until an admitted run actually settles", async () => { useVoiceConversationStore.setState({ status: { @@ -239,7 +276,11 @@ describe("voice transcript delivery coordination", () => { voiceApiMocks.updateStatusSounds.mockClear(); useChatStore.getState().setActiveRunId("session-1", "run-2"); expect(useVoiceConversationStore.getState().uiState).toBe("agent-working"); - expect(voiceApiMocks.updateStatusSounds).not.toHaveBeenCalled(); + expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: "session-1" }), + "waiting", + { mode: "off" }, + ); useChatStore.getState().markToolCallInRun("session-1"); expect(voiceApiMocks.updateStatusSounds).toHaveBeenCalledWith( expect.objectContaining({ sessionId: "session-1", revision: 3 }), diff --git a/src/features/voice-conversation/hooks/useVoiceConversationController.ts b/src/features/voice-conversation/hooks/useVoiceConversationController.ts index 6d113996d..4a5cb37bb 100644 --- a/src/features/voice-conversation/hooks/useVoiceConversationController.ts +++ b/src/features/voice-conversation/hooks/useVoiceConversationController.ts @@ -450,12 +450,14 @@ export function observeChainedVoiceStatus(): () => void { return; } const runtime = useChatStore.getState().getSessionRuntime(status.sessionId); + const running = + runtime.activeRunId !== null || isSessionRunning(runtime.chatState); const conversationStatus = - runtime.hasToolCallInRun && - (runtime.activeRunId !== null || isSessionRunning(runtime.chatState)) - ? "working" - : "waiting"; - const settings = getStatusSoundPreference(); + runtime.hasToolCallInRun && running ? "working" : "waiting"; + const settings = + running && !runtime.hasToolCallInRun + ? { mode: "off" as const } + : getStatusSoundPreference(); const next = JSON.stringify([ status.sessionId, status.revision, From 2fe690efc29b814239f2dd8547d413d5ffe546ee Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 11 Sep 2026 15:05:44 -0400 Subject: [PATCH 29/29] test(voice): name immediate status cue invariant --- src-tauri/crates/berd-voice/src/status_sounds.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/crates/berd-voice/src/status_sounds.rs b/src-tauri/crates/berd-voice/src/status_sounds.rs index ab44a7efa..935236172 100644 --- a/src-tauri/crates/berd-voice/src/status_sounds.rs +++ b/src-tauri/crates/berd-voice/src/status_sounds.rs @@ -564,7 +564,7 @@ mod tests { } #[test] - fn resuming_after_conversation_audio_plays_without_waiting_for_old_cadence() { + fn resuming_after_conversation_audio_schedules_cue_immediately() { let mut runtime = StatusSoundRuntime::default(); runtime.update( ConversationStatus::Working,