diff --git a/packages/react-client/src/hooks/useLocalVAD.ts b/packages/react-client/src/hooks/useLocalVAD.ts deleted file mode 100644 index 8c8578fd6..000000000 --- a/packages/react-client/src/hooks/useLocalVAD.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { useContext, useEffect, useState } from "react"; - -import { FishjamClientContext } from "../contexts/fishjamClient"; -import type { PeerId } from "../types/public"; -import { usePeers } from "./usePeers"; - -// This is a dBov-to-linear conversion. -32 dBov number is taken from backend VAD threshold -// formula for dBov to linear conversion: linear = 10 ^ (dBov / 20) -// So -32 dBov = 10^(-32/20) ≈ 0.025. This is the minimum audio level considered "speech". -const THRESHOLD = 10 ** (-32 / 20); - -// Number of consecutive "silence" ticks before we consider speech to have stopped. Helps with smoothing out brief pauses in speech. -const SILENCE_DEBOUNCE_TICKS = 2; - -/** - * Client-side voice activity detection for the local peer. - * - * Polls the local microphone's audio level every 100ms and derives a speech/silence - * state from it. A level above ~0.025 (approximately −32 dBov, scaled to [0, 1]) - * is treated as speech. Silence is debounced over 2 consecutive ticks (~200ms) - * to prevent rapid flapping. - * - * This is purely client-side — it does not signal other peers. Remote participants - * receive the local peer's VAD status via backend `vadNotification` messages. - * - * @internal Used by `useVAD` when the local peer's id is included in `peerIds`. - * @returns A record mapping the local peer's id to its current speaking state, - * or an empty object if `options.disabled` is true, the local peer is not available, or no microphone track is found. - */ -export const useLocalVAD = (options: { disabled: boolean }): Record => { - const fishjamClient = useContext(FishjamClientContext); - const [isSpeaking, setIsSpeaking] = useState(false); - const { localPeer } = usePeers(); - const localPeerId = localPeer?.id; - const microphoneTrackId = localPeer?.microphoneTrack?.trackId; - - useEffect(() => { - if (options.disabled || !localPeerId || !microphoneTrackId) return; - - let silenceTicks = 0; - let timeoutId: ReturnType | undefined; - const controller = new AbortController(); - const { signal } = controller; - - const poll = async () => { - if (signal.aborted) return; - - const trackAudio = await fishjamClient?.current?.getLocalTrackAudioLevel(microphoneTrackId); - if (signal.aborted) return; - - if (trackAudio != null && trackAudio.level > THRESHOLD) { - silenceTicks = 0; - setIsSpeaking(true); - } else { - silenceTicks += 1; - if (silenceTicks >= SILENCE_DEBOUNCE_TICKS) { - setIsSpeaking(false); - } - } - - if (signal.aborted) return; - timeoutId = setTimeout(poll, 100); - }; - - timeoutId = setTimeout(poll, 0); - - return () => { - controller.abort(); - clearTimeout(timeoutId); - setIsSpeaking(false); - }; - }, [options.disabled, fishjamClient, localPeerId, microphoneTrackId]); - - if (!localPeerId || options.disabled || !microphoneTrackId) return {}; - return { [localPeerId]: isSpeaking }; -}; diff --git a/packages/react-client/src/hooks/useVAD.ts b/packages/react-client/src/hooks/useVAD.ts index 8c5068e6e..3ec863093 100644 --- a/packages/react-client/src/hooks/useVAD.ts +++ b/packages/react-client/src/hooks/useVAD.ts @@ -1,91 +1,29 @@ -import { useContext, useEffect, useMemo, useReducer } from "react"; +import { useContext, useMemo, useSyncExternalStore } from "react"; -import { FishjamClientStateContext } from "../contexts/fishjamState"; +import { FishjamClientContext } from "../contexts/fishjamClient"; import type { PeerId } from "../types/public"; -import { useLocalVAD } from "./useLocalVAD"; /** - * Voice activity detection. Use this hook to check if voice is detected in the audio track for given peer(s). + * Hook that reports which of the requested peers are currently speaking. * - * Remote peer VAD is driven by `vadNotification` messages from the backend. - * If the local peer's id is included in `peerIds`, local VAD is determined client-side - * by polling the microphone's audio level (see `useLocalVAD`). + * Remote peers are voice-activity-detected by the backend; the local peer is + * detected by sampling the microphone's audio level. * - * @param options - Options object. - * @param options.peerIds - List of peer ids to subscribe to for VAD notifications. - * Include the local peer's id to also track whether the local user is speaking. - * - * Example usage: - * ```tsx - * import { useVAD, type PeerId } from "@fishjam-cloud/react-client"; - * - * function WhoIsTalkingComponent({ peerIds }: { peerIds: PeerId[] }) { - * const peersInfo = useVAD({ peerIds }); - * const activePeers = (Object.keys(peersInfo) as PeerId[]).filter((peerId) => peersInfo[peerId]); - * - * return "Now talking: " + activePeers.join(", "); - * } - * ``` * @category Connection * @group Hooks - * @returns A record where each key is a peer id and the boolean value indicates - * whether voice activity is currently detected for that peer. */ -export const useVAD = (options: { peerIds: ReadonlyArray }): Record => { - const { peerIds } = options; - const clientState = useContext(FishjamClientStateContext); - if (!clientState) throw Error("useVAD must be used within FishjamProvider"); - const showLocalPeerVAD = useMemo( - () => (clientState.localPeer?.id ? peerIds.includes(clientState.localPeer?.id) : false), - [clientState.localPeer?.id, peerIds], - ); - - const micTracksWithSelectedPeerIds = useMemo( - () => - Object.values(clientState.peers) - .filter((peer) => peerIds.includes(peer.id)) - .map((peer) => ({ - peerId: peer.id, - microphoneTrack: Array.from(peer.tracks.values()).find(({ metadata }) => metadata?.type === "microphone"), - })), - [clientState.peers, peerIds], - ); - - // `voiceActivityChanged` mutates the track context in place and does not flow through - // `useFishjamClientState`, so we need an explicit re-render trigger to re-read the - // current `vadStatus` off each mic track. - const [version, bumpVersion] = useReducer((n: number) => n + 1, 0); - - useEffect(() => { - const unsubs = micTracksWithSelectedPeerIds.map(({ microphoneTrack }) => { - if (!microphoneTrack) return () => {}; - - microphoneTrack.on("voiceActivityChanged", bumpVersion); - - return () => { - microphoneTrack.off("voiceActivityChanged", bumpVersion); - }; - }); - - return () => unsubs.forEach((unsub) => unsub()); - }, [micTracksWithSelectedPeerIds]); - - const localVAD = useLocalVAD({ disabled: !showLocalPeerVAD }); - - const vadStatuses = useMemo(() => { - // Referencing `version` makes the memo recompute on every `voiceActivityChanged` - // event, so we re-read each current mic track's mutable `vadStatus`. - void version; - return { - ...Object.fromEntries( - micTracksWithSelectedPeerIds.map(({ peerId, microphoneTrack }) => [ - peerId, - microphoneTrack?.vadStatus === "speech", - ]), - ), - ...localVAD, - } satisfies Record; - }, [micTracksWithSelectedPeerIds, localVAD, version]); - - return vadStatuses; -}; +export function useVAD(options: { peerIds: ReadonlyArray }): Record { + const fishjamClientRef = useContext(FishjamClientContext); + if (!fishjamClientRef) throw Error("useVAD must be used within FishjamProvider"); + const client = fishjamClientRef.current; + + const voiceActivity = useSyncExternalStore(client.subscribeToVoiceActivity, client.getVoiceActivity); + + return useMemo(() => { + const requested: Record = {}; + for (const peerId of options.peerIds) { + if (peerId in voiceActivity) requested[peerId] = voiceActivity[peerId]; + } + return requested; + }, [voiceActivity, options.peerIds]); +} diff --git a/packages/tsunami/src/FishjamClient.ts b/packages/tsunami/src/FishjamClient.ts index a83d0fdc3..aebe48d00 100644 --- a/packages/tsunami/src/FishjamClient.ts +++ b/packages/tsunami/src/FishjamClient.ts @@ -36,6 +36,7 @@ import type { } from "./mediaTypes"; import { type ClientState, createInitialClientState } from "./state/clientState"; import { StateStore, type StoreListener } from "./state/StateStore"; +import { VoiceActivityMonitor } from "./vad/VoiceActivityMonitor"; type LegacyClientInternals = { reconnectManager?: { reset(metadata: PeerMetadata): void }; @@ -291,6 +292,31 @@ export class FishjamClient void) => this.store.subscribe(listener); + private voiceActivityMonitor: VoiceActivityMonitor | null = null; + + /** + * Voice activity keyed by peer id, for every peer with a published + * microphone track. Stable reference until a value changes. High-frequency + * channel — deliberately separate from {@link getState}. + */ + public getVoiceActivity = (): Record => this.requireVoiceActivityMonitor().getSnapshot(); + + /** Notifies on any voice-activity change; returns an unsubscribe function. */ + public subscribeToVoiceActivity = (listener: () => void): (() => void) => + this.requireVoiceActivityMonitor().subscribe(listener); + + private requireVoiceActivityMonitor(): VoiceActivityMonitor { + if (!this.voiceActivityMonitor) { + this.voiceActivityMonitor = new VoiceActivityMonitor({ + getLocalPeer: () => this.getLocalPeer(), + getRemotePeers: () => this.getRemotePeers(), + getLocalTrackAudioLevel: (trackId) => this.getLocalTrackAudioLevel(trackId), + subscribeToPeerChanges: this.subscribe, + }); + } + return this.voiceActivityMonitor; + } + /** Notifies only when the selected part of the state changes (`Object.is`). */ public subscribeToSlice = ( selector: (state: ClientState) => Slice, @@ -486,6 +512,7 @@ export class FishjamClient Peer | null; + getRemotePeers: () => Record>; + getLocalTrackAudioLevel: (trackId: string) => Promise<{ level: number } | null>; + /** Change notifications that re-scan peers and their microphone tracks. */ + subscribeToPeerChanges: (listener: () => void) => () => void; +}; + +const findMicrophoneTrack = (peer: Peer): FishjamTrackContext | undefined => + [...peer.tracks.values()].find((trackContext) => trackContext.metadata?.type === "microphone"); + +/** + * Voice activity for every peer with a published microphone track, keyed by + * peer id. Kept OUTSIDE the client state store: local speech detection polls + * at 10 Hz and remote activity flips per utterance — routing that through + * state snapshots would notify every store subscriber. + * + * Remote activity mirrors the signalling `voiceActivityChanged` events; local + * activity is detected by polling the microphone's audio level (speech is + * reported instantly, silence after a short debounce). Polling runs only + * while at least one subscriber is registered. + */ +export class VoiceActivityMonitor { + private readonly listeners = new Set<() => void>(); + private snapshot: Record = {}; + + private peerChangesUnsubscribe: (() => void) | null = null; + private readonly remoteTrackSubscriptions = new Map void }>(); + + private localMicrophoneTrackId: string | null = null; + private localSpeaking = false; + private pollAbort: AbortController | null = null; + private pollTimeoutId: ReturnType | null = null; + + public constructor(private readonly deps: VoiceActivityMonitorDeps) {} + + public getSnapshot = (): Record => this.snapshot; + + public subscribe = (listener: () => void): (() => void) => { + this.listeners.add(listener); + if (this.listeners.size === 1) this.start(); + + return () => { + if (!this.listeners.delete(listener)) return; + if (this.listeners.size === 0) this.stop(); + }; + }; + + public dispose(): void { + this.stop(); + this.listeners.clear(); + this.snapshot = {}; + } + + private start(): void { + this.peerChangesUnsubscribe = this.deps.subscribeToPeerChanges(() => this.synchronize()); + this.synchronize(); + } + + private stop(): void { + this.peerChangesUnsubscribe?.(); + this.peerChangesUnsubscribe = null; + for (const { detach } of this.remoteTrackSubscriptions.values()) detach(); + this.remoteTrackSubscriptions.clear(); + this.stopLocalPolling(); + } + + /** Re-scans peers, reconciling remote track listeners and the local poll. */ + private synchronize(): void { + const remotePeers = this.deps.getRemotePeers(); + + for (const [peerId, subscription] of this.remoteTrackSubscriptions) { + const currentContext = remotePeers[peerId] && findMicrophoneTrack(remotePeers[peerId]); + if (currentContext !== subscription.context) { + subscription.detach(); + this.remoteTrackSubscriptions.delete(peerId); + } + } + + for (const [peerId, peer] of Object.entries(remotePeers)) { + if (this.remoteTrackSubscriptions.has(peerId)) continue; + const microphoneContext = findMicrophoneTrack(peer); + if (!microphoneContext) continue; + + const onVoiceActivityChanged = () => this.recompute(); + microphoneContext.on("voiceActivityChanged", onVoiceActivityChanged); + this.remoteTrackSubscriptions.set(peerId, { + context: microphoneContext, + detach: () => microphoneContext.off("voiceActivityChanged", onVoiceActivityChanged), + }); + } + + const localMicrophoneTrackId = this.resolveLocalMicrophoneTrackId(); + if (localMicrophoneTrackId !== this.localMicrophoneTrackId) { + this.localMicrophoneTrackId = localMicrophoneTrackId; + this.stopLocalPolling(); + if (localMicrophoneTrackId) this.startLocalPolling(localMicrophoneTrackId); + } + + this.recompute(); + } + + private resolveLocalMicrophoneTrackId(): string | null { + const localPeer = this.deps.getLocalPeer(); + if (!localPeer) return null; + return findMicrophoneTrack(localPeer)?.trackId ?? null; + } + + private startLocalPolling(microphoneTrackId: string): void { + const abort = new AbortController(); + this.pollAbort = abort; + let silenceTicks = 0; + + const poll = async () => { + if (abort.signal.aborted) return; + + const trackAudio = await this.deps.getLocalTrackAudioLevel(microphoneTrackId); + if (abort.signal.aborted) return; + + if (trackAudio != null && trackAudio.level > SPEECH_THRESHOLD) { + silenceTicks = 0; + this.setLocalSpeaking(true); + } else { + silenceTicks += 1; + if (silenceTicks >= SILENCE_DEBOUNCE_TICKS) this.setLocalSpeaking(false); + } + + if (abort.signal.aborted) return; + this.pollTimeoutId = setTimeout(() => void poll(), LOCAL_POLL_INTERVAL_MS); + }; + + this.pollTimeoutId = setTimeout(() => void poll(), 0); + } + + private stopLocalPolling(): void { + this.pollAbort?.abort(); + this.pollAbort = null; + if (this.pollTimeoutId !== null) clearTimeout(this.pollTimeoutId); + this.pollTimeoutId = null; + this.localSpeaking = false; + } + + private setLocalSpeaking(isSpeaking: boolean): void { + if (this.localSpeaking === isSpeaking) return; + this.localSpeaking = isSpeaking; + this.recompute(); + } + + private recompute(): void { + const next: Record = {}; + + for (const [peerId, peer] of Object.entries(this.deps.getRemotePeers())) { + next[peerId] = findMicrophoneTrack(peer)?.vadStatus === "speech"; + } + + const localPeer = this.deps.getLocalPeer(); + if (localPeer && this.localMicrophoneTrackId) { + next[localPeer.id] = this.localSpeaking; + } + + const previous = this.snapshot; + const previousKeys = Object.keys(previous); + const nextKeys = Object.keys(next); + const isUnchanged = + previousKeys.length === nextKeys.length && nextKeys.every((key) => previous[key] === next[key]); + if (isUnchanged) return; + + this.snapshot = next; + for (const listener of [...this.listeners]) listener(); + } +}