FCE-3579: voice activity detection as a high-frequency channel

packages/react-client/src/hooks/useVAD.ts

@@ -1,91 +1,29 @@
-import { useContext, useEffect, useMemo, useReducer } from "react";
+import { useContext, useMemo, useSyncExternalStore } from "react";
 
-import { FishjamClientStateContext } from "../contexts/fishjamState";
+import { FishjamClientContext } from "../contexts/fishjamClient";
 import type { PeerId } from "../types/public";
-import { useLocalVAD } from "./useLocalVAD";
 
 /**
- * Voice activity detection. Use this hook to check if voice is detected in the audio track for given peer(s).
+ * Hook that reports which of the requested peers are currently speaking.
  *
- * Remote peer VAD is driven by `vadNotification` messages from the backend.
- * If the local peer's id is included in `peerIds`, local VAD is determined client-side
- * by polling the microphone's audio level (see `useLocalVAD`).
+ * Remote peers are voice-activity-detected by the backend; the local peer is
+ * detected by sampling the microphone's audio level.
  *
- * @param options - Options object.
- * @param options.peerIds - List of peer ids to subscribe to for VAD notifications.
- *   Include the local peer's id to also track whether the local user is speaking.
- *
- * Example usage:
- * ```tsx
- * import { useVAD, type PeerId } from "@fishjam-cloud/react-client";
- *
- * function WhoIsTalkingComponent({ peerIds }: { peerIds: PeerId[] }) {
- *   const peersInfo = useVAD({ peerIds });
- *   const activePeers = (Object.keys(peersInfo) as PeerId[]).filter((peerId) => peersInfo[peerId]);
- *
- *   return "Now talking: " + activePeers.join(", ");
- * }
- * ```
  * @category Connection
  * @group Hooks
- * @returns A record where each key is a peer id and the boolean value indicates
- * whether voice activity is currently detected for that peer.
  */
-export const useVAD = (options: { peerIds: ReadonlyArray<PeerId> }): Record<PeerId, boolean> => {
-  const { peerIds } = options;
-  const clientState = useContext(FishjamClientStateContext);
-  if (!clientState) throw Error("useVAD must be used within FishjamProvider");
-  const showLocalPeerVAD = useMemo(
-    () => (clientState.localPeer?.id ? peerIds.includes(clientState.localPeer?.id) : false),
-    [clientState.localPeer?.id, peerIds],
-  );
-
-  const micTracksWithSelectedPeerIds = useMemo(
-    () =>
-      Object.values(clientState.peers)
-        .filter((peer) => peerIds.includes(peer.id))
-        .map((peer) => ({
-          peerId: peer.id,
-          microphoneTrack: Array.from(peer.tracks.values()).find(({ metadata }) => metadata?.type === "microphone"),
-        })),
-    [clientState.peers, peerIds],
-  );
-
-  // `voiceActivityChanged` mutates the track context in place and does not flow through
-  // `useFishjamClientState`, so we need an explicit re-render trigger to re-read the
-  // current `vadStatus` off each mic track.
-  const [version, bumpVersion] = useReducer((n: number) => n + 1, 0);
-
-  useEffect(() => {
-    const unsubs = micTracksWithSelectedPeerIds.map(({ microphoneTrack }) => {
-      if (!microphoneTrack) return () => {};
-
-      microphoneTrack.on("voiceActivityChanged", bumpVersion);
-
-      return () => {
-        microphoneTrack.off("voiceActivityChanged", bumpVersion);
-      };
-    });
-
-    return () => unsubs.forEach((unsub) => unsub());
-  }, [micTracksWithSelectedPeerIds]);
-
-  const localVAD = useLocalVAD({ disabled: !showLocalPeerVAD });
-
-  const vadStatuses = useMemo(() => {
-    // Referencing `version` makes the memo recompute on every `voiceActivityChanged`
-    // event, so we re-read each current mic track's mutable `vadStatus`.
-    void version;
-    return {
-      ...Object.fromEntries(
-        micTracksWithSelectedPeerIds.map(({ peerId, microphoneTrack }) => [
-          peerId,
-          microphoneTrack?.vadStatus === "speech",
-        ]),
-      ),
-      ...localVAD,
-    } satisfies Record<PeerId, boolean>;
-  }, [micTracksWithSelectedPeerIds, localVAD, version]);
-
-  return vadStatuses;
-};
+export function useVAD(options: { peerIds: ReadonlyArray<PeerId> }): Record<PeerId, boolean> {
+  const fishjamClientRef = useContext(FishjamClientContext);
+  if (!fishjamClientRef) throw Error("useVAD must be used within FishjamProvider");
+  const client = fishjamClientRef.current;
+
+  const voiceActivity = useSyncExternalStore(client.subscribeToVoiceActivity, client.getVoiceActivity);
+
+  return useMemo(() => {
+    const requested: Record<PeerId, boolean> = {};
+    for (const peerId of options.peerIds) {
+      if (peerId in voiceActivity) requested[peerId] = voiceActivity[peerId];
+    }
+    return requested;
+  }, [voiceActivity, options.peerIds]);
+}

packages/tsunami/src/FishjamClient.ts

@@ -36,6 +36,7 @@ import type {
 } from "./mediaTypes";
 import { type ClientState, createInitialClientState } from "./state/clientState";
 import { StateStore, type StoreListener } from "./state/StateStore";
+import { VoiceActivityMonitor } from "./vad/VoiceActivityMonitor";
 
 type LegacyClientInternals<PeerMetadata> = {
   reconnectManager?: { reset(metadata: PeerMetadata): void };
@@ -291,6 +292,31 @@ export class FishjamClient<PeerMetadata = GenericMetadata, ServerMetadata = Gene
   /** Notifies on every state change; returns an unsubscribe function. */
   public subscribe = (listener: StoreListener): (() => void) => this.store.subscribe(listener);
 
+  private voiceActivityMonitor: VoiceActivityMonitor | null = null;
+
+  /**
+   * Voice activity keyed by peer id, for every peer with a published
+   * microphone track. Stable reference until a value changes. High-frequency
+   * channel — deliberately separate from {@link getState}.
+   */
+  public getVoiceActivity = (): Record<string, boolean> => this.requireVoiceActivityMonitor().getSnapshot();
+
+  /** Notifies on any voice-activity change; returns an unsubscribe function. */
+  public subscribeToVoiceActivity = (listener: () => void): (() => void) =>
+    this.requireVoiceActivityMonitor().subscribe(listener);
+
+  private requireVoiceActivityMonitor(): VoiceActivityMonitor {
+    if (!this.voiceActivityMonitor) {
+      this.voiceActivityMonitor = new VoiceActivityMonitor({
+        getLocalPeer: () => this.getLocalPeer(),
+        getRemotePeers: () => this.getRemotePeers(),
+        getLocalTrackAudioLevel: (trackId) => this.getLocalTrackAudioLevel(trackId),
+        subscribeToPeerChanges: this.subscribe,
+      });
+    }
+    return this.voiceActivityMonitor;
+  }
+
   /** Notifies only when the selected part of the state changes (`Object.is`). */
   public subscribeToSlice = <Slice>(
     selector: (state: ClientState<PeerMetadata, ServerMetadata>) => Slice,
@@ -486,6 +512,7 @@ export class FishjamClient<PeerMetadata = GenericMetadata, ServerMetadata = Gene
 
     this.resources.dispose();
     this.deviceOrchestrator?.dispose();
+    this.voiceActivityMonitor?.dispose();
     this.store.clear();
 
     const tsClient = this.tsClient;