diff --git a/packages/react-client/src/FishjamProvider.tsx b/packages/react-client/src/FishjamProvider.tsx index bb6383c75..d6ad0b8f6 100644 --- a/packages/react-client/src/FishjamProvider.tsx +++ b/packages/react-client/src/FishjamProvider.tsx @@ -1,6 +1,17 @@ import { type FishjamClient, getLogger, type ReconnectConfig } from "@fishjam-cloud/ts-client"; -import { FishjamClient as TsunamiClient } from "@fishjam-cloud/tsunami"; -import { type PropsWithChildren, type RefObject, useMemo, useRef } from "react"; +import { + type DeviceError as CoreDeviceError, + type DeviceItem, + FishjamClient as TsunamiClient, + type IDevicePersistence, + type InitializeDevicesResult as CoreInitializeDevicesResult, + type LocalDeviceState, + type PlatformMediaStreamTrack, + type TrackDeviceController, + type TrackMiddleware as CoreTrackMiddleware, + WebDeviceManager, +} from "@fishjam-cloud/tsunami"; +import { type PropsWithChildren, type RefObject, useCallback, useMemo, useRef, useSyncExternalStore } from "react"; import { CameraContext } from "./contexts/camera"; import { CustomSourceContext } from "./contexts/customSource"; @@ -11,15 +22,19 @@ import { InitDevicesContext } from "./contexts/initDevices"; import { MicrophoneContext } from "./contexts/microphone"; import { PeerStatusContext } from "./contexts/peerStatus"; import { ScreenshareContext } from "./contexts/screenshare"; -import { VIDEO_TRACK_CONSTRAINTS } from "./devices/constraints"; -import { useMediaDevices } from "./hooks/internal/devices/useMediaDevices"; import { useCustomSourceManager } from "./hooks/internal/useCustomSourceManager"; import { useFishjamClientState } from "./hooks/internal/useFishjamClientState"; import { usePeerStatus } from "./hooks/internal/usePeerStatus"; import { useScreenShareManager } from "./hooks/internal/useScreenshareManager"; -import { useTrackManager } from "./hooks/internal/useTrackManager"; -import type { BandwidthLimits, PersistLastDeviceHandlers, StreamConfig } from "./types/public"; -import { mergeWithDefaultBandwitdthLimits } from "./utils/bandwidth"; +import type { DeviceManager, TrackManager } from "./types/internal"; +import type { + BandwidthLimits, + DeviceError, + InitializeDevicesResult, + PersistLastDeviceHandlers, + StreamConfig, + TrackMiddleware, +} from "./types/public"; import { getLastDevice, saveLastDevice } from "./utils/localStorage"; /** @@ -68,64 +83,168 @@ export interface FishjamProviderProps extends PropsWithChildren { fishjamClient?: FishjamClient; } +const asLegacyDeviceError = (error: CoreDeviceError | null): DeviceError | null => + error === null ? null : { name: error.name }; + +const asDomTrack = (track: PlatformMediaStreamTrack | null): MediaStreamTrack | null => + track as MediaStreamTrack | null; + +const asStartDeviceResult = async ( + result: Promise<[PlatformMediaStreamTrack, null] | [null, CoreDeviceError]>, +): Promise<[MediaStreamTrack, null] | [null, DeviceError]> => { + const [track, error] = await result; + if (error) return [null, { name: error.name }]; + return [track as MediaStreamTrack, null]; +}; + +const asSelectDeviceResult = async (result: Promise): Promise => { + const error = await result; + return error && { name: error.name }; +}; + +const toDevicePersistence = (handlers: PersistLastDeviceHandlers): IDevicePersistence => ({ + getLastDevice: (deviceType) => { + const device = handlers.getLastDevice(deviceType); + return device && { deviceId: device.deviceId, label: device.label, kind: deviceType }; + }, + saveLastDevice: (deviceType, device) => + handlers.saveLastDevice({ deviceId: device.deviceId, label: device.label } as MediaDeviceInfo, deviceType), +}); + /** - * Provides the Fishjam Context + * Provides the Fishjam Context. + * + * Device, track, and session logic lives in `@fishjam-cloud/tsunami`; this + * provider adapts the core client's store snapshots into the context shapes + * the hooks render from. + * * @category Components */ export function FishjamProvider(props: FishjamProviderProps) { const fishjamClientRef = useRef(null); if (fishjamClientRef.current === null) { + const persistHandlers = + props.persistLastDevice === false + ? undefined + : typeof props.persistLastDevice === "object" + ? props.persistLastDevice + : { getLastDevice, saveLastDevice }; + fishjamClientRef.current = new TsunamiClient({ reconnect: props.reconnect, debug: props.debug, signallingClient: props.fishjamClient, + deviceManager: new WebDeviceManager({ + persistence: persistHandlers && toDevicePersistence(persistHandlers), + }), + videoConstraints: props.constraints?.video, + audioConstraints: props.constraints?.audio, + bandwidthLimits: props.bandwidthLimits, + videoStreamConfig: props.videoConfig, + audioStreamConfig: props.audioConfig, }); } const client = fishjamClientRef.current; + const devices = client.devices; + if (!devices) throw Error("FishjamProvider always injects a device manager"); - const persistHandlers = useMemo(() => { - if (props.persistLastDevice === false) return undefined; - - if (typeof props.persistLastDevice === "object") return props.persistLastDevice; - - return { getLastDevice, saveLastDevice }; - }, [props.persistLastDevice]); + const clientState = useSyncExternalStore(client.subscribe, client.getState); + const peerStatus = usePeerStatus(client); - const logger = useMemo(() => getLogger(props.debug ?? false), [props.debug]); + const buildDeviceManager = useCallback( + ( + controller: TrackDeviceController, + deviceState: LocalDeviceState, + deviceList: DeviceItem[], + deviceError: CoreDeviceError | null, + ): DeviceManager => ({ + startDevice: (deviceId) => asStartDeviceResult(controller.startDevice(deviceId ?? undefined)), + stopDevice: () => controller.stopDevice(), + selectDevice: (deviceId) => asStartDeviceResult(controller.startDevice(deviceId)), + activeDevice: deviceState.activeDevice, + deviceTrack: asDomTrack(deviceState.track), + deviceList, + deviceEnabled: deviceState.isEnabled, + enableDevice: () => controller.enableDevice(), + disableDevice: () => controller.disableDevice(), + currentMiddleware: deviceState.middleware as TrackMiddleware, + applyMiddleware: (middleware) => controller.applyMiddleware(middleware as CoreTrackMiddleware).then(asDomTrack), + deviceError: asLegacyDeviceError(deviceError), + selectedDevice: (deviceState.selectedDevice as unknown as MediaDeviceInfo) ?? null, + }), + [], + ); - const { cameraManager, microphoneManager, initializeDevices } = useMediaDevices({ - videoConstraints: props.constraints?.video ?? VIDEO_TRACK_CONSTRAINTS, - audioConstraints: props.constraints?.audio ?? true, - persistHandlers, - logger, - }); + const buildTrackManager = useCallback( + (controller: TrackDeviceController, deviceState: LocalDeviceState): TrackManager => ({ + selectDevice: (deviceId) => asSelectDeviceResult(controller.selectDevice(deviceId)), + stopDevice: () => controller.stopDevice(), + startDevice: (deviceId) => asStartDeviceResult(controller.startDevice(deviceId ?? undefined)), + deviceTrack: asDomTrack(deviceState.track), + currentMiddleware: deviceState.middleware as TrackMiddleware, + setTrackMiddleware: (middleware) => controller.setTrackMiddleware(middleware as CoreTrackMiddleware), + toggleMute: () => controller.toggleMute(), + toggleDevice: () => asSelectDeviceResult(controller.toggleDevice()), + }), + [], + ); - const peerStatus = usePeerStatus(client); + const cameraContext = useMemo( + () => ({ + videoTrackManager: buildTrackManager(devices.camera, clientState.camera), + cameraManager: buildDeviceManager( + devices.camera, + clientState.camera, + clientState.availableCameras, + clientState.cameraError, + ), + }), + [ + buildTrackManager, + buildDeviceManager, + devices, + clientState.camera, + clientState.availableCameras, + clientState.cameraError, + ], + ); - const mergedBandwidthLimits = useMemo( - () => mergeWithDefaultBandwitdthLimits(props.bandwidthLimits), - [props.bandwidthLimits], + const microphoneContext = useMemo( + () => ({ + audioTrackManager: buildTrackManager(devices.microphone, clientState.microphone), + microphoneManager: buildDeviceManager( + devices.microphone, + clientState.microphone, + clientState.availableMicrophones, + clientState.microphoneError, + ), + }), + [ + buildTrackManager, + buildDeviceManager, + devices, + clientState.microphone, + clientState.availableMicrophones, + clientState.microphoneError, + ], ); - const audioTrackManager = useTrackManager({ - tsClient: client, - peerStatus, - deviceManager: microphoneManager, - bandwidthLimits: mergedBandwidthLimits, - streamConfig: props.audioConfig, - type: "microphone", - logger, - }); + const initializeDevices = useCallback( + async (settings?: { enableVideo?: boolean; enableAudio?: boolean }): Promise => { + const result: CoreInitializeDevicesResult = await client.initializeDevices(settings); + return { + status: result.status, + stream: result.stream as MediaStream | null, + errors: result.errors && { + audio: asLegacyDeviceError(result.errors.audio), + video: asLegacyDeviceError(result.errors.video), + }, + }; + }, + [client], + ); - const videoTrackManager = useTrackManager({ - tsClient: client, - peerStatus, - deviceManager: cameraManager, - bandwidthLimits: mergedBandwidthLimits, - streamConfig: props.videoConfig, - type: "camera", - logger, - }); + const logger = useMemo(() => getLogger(props.debug ?? false), [props.debug]); const screenShareManager = useScreenShareManager({ fishjamClient: client, @@ -133,12 +252,6 @@ export function FishjamProvider(props: FishjamProviderProps) { logger, }); - const cameraContext = useMemo(() => ({ videoTrackManager, cameraManager }), [videoTrackManager, cameraManager]); - const microphoneContext = useMemo( - () => ({ audioTrackManager, microphoneManager }), - [audioTrackManager, microphoneManager], - ); - const customSourceManager = useCustomSourceManager({ fishjamClient: client, peerStatus, diff --git a/packages/react-client/src/contexts/camera.ts b/packages/react-client/src/contexts/camera.ts index f44ee6796..16a54e947 100644 --- a/packages/react-client/src/contexts/camera.ts +++ b/packages/react-client/src/contexts/camera.ts @@ -1,7 +1,6 @@ import { createContext } from "react"; -import type { DeviceManager } from "../hooks/internal/devices/useDeviceManager"; -import type { TrackManager } from "../types/internal"; +import type { DeviceManager, TrackManager } from "../types/internal"; export type CameraContextType = { videoTrackManager: TrackManager; diff --git a/packages/react-client/src/contexts/initDevices.ts b/packages/react-client/src/contexts/initDevices.ts index c6e498e02..b34a5df93 100644 --- a/packages/react-client/src/contexts/initDevices.ts +++ b/packages/react-client/src/contexts/initDevices.ts @@ -1,7 +1,6 @@ import { createContext } from "react"; -import type { InitializeDevicesSettings } from "../hooks/internal/devices/useMediaDevices"; -import type { InitializeDevicesResult } from "../types/public"; +import type { InitializeDevicesResult, InitializeDevicesSettings } from "../types/public"; export const InitDevicesContext = createContext< ((settings?: InitializeDevicesSettings) => Promise) | null diff --git a/packages/react-client/src/contexts/microphone.ts b/packages/react-client/src/contexts/microphone.ts index dd0c905b8..78ad87910 100644 --- a/packages/react-client/src/contexts/microphone.ts +++ b/packages/react-client/src/contexts/microphone.ts @@ -1,7 +1,6 @@ import { createContext } from "react"; -import type { DeviceManager } from "../hooks/internal/devices/useDeviceManager"; -import type { TrackManager } from "../types/internal"; +import type { DeviceManager, TrackManager } from "../types/internal"; export type MicrophoneContextType = { audioTrackManager: TrackManager; diff --git a/packages/react-client/src/devices/constraints.ts b/packages/react-client/src/devices/constraints.ts deleted file mode 100644 index bfedbe721..000000000 --- a/packages/react-client/src/devices/constraints.ts +++ /dev/null @@ -1,50 +0,0 @@ -export const VIDEO_TRACK_CONSTRAINTS: MediaTrackConstraints = { - width: { - max: 1280, - ideal: 1280, - min: 640, - }, - height: { - max: 720, - ideal: 720, - min: 320, - }, - frameRate: { - max: 30, - ideal: 24, - }, -}; - -export const SCREEN_SHARING_MEDIA_CONSTRAINTS: MediaStreamConstraints = { - video: { - frameRate: { ideal: 20, max: 25 }, - width: { max: 1920, ideal: 1920 }, - height: { max: 1080, ideal: 1080 }, - }, -}; - -export const prepareMediaTrackConstraints = ( - deviceId: string | undefined, - constraints: MediaTrackConstraints | undefined | boolean, -): MediaTrackConstraints | boolean => { - const trackConstraints = typeof constraints === "boolean" ? {} : constraints; - - if (!deviceId) return { ...trackConstraints }; - - return { ...trackConstraints, deviceId: { exact: deviceId } }; -}; - -export const prepareConstraints = ( - deviceIdToStart: string | undefined, - constraints: MediaTrackConstraints | undefined | boolean, -): MediaTrackConstraints | undefined | boolean => { - if (!deviceIdToStart) return constraints; - - // The resulting stream will not contain a track of this type, - // which means that the device will not be activated. - if (constraints === false) return false; - - const constraintsObj = constraints === true ? {} : constraints; - - return { ...constraintsObj, deviceId: { ideal: deviceIdToStart } }; -}; diff --git a/packages/react-client/src/devices/mediaInitializer.ts b/packages/react-client/src/devices/mediaInitializer.ts deleted file mode 100644 index 6d2c9f890..000000000 --- a/packages/react-client/src/devices/mediaInitializer.ts +++ /dev/null @@ -1,151 +0,0 @@ -import type { AudioVideo, CurrentDevices } from "../types/internal"; -import type { DeviceError } from "../types/public"; -import { NOT_FOUND_ERROR, OVERCONSTRAINED_ERROR, PERMISSION_DENIED, UNHANDLED_ERROR } from "../utils/errors"; -import { prepareConstraints } from "./constraints"; - -type MediaConstraints = AudioVideo; -type PreviousDevices = AudioVideo; - -type GetMediaResult = { stream: MediaStream | null; errors: AudioVideo }; - -const defaultErrors = { audio: null, video: null }; - -const errorMap: Record = { - NotFoundError: NOT_FOUND_ERROR, - OverconstrainedError: OVERCONSTRAINED_ERROR, - NotAllowedError: PERMISSION_DENIED, -}; - -const getSingleMedia = async ( - type: T, - constraints: MediaStreamConstraints[T], -): Promise<[MediaStream, null] | [null, DeviceError]> => { - const baseConstraints = { audio: undefined, video: undefined }; - - try { - const stream = await navigator.mediaDevices.getUserMedia({ ...baseConstraints, [type]: constraints }); - return [stream, null]; - } catch (err) { - const name = err instanceof Error ? err.name : ""; - return [null, errorMap[name] ?? UNHANDLED_ERROR]; - } -}; - -const tryToGetAudioOnlyThenVideoOnly = async ( - constraints: MediaStreamConstraints, - initialError: DeviceError, -): Promise => { - const [audioStream, audioErr] = await getSingleMedia("audio", constraints.audio); - if (audioStream) return { stream: audioStream, errors: { video: initialError, audio: null } }; - - const [videoStream, videoErr] = await getSingleMedia("video", constraints.video); - return { stream: videoStream, errors: { audio: audioErr, video: videoErr } }; -}; - -export const getAvailableMedia = async ( - constraints: MediaStreamConstraints, - errors: AudioVideo = defaultErrors, -): Promise => { - try { - return { stream: await navigator.mediaDevices.getUserMedia(constraints), errors }; - } catch (err: unknown) { - const name = err instanceof Error ? err.name : ""; - switch (name) { - case errors.audio?.name: - case errors.video?.name: - return { stream: null, errors }; - case "NotFoundError": - return tryToGetAudioOnlyThenVideoOnly(constraints, PERMISSION_DENIED); - case "OverconstrainedError": - return getAvailableMedia( - { audio: unspecifyDevice(constraints.audio), video: unspecifyDevice(constraints.video) }, - { audio: OVERCONSTRAINED_ERROR, video: OVERCONSTRAINED_ERROR }, - ); - case "NotAllowedError": - return tryToGetAudioOnlyThenVideoOnly(constraints, PERMISSION_DENIED); - default: - return { stream: null, errors: { audio: UNHANDLED_ERROR, video: UNHANDLED_ERROR } }; - } - } -}; - -// Safari changes deviceId between sessions, therefore we cannot rely on deviceId for identification purposes. -// We can switch a random device that comes from safari to one that has the same label as the one used in the previous session. -export const correctDevicesOnSafari = async ( - stream: MediaStream, - errors: AudioVideo, - devices: MediaDeviceInfo[], - constraints: MediaConstraints, - previousDevices: PreviousDevices, -): Promise => { - const shouldCorrectDevices = isAnyDeviceDifferentFromLastSession( - previousDevices.video, - previousDevices.audio, - getCurrentDevicesSettings(stream, devices), - ); - - if (!shouldCorrectDevices) return { stream, errors }; - - const videoIdToStart = devices.find((info) => info.label === previousDevices.video?.label)?.deviceId; - const audioIdToStart = devices.find((info) => info.label === previousDevices.audio?.label)?.deviceId; - - if (!videoIdToStart && !audioIdToStart) return { stream, errors }; - - stopTracks(stream); - - const exactConstraints: MediaStreamConstraints = { - video: !errors.video && prepareConstraints(videoIdToStart, constraints.video), - audio: !errors.audio && prepareConstraints(audioIdToStart, constraints.audio), - }; - - return await getAvailableMedia(exactConstraints, errors); -}; - -const getCurrentDevicesSettings = ( - requestedDevices: MediaStream, - mediaDeviceInfos: MediaDeviceInfo[], -): CurrentDevices => { - const currentDevices: CurrentDevices = { videoinput: null, audioinput: null }; - - for (const track of requestedDevices.getTracks()) { - const settings = track.getSettings(); - if (settings.deviceId) { - const currentDevice = mediaDeviceInfos.find((device) => device.deviceId == settings.deviceId); - const kind = currentDevice?.kind ?? null; - if ((currentDevice && kind === "videoinput") || kind === "audioinput") { - currentDevices[kind] = currentDevice ?? null; - } - } - } - return currentDevices; -}; - -const isDeviceDifferentFromLastSession = (lastDevice: MediaDeviceInfo | null, currentDevice: MediaDeviceInfo | null) => - lastDevice && (currentDevice?.deviceId !== lastDevice.deviceId || currentDevice?.label !== lastDevice?.label); - -const isAnyDeviceDifferentFromLastSession = ( - lastVideoDevice: MediaDeviceInfo | null, - lastAudioDevice: MediaDeviceInfo | null, - currentDevices: CurrentDevices | null, -): boolean => - !!( - (currentDevices?.videoinput && - isDeviceDifferentFromLastSession(lastVideoDevice, currentDevices?.videoinput || null)) || - (currentDevices?.audioinput && - isDeviceDifferentFromLastSession(lastAudioDevice, currentDevices?.audioinput || null)) - ); - -const stopTracks = (requestedDevices: MediaStream) => { - for (const track of requestedDevices.getTracks()) { - track.stop(); - } -}; - -const unspecifyDevice = ( - trackConstraints?: boolean | MediaTrackConstraints, -): boolean | MediaTrackConstraints | undefined => { - if (typeof trackConstraints === "object") { - return { ...trackConstraints, deviceId: undefined }; - } - return trackConstraints; -}; diff --git a/packages/react-client/src/hooks/internal/devices/useDeviceManager.ts b/packages/react-client/src/hooks/internal/devices/useDeviceManager.ts deleted file mode 100644 index f3f342b33..000000000 --- a/packages/react-client/src/hooks/internal/devices/useDeviceManager.ts +++ /dev/null @@ -1,206 +0,0 @@ -import type { Logger } from "@fishjam-cloud/ts-client"; -import type { SetStateAction } from "react"; -import { useCallback, useMemo, useRef, useState } from "react"; - -import type { DeviceError, DeviceItem, TrackMiddleware } from "../../../types/public"; -import { parseUserMediaError } from "../../../utils/errors"; -import { getTrackFromStream, stopStream } from "../../../utils/track"; -import { useTrackMiddleware } from "../useTrackMiddleware"; -import { useHandleTrackEnd } from "./useHandleTrackEnd"; - -type DeviceManagerProps = { - mediaStream: MediaStream | null; - setMediaStream: (action: SetStateAction) => void; - deviceError: DeviceError | null; - setDeviceError: (action: SetStateAction) => void; - getInitialStream: () => Promise; - deviceType: "audio" | "video"; - allDevicesList: MediaDeviceInfo[]; - constraints?: MediaTrackConstraints | boolean; - setSelectedDevice: (device: MediaDeviceInfo) => void; - selectedDevice: MediaDeviceInfo | null; - logger: Logger; -}; - -export type DeviceManager = { - startDevice: (deviceId?: string | null) => Promise<[MediaStreamTrack, null] | [null, DeviceError]>; - stopDevice: () => void; - selectDevice: (deviceId: string) => Promise<[MediaStreamTrack, null] | [null, DeviceError]> | undefined; - activeDevice: DeviceItem | null; - deviceTrack: MediaStreamTrack | null; - deviceList: DeviceItem[]; - deviceEnabled: boolean; - enableDevice: () => void; - disableDevice: () => void; - currentMiddleware: TrackMiddleware; - applyMiddleware: (middleware: TrackMiddleware) => Promise; - deviceError: DeviceError | null; - selectedDevice: MediaDeviceInfo | null; -}; - -async function getDeviceStream( - type: "audio" | "video", - constraints: MediaTrackConstraints | boolean | undefined, - deviceId: string | null, -) { - constraints = typeof constraints === "object" ? constraints : {}; - if (deviceId) { - constraints.deviceId = { exact: deviceId }; - } - const stream = await navigator.mediaDevices.getUserMedia({ - [type]: constraints, - }); - return stream; -} - -export const useDeviceManager = ({ - mediaStream, - setMediaStream, - getInitialStream, - deviceType, - constraints, - allDevicesList, - setSelectedDevice, - deviceError, - setDeviceError, - selectedDevice, - logger, -}: DeviceManagerProps): DeviceManager => { - const mediaStreamRef = useRef(mediaStream); - mediaStreamRef.current = mediaStream; - - const rawTrack = useMemo(() => mediaStream && getTrackFromStream(mediaStream, deviceType), [mediaStream, deviceType]); - - const clearStream = useCallback(() => { - setMediaStream(null); - }, [setMediaStream]); - - useHandleTrackEnd(rawTrack, clearStream); - - const { processedTrack, applyMiddleware, currentMiddleware } = useTrackMiddleware(rawTrack); - - const currentTrack = processedTrack ?? rawTrack; - - const deviceList = useMemo( - () => allDevicesList.filter(({ kind }) => kind === `${deviceType}input`), - [allDevicesList, deviceType], - ); - - const activeDevice = useMemo(() => { - const currentDevice = - mediaStream && - deviceList.find( - (device) => device.deviceId === getTrackFromStream(mediaStream, deviceType)?.getSettings().deviceId, - ); - if (!currentDevice) return null; - return { label: currentDevice.label, deviceId: currentDevice.deviceId }; - }, [mediaStream, deviceList, deviceType]); - - const [deviceEnabled, setDeviceEnabled] = useState(true); - - const setSelectedDeviceId = useCallback( - (deviceId: string) => { - const device = deviceList.find((d) => d.deviceId === deviceId); - if (!device) return; - setSelectedDevice(device); - }, - [deviceList, setSelectedDevice], - ); - - const startDevice: DeviceManager["startDevice"] = useCallback( - async (deviceId = selectedDevice?.deviceId) => { - const initialStream = await getInitialStream(); - - const track = initialStream && getTrackFromStream(initialStream, deviceType); - const isUsingDesiredDevice = !deviceId || deviceId === track?.getSettings().deviceId; - - if (track?.enabled && isUsingDesiredDevice) { - return [track, null]; - } - - try { - const stream = await getDeviceStream(deviceType, constraints, deviceId ?? null); - - if (mediaStreamRef.current) { - stopStream(mediaStreamRef.current, deviceType); - } - setMediaStream(stream); - - const retrievedTrack = stream && getTrackFromStream(stream, deviceType); - - const retrievedTrackDeviceId = retrievedTrack.getSettings().deviceId; - - if (retrievedTrackDeviceId) { - setSelectedDeviceId(retrievedTrackDeviceId); - } - - if (retrievedTrack && !deviceEnabled) { - retrievedTrack.enabled = false; - } - - return [retrievedTrack, null]; - } catch (err) { - const parsedError = parseUserMediaError(err, logger); - setDeviceError(parsedError); - return [null, parsedError]; - } - }, - [ - selectedDevice?.deviceId, - getInitialStream, - deviceType, - constraints, - setMediaStream, - deviceEnabled, - setSelectedDeviceId, - logger, - setDeviceError, - ], - ); - - const selectDevice = useCallback( - (deviceId: string) => { - if (currentTrack) { - return startDevice(deviceId); - } else { - setSelectedDeviceId(deviceId); - } - }, - [currentTrack, setSelectedDeviceId, startDevice], - ); - - const stopDevice = useCallback(() => { - if (mediaStreamRef.current) { - stopStream(mediaStreamRef.current, deviceType); - } - setMediaStream(null); - }, [setMediaStream, deviceType]); - - const enableDevice = useCallback(() => { - if (!currentTrack) return; - currentTrack.enabled = true; - setDeviceEnabled(true); - }, [currentTrack]); - - const disableDevice = useCallback(() => { - if (!currentTrack) return; - currentTrack.enabled = false; - setDeviceEnabled(false); - }, [currentTrack]); - - return { - startDevice, - stopDevice, - selectDevice, - activeDevice, - deviceTrack: processedTrack ?? rawTrack, - deviceList, - enableDevice, - disableDevice, - deviceEnabled, - currentMiddleware, - applyMiddleware, - deviceError, - selectedDevice, - }; -}; diff --git a/packages/react-client/src/hooks/internal/devices/useMediaDevices.ts b/packages/react-client/src/hooks/internal/devices/useMediaDevices.ts deleted file mode 100644 index ccc71b56f..000000000 --- a/packages/react-client/src/hooks/internal/devices/useMediaDevices.ts +++ /dev/null @@ -1,174 +0,0 @@ -import type { Logger } from "@fishjam-cloud/ts-client"; -import { useCallback, useEffect, useRef, useState } from "react"; - -import { prepareConstraints } from "../../../devices/constraints"; -import { correctDevicesOnSafari, getAvailableMedia } from "../../../devices/mediaInitializer"; -import type { DeviceError, InitializeDevicesResult, PersistLastDeviceHandlers } from "../../../types/public"; -import { useDeviceManager } from "./useDeviceManager"; - -interface UseDevicesProps { - videoConstraints?: MediaTrackConstraints | boolean; - audioConstraints?: MediaTrackConstraints | boolean; - persistHandlers?: PersistLastDeviceHandlers; - logger: Logger; -} - -export type InitializeDevicesSettings = { enableVideo?: boolean; enableAudio?: boolean }; - -export const useMediaDevices = ({ videoConstraints, audioConstraints, persistHandlers, logger }: UseDevicesProps) => { - const [deviceList, setDeviceList] = useState([]); - - const [videoStream, setVideoStream] = useState(null); - const [audioStream, setAudioStream] = useState(null); - - const [videoError, setVideoError] = useState(null); - const [audioError, setAudioError] = useState(null); - - const [selectedCamera, setSelectedCamera] = useState( - persistHandlers?.getLastDevice("video") ?? null, - ); - const [selectedMic, setSelectedMic] = useState( - persistHandlers?.getLastDevice("audio") ?? null, - ); - - const isInitializedRef = useRef(false); - const initializationRef = useRef | null>(null); - - const selectCamera = useCallback( - (deviceInfo: MediaDeviceInfo) => { - setSelectedCamera(deviceInfo); - persistHandlers?.saveLastDevice(deviceInfo, "video"); - }, - [persistHandlers], - ); - - const selectMic = useCallback( - (deviceInfo: MediaDeviceInfo) => { - setSelectedMic(deviceInfo); - persistHandlers?.saveLastDevice(deviceInfo, "audio"); - }, - [persistHandlers], - ); - - const initializeDevices = useCallback( - async (settings?: InitializeDevicesSettings): Promise => { - if (isInitializedRef.current) { - return { stream: null, errors: null, status: "already_initialized" }; - } - - if (initializationRef.current) { - return initializationRef.current; - } - - const lastUsed = { - audio: persistHandlers?.getLastDevice("audio") ?? null, - video: persistHandlers?.getLastDevice("video") ?? null, - }; - - const constraints = { - video: settings?.enableVideo !== false && prepareConstraints(lastUsed.video?.deviceId, videoConstraints), - audio: settings?.enableAudio !== false && prepareConstraints(lastUsed.audio?.deviceId, audioConstraints), - }; - - const intitialize = async (): Promise => { - let media = await getAvailableMedia(constraints); - const fetchedDevices = await navigator.mediaDevices.enumerateDevices(); - setDeviceList(fetchedDevices); - - if (media.stream) { - media = await correctDevicesOnSafari(media.stream, media.errors, fetchedDevices, constraints, lastUsed); - } - - const { stream, errors } = media; - - const videoDeviceId = stream?.getVideoTracks()[0]?.getSettings().deviceId; - const audioDeviceId = stream?.getAudioTracks()[0]?.getSettings().deviceId; - - const videoDevice = fetchedDevices.find((device) => device.deviceId === videoDeviceId); - const audioDevice = fetchedDevices.find((device) => device.deviceId === audioDeviceId); - - if (videoDevice) { - selectCamera(videoDevice); - } - if (audioDevice) { - selectMic(audioDevice); - } - - setVideoStream(stream); - setAudioStream(stream); - setVideoError(errors.video); - setAudioError(errors.audio); - - if (!stream) { - return { status: "failed", errors, stream: null }; - } else if (errors.video || errors.audio) { - return { status: "initialized_with_errors", errors, stream: null }; - } else { - return { status: "initialized", errors: null, stream }; - } - }; - - const initializePromise = intitialize().then( - (result) => { - isInitializedRef.current = true; - return result; - }, - (error) => { - initializationRef.current = null; - throw error; - }, - ); - initializationRef.current = initializePromise; - - return await initializePromise; - }, - [videoConstraints, audioConstraints, selectCamera, selectMic, persistHandlers], - ); - - useEffect(() => { - const isInitialStreamIrrelevant = videoStream !== audioStream; - - if (isInitialStreamIrrelevant) { - initializationRef.current = null; - } - }, [videoStream, audioStream]); - - const getInitialStream = useCallback(async () => { - const result = await initializationRef.current; - return result?.stream ?? null; - }, []); - - const cameraManager = useDeviceManager({ - mediaStream: videoStream, - setMediaStream: setVideoStream, - deviceError: videoError, - setDeviceError: setVideoError, - getInitialStream, - deviceType: "video", - allDevicesList: deviceList, - constraints: videoConstraints, - setSelectedDevice: selectCamera, - selectedDevice: selectedCamera, - logger, - }); - - const microphoneManager = useDeviceManager({ - mediaStream: audioStream, - setMediaStream: setAudioStream, - deviceError: audioError, - setDeviceError: setAudioError, - getInitialStream, - deviceType: "audio", - allDevicesList: deviceList, - constraints: audioConstraints, - setSelectedDevice: selectMic, - selectedDevice: selectedMic, - logger, - }); - - return { - initializeDevices, - cameraManager, - microphoneManager, - }; -}; diff --git a/packages/react-client/src/hooks/internal/useTrackManager.ts b/packages/react-client/src/hooks/internal/useTrackManager.ts deleted file mode 100644 index d06c2f2a6..000000000 --- a/packages/react-client/src/hooks/internal/useTrackManager.ts +++ /dev/null @@ -1,200 +0,0 @@ -import { type Logger, type TrackMetadata, TrackTypeError, Variant } from "@fishjam-cloud/ts-client"; -import type { FishjamClient } from "@fishjam-cloud/tsunami"; -import { useEffect, useRef } from "react"; - -import type { TrackManager } from "../../types/internal"; -import type { BandwidthLimits, PeerStatus, StreamConfig, TrackMiddleware } from "../../types/public"; -import { getConfigAndBandwidthFromProps, getRemoteOrLocalTrack } from "../../utils/track"; -import type { DeviceManager } from "./devices/useDeviceManager"; -import { useCurrentCallback } from "./useCurrentCallback"; - -interface TrackManagerConfig { - deviceManager: DeviceManager; - tsClient: FishjamClient; - peerStatus: PeerStatus; - bandwidthLimits: BandwidthLimits; - streamConfig?: StreamConfig; - type: "camera" | "microphone"; - logger: Logger; -} - -export const useTrackManager = ({ - deviceManager, - tsClient, - peerStatus, - bandwidthLimits, - streamConfig, - type, - logger, -}: TrackManagerConfig): TrackManager => { - const currentTrackIdRef = useRef(null); - const connectionPromiseRef = useRef | null>(null); - - const { - startDevice, - stopDevice, - enableDevice, - disableDevice, - deviceTrack, - applyMiddleware, - currentMiddleware, - selectDevice: _selectDevice, - } = deviceManager; - - // Read live deviceTrack from the `joined` listener without re-subscribing - // every time it changes. - const getDeviceTrack = useCurrentCallback(() => deviceTrack); - - const getCurrentTrackId = async (): Promise => { - if (connectionPromiseRef.current) { - await connectionPromiseRef.current; - } - const refTrackId = currentTrackIdRef.current; - if (!refTrackId) return null; - const currentTrack = getRemoteOrLocalTrack(tsClient, refTrackId); - return currentTrack?.trackId ?? null; - }; - - const selectDevice = useCurrentCallback(async (deviceId: string) => { - const result = await _selectDevice(deviceId); - if (!result) return; - - const [newTrack, error] = result; - if (error) return error; - - const currentTrackId = await getCurrentTrackId(); - if (!currentTrackId) return; - - await tsClient.replaceTrack(currentTrackId, newTrack); - }); - - const setTrackMiddleware = useCurrentCallback(async (middleware: TrackMiddleware) => { - const processedTrack = await applyMiddleware(middleware); - - const currentTrackId = await getCurrentTrackId(); - if (!currentTrackId) return; - - await tsClient.replaceTrack(currentTrackId, processedTrack); - }); - - const startStreaming = useCurrentCallback( - async ( - track: MediaStreamTrack, - props: StreamConfig = { sentQualities: [Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH] }, - ) => { - // temporarily setting the local trackId until we have the remoteTrackId - currentTrackIdRef.current = track.id; - - const trackMetadata: TrackMetadata = { type, paused: false }; - - const displayName = tsClient.getLocalPeer()?.metadata?.peer?.displayName; - if (typeof displayName === "string") { - trackMetadata.displayName = displayName; - } - - const [maxBandwidth, simulcastConfig] = getConfigAndBandwidthFromProps(props.sentQualities, bandwidthLimits); - - try { - const addTrackJob = tsClient.addTrack(track, trackMetadata, simulcastConfig, maxBandwidth); - connectionPromiseRef.current = addTrackJob; - const remoteTrackId = await addTrackJob; - currentTrackIdRef.current = remoteTrackId; - } catch (err) { - if (err instanceof TrackTypeError) { - logger.warn(err.message); - currentTrackIdRef.current = null; - } - throw err; - } - }, - ); - - const pauseStreaming = useCurrentCallback(async (trackId: string) => { - if (peerStatus !== "connected") return; - await tsClient.replaceTrack(trackId, null); - return tsClient.updateTrackMetadata(trackId, { type, paused: true } satisfies TrackMetadata); - }); - - const resumeStreaming = useCurrentCallback(async (trackId: string, track: MediaStreamTrack) => { - if (peerStatus !== "connected") return; - await tsClient.replaceTrack(trackId, track); - return tsClient.updateTrackMetadata(trackId, { type, paused: false } satisfies TrackMetadata); - }); - - /** - * @see {@link TrackManager#toggleMute} for more details. - */ - const toggleMute = useCurrentCallback(async () => { - const currentTrackId = await getCurrentTrackId(); - const isTrackCurrentlyEnabled = Boolean(deviceTrack?.enabled); - if (!currentTrackId) { - logger.warn("Toggling mute is only possible while connected to a room."); - return; - } - - if (isTrackCurrentlyEnabled) { - disableDevice(); - await pauseStreaming(currentTrackId); - } else if (deviceTrack) { - enableDevice(); - await resumeStreaming(currentTrackId, deviceTrack); - } - }); - - /** - * @see {@link TrackManager#toggleDevice} for more details. - */ - const toggleDevice = useCurrentCallback(async () => { - const currentTrackId = await getCurrentTrackId(); - if (deviceTrack) { - stopDevice(); - if (currentTrackId) { - await pauseStreaming(currentTrackId); - } - } else { - const [newTrack, error] = await startDevice(); - if (error) return error; - - if (currentTrackId) { - await resumeStreaming(currentTrackId, newTrack); - } else if (peerStatus === "connected") { - await startStreaming(newTrack, streamConfig); - } - } - }); - - useEffect(() => { - const onJoinedRoom = () => { - const currentDeviceTrack = getDeviceTrack(); - if (!currentDeviceTrack) return; - // The handler is sync; observe rejections so non-TrackTypeError failures - // from addTrack don't surface as unhandledrejection. - void startStreaming(currentDeviceTrack, streamConfig).catch((err) => { - if (err instanceof TrackTypeError) return; - logger.error(err); - }); - }; - - const onLeftRoom = () => { - currentTrackIdRef.current = null; - }; - - tsClient.on("joined", onJoinedRoom); - tsClient.on("disconnected", onLeftRoom); - return () => { - tsClient.off("joined", onJoinedRoom); - tsClient.off("disconnected", onLeftRoom); - }; - }, [startStreaming, tsClient, streamConfig, getDeviceTrack, logger]); - - return { - deviceTrack, - currentMiddleware, - setTrackMiddleware, - selectDevice, - toggleMute, - toggleDevice, - stopDevice, - startDevice, - }; -}; diff --git a/packages/react-client/src/hooks/internal/useTrackMiddleware.ts b/packages/react-client/src/hooks/internal/useTrackMiddleware.ts deleted file mode 100644 index 34e591791..000000000 --- a/packages/react-client/src/hooks/internal/useTrackMiddleware.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { useCallback, useEffect, useRef, useState } from "react"; - -import type { TrackMiddleware } from "../../types/public"; - -export const useTrackMiddleware = (rawTrack: MediaStreamTrack | null) => { - const [currentMiddleware, setMiddleware] = useState(null); - const [processedTrack, setProcessedTrack] = useState(null); - const cleanupRef = useRef<(() => void) | undefined>(undefined); - - useEffect(() => { - if (!rawTrack && processedTrack) { - processedTrack.stop(); - cleanupRef.current?.(); - setProcessedTrack(null); - } - }, [rawTrack, processedTrack]); - - const applyMiddleware = useCallback( - async (newMiddleware: TrackMiddleware) => { - cleanupRef.current?.(); - setMiddleware(() => newMiddleware); - - if (newMiddleware && rawTrack) { - const { track, onClear } = await newMiddleware(rawTrack); - cleanupRef.current = onClear; - setProcessedTrack(track); - return track; - } - - setProcessedTrack(null); - return rawTrack; - }, - [rawTrack], - ); - - return { processedTrack, applyMiddleware, currentMiddleware }; -}; diff --git a/packages/react-client/src/index.ts b/packages/react-client/src/index.ts index 5f404ec44..c30077099 100644 --- a/packages/react-client/src/index.ts +++ b/packages/react-client/src/index.ts @@ -7,7 +7,6 @@ export { FishjamProvider, type FishjamProviderProps } from "./FishjamProvider"; export { useCamera } from "./hooks/devices/useCamera"; export { useInitializeDevices, UseInitializeDevicesParams } from "./hooks/devices/useInitializeDevices"; export { useMicrophone } from "./hooks/devices/useMicrophone"; -export { InitializeDevicesSettings } from "./hooks/internal/devices/useMediaDevices"; export { type JoinRoomConfig, useConnection } from "./hooks/useConnection"; export { useCustomSource } from "./hooks/useCustomSource"; export { useDataChannel } from "./hooks/useDataChannel"; @@ -49,6 +48,7 @@ export type { TracksMiddlewareResult, UseDataChannelResult, } from "./types/public"; +export { InitializeDevicesSettings } from "./types/public"; export type { AuthErrorReason, DataCallback, diff --git a/packages/react-client/src/types/internal.ts b/packages/react-client/src/types/internal.ts index f7f3abdbd..b39c54ecb 100644 --- a/packages/react-client/src/types/internal.ts +++ b/packages/react-client/src/types/internal.ts @@ -1,6 +1,6 @@ import type { Peer } from "@fishjam-cloud/ts-client"; -import type { DeviceError, PeerId, TrackMiddleware, TracksMiddleware } from "./public"; +import type { DeviceError, DeviceItem, PeerId, TrackMiddleware, TracksMiddleware } from "./public"; export type AudioVideo = { audio: T; video: T }; @@ -51,3 +51,19 @@ export type CustomSourceState = { stream: MediaStream; trackIds?: CustomSourceTracks; }; + +export type DeviceManager = { + startDevice: (deviceId?: string | null) => Promise<[MediaStreamTrack, null] | [null, DeviceError]>; + stopDevice: () => void; + selectDevice: (deviceId: string) => Promise<[MediaStreamTrack, null] | [null, DeviceError]> | undefined; + activeDevice: DeviceItem | null; + deviceTrack: MediaStreamTrack | null; + deviceList: DeviceItem[]; + deviceEnabled: boolean; + enableDevice: () => void; + disableDevice: () => void; + currentMiddleware: TrackMiddleware; + applyMiddleware: (middleware: TrackMiddleware) => Promise; + deviceError: DeviceError | null; + selectedDevice: MediaDeviceInfo | null; +}; diff --git a/packages/react-client/src/types/public.ts b/packages/react-client/src/types/public.ts index 576558475..b7fc599a5 100644 --- a/packages/react-client/src/types/public.ts +++ b/packages/react-client/src/types/public.ts @@ -124,3 +124,5 @@ export type UseDataChannelResult = { */ dataChannelError: Error | null; }; + +export type InitializeDevicesSettings = { enableVideo?: boolean; enableAudio?: boolean }; diff --git a/packages/react-client/src/utils/bandwidth.ts b/packages/react-client/src/utils/bandwidth.ts deleted file mode 100644 index 6877946e0..000000000 --- a/packages/react-client/src/utils/bandwidth.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Variant } from "@fishjam-cloud/ts-client"; - -import type { BandwidthLimits } from "../types/public"; - -export const ALL_VARIANTS_SIMULCAST = [Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH] as const; - -export const mergeWithDefaultBandwitdthLimits = (limits?: Partial): BandwidthLimits => ({ - singleStream: limits?.singleStream ?? 0, - simulcast: limits?.simulcast ?? { [Variant.VARIANT_LOW]: 0, [Variant.VARIANT_MEDIUM]: 0, [Variant.VARIANT_HIGH]: 0 }, -}); diff --git a/packages/react-client/src/utils/errors.ts b/packages/react-client/src/utils/errors.ts index 800415f83..e96ce48af 100644 --- a/packages/react-client/src/utils/errors.ts +++ b/packages/react-client/src/utils/errors.ts @@ -1,29 +1,3 @@ -import type { getLogger } from "@fishjam-cloud/ts-client"; - -import type { DeviceError } from "../types/public"; - -export const PERMISSION_DENIED: DeviceError = { name: "NotAllowedError" }; -export const OVERCONSTRAINED_ERROR: DeviceError = { name: "OverconstrainedError" }; -export const NOT_FOUND_ERROR: DeviceError = { name: "NotFoundError" }; -export const UNHANDLED_ERROR: DeviceError = { name: "UNHANDLED_ERROR" }; - -// https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getUserMedia#exceptions -// OverconstrainedError has higher priority than NotAllowedError -export const parseUserMediaError = (error: unknown, logger: ReturnType): DeviceError => { - const name = error instanceof Error ? error.name : ""; - switch (name) { - case "NotAllowedError": - return PERMISSION_DENIED; - case "OverconstrainedError": - return OVERCONSTRAINED_ERROR; - case "NotFoundError": - return NOT_FOUND_ERROR; - default: - logger.warn({ name: "Unhandled getUserMedia error", error }); - return UNHANDLED_ERROR; - } -}; - export class MissingSandboxApiUrlError extends Error { constructor() { super("useSandbox requires a sandboxApiUrl, you can get it at: https://fishjam.io/app/sandbox"); diff --git a/packages/react-client/src/utils/track.ts b/packages/react-client/src/utils/track.ts deleted file mode 100644 index a3f9db1e9..000000000 --- a/packages/react-client/src/utils/track.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { SimulcastConfig, TrackContext, TrackMetadata } from "@fishjam-cloud/ts-client"; -import { Variant } from "@fishjam-cloud/ts-client"; -import type { FishjamClient } from "@fishjam-cloud/tsunami"; - -import type { BandwidthLimits, Track, TrackId } from "../types/public"; - -// In most cases, the track is identified by its remote track ID. -// This ID comes from the ts-client `addTrack` method. -// However, we don't have that ID before the `addTrack` method returns it. -// -// The `addTrack` method emits the `localTrackAdded` event. -// This event will refresh the internal state of this object. -// However, in that event handler, we don't yet have the remote track ID. -// Therefore, for that brief moment, we will use the local track ID from the MediaStreamTrack object to identify the track. -const getRemoteOrLocalTrackContext = ( - tsClient: FishjamClient, - remoteOrLocalTrackId: string, -): TrackContext | null => { - const tracks = tsClient?.getLocalPeer()?.tracks; - if (!tracks) return null; - - const trackByRemoteId = tracks?.get(remoteOrLocalTrackId); - if (trackByRemoteId) return trackByRemoteId; - - const trackByLocalId = [...tracks.values()].find(({ track }) => track?.id === remoteOrLocalTrackId); - return trackByLocalId ?? null; -}; - -const getTrackFromContext = (context: TrackContext): Track => ({ - metadata: context.metadata as TrackMetadata, - trackId: context.trackId as TrackId, - stream: context.stream, - simulcastConfig: context.simulcastConfig || null, - track: context.track, -}); - -export const getRemoteOrLocalTrack = (tsClient: FishjamClient, remoteOrLocalTrackId: string) => { - const context = getRemoteOrLocalTrackContext(tsClient, remoteOrLocalTrackId); - if (!context) return null; - return getTrackFromContext(context); -}; - -export function setupOnEndedCallback( - track: MediaStreamTrack, - getCurrentTrackId: () => string | undefined, - callback: () => Promise, -) { - track.addEventListener("ended", async (event: Event) => { - const trackId = (event.target as MediaStreamTrack).id; - if (trackId === getCurrentTrackId()) { - await callback(); - } - }); -} - -const getDisabledEncodings = (activeEncodings: Variant[] = []) => { - const allEncodings: Variant[] = [Variant.VARIANT_LOW, Variant.VARIANT_MEDIUM, Variant.VARIANT_HIGH]; - return allEncodings.filter((encoding) => !activeEncodings.includes(encoding)); -}; - -export const getConfigAndBandwidthFromProps = ( - encodings: Variant[] | false | undefined, - bandwidthLimits: BandwidthLimits, -) => { - if (!encodings) return [bandwidthLimits.singleStream, undefined] as const; - - const config: SimulcastConfig = { - enabled: true, - enabledVariants: encodings, - disabledVariants: getDisabledEncodings(encodings), - }; - - const variantEntries = Object.entries(bandwidthLimits.simulcast).map( - ([key, value]) => [Number(key), value] as [Variant, number], - ); - - const bandwidth = new Map(variantEntries); - return [bandwidth, config] as const; -}; - -function getCertainTypeTracks(stream: MediaStream, type: "audio" | "video") { - if (type === "audio") return stream.getAudioTracks(); - return stream.getVideoTracks(); -} - -export function getTrackFromStream(stream: MediaStream, type: "audio" | "video") { - return getCertainTypeTracks(stream, type)[0] ?? null; -} - -export function stopStream(stream: MediaStream, type: "audio" | "video") { - getCertainTypeTracks(stream, type).forEach((track) => { - track.enabled = false; - track.stop(); - }); -}