diff --git a/packages/tsunami/src/FishjamClient.ts b/packages/tsunami/src/FishjamClient.ts index c2fcf139..825b1e2f 100644 --- a/packages/tsunami/src/FishjamClient.ts +++ b/packages/tsunami/src/FishjamClient.ts @@ -8,17 +8,30 @@ import { FishjamClient as TsClient, type FishjamTrackContext, type GenericMetadata, + getLogger, type MessageEvents, type Peer, type SimulcastConfig, type TrackBandwidthLimit, type TrackMetadata, - type Variant, + Variant, } from "@fishjam-cloud/ts-client"; import { EventEmitter } from "events"; import type TypedEmitter from "typed-emitter"; import { ClientResourceScope } from "./ClientResourceScope"; +import { DeviceOrchestrator } from "./controllers/DeviceOrchestrator"; +import type { TrackPublisher } from "./controllers/TrackPublisher"; +import { VIDEO_TRACK_CONSTRAINTS } from "./devices/constraints"; +import type { IDeviceManager, PlatformMediaStream, PlatformMediaStreamTrack } from "./devices/deviceManager"; +import { DeviceManagerMissingError } from "./errors/lifecycleErrors"; +import type { + BandwidthLimits, + InitializeDevicesResult, + InitializeDevicesSettings, + StreamConfig, + TrackMiddleware, +} from "./mediaTypes"; import { type ClientState, createInitialClientState } from "./state/clientState"; import { StateStore, type StoreListener } from "./state/StateStore"; @@ -28,6 +41,19 @@ type LegacyClientInternals = { }; export type FishjamClientConfig = CreateConfig & { + /** + * Platform boundary for local media acquisition. Any implementation works + * equally: `WebDeviceManager` (what the React provider injects on the web), + * a native implementation, or a custom one. When omitted (or `null`) the + * client is signalling-only and all device-related state stays at its zero + * values. + */ + deviceManager?: IDeviceManager | null; + videoConstraints?: MediaTrackConstraints | boolean; + audioConstraints?: MediaTrackConstraints | boolean; + bandwidthLimits?: Partial; + videoStreamConfig?: StreamConfig; + audioStreamConfig?: StreamConfig; /** * Strangler-migration hook: use an externally created signalling client * instead of constructing one. Also the seam tests inject fakes through. @@ -90,9 +116,20 @@ export class FishjamClient(), ); + private readonly deviceOrchestrator: DeviceOrchestrator | null = null; + public constructor(config?: FishjamClientConfig) { super(); - const { signallingClient, ...createConfig } = config ?? {}; + const { + deviceManager, + videoConstraints, + audioConstraints, + bandwidthLimits, + videoStreamConfig, + audioStreamConfig, + signallingClient, + ...createConfig + } = config ?? {}; this.config = createConfig; this.bindSessionStateEvents(); @@ -103,6 +140,131 @@ export class FishjamClient({ + publisher: this.createTrackPublisher(), + deviceManager, + store: this.store, + logger: getLogger(config?.debug ?? false), + videoConstraints: videoConstraints ?? VIDEO_TRACK_CONSTRAINTS, + audioConstraints: audioConstraints ?? true, + videoStreamConfig, + audioStreamConfig, + bandwidthLimits: { + singleStream: bandwidthLimits?.singleStream ?? 0, + simulcast: bandwidthLimits?.simulcast ?? { + [Variant.VARIANT_LOW]: 0, + [Variant.VARIANT_MEDIUM]: 0, + [Variant.VARIANT_HIGH]: 0, + }, + }, + }); + } + } + + /** + * Device controllers backing the high-level device API. Exposed for the + * framework adapters during the strangler migration; application code + * should use the `startCamera`-style methods instead. + * + * @internal + */ + public get devices(): DeviceOrchestrator | null { + return this.deviceOrchestrator; + } + + // --- device API (available when a deviceManager was injected) --- + + public initializeDevices(settings?: InitializeDevicesSettings): Promise { + return this.requireDevices().initializeDevices(settings); + } + + public async startCamera(deviceId?: string): Promise { + await this.requireDevices().camera.start(deviceId); + } + + public async stopCamera(): Promise { + await this.requireDevices().camera.stop(); + } + + public async toggleCamera(): Promise { + await this.requireDevices().camera.toggleDevice(); + } + + public async selectCamera(deviceId: string): Promise { + await this.requireDevices().camera.selectDevice(deviceId); + } + + public async setCameraTrackMiddleware(middleware: TrackMiddleware): Promise { + await this.requireDevices().camera.setTrackMiddleware(middleware); + } + + public async startMicrophone(deviceId?: string): Promise { + await this.requireDevices().microphone.start(deviceId); + } + + public async stopMicrophone(): Promise { + await this.requireDevices().microphone.stop(); + } + + public async toggleMicrophone(): Promise { + await this.requireDevices().microphone.toggleDevice(); + } + + public async toggleMicrophoneMute(): Promise { + await this.requireDevices().microphone.toggleMute(); + } + + public async selectMicrophone(deviceId: string): Promise { + await this.requireDevices().microphone.selectDevice(deviceId); + } + + public async setMicrophoneTrackMiddleware(middleware: TrackMiddleware): Promise { + await this.requireDevices().microphone.setTrackMiddleware(middleware); + } + + private requireDevices(): DeviceOrchestrator { + this.resources.assertActive(); + if (!this.deviceOrchestrator) throw new DeviceManagerMissingError(); + return this.deviceOrchestrator; + } + + private createTrackPublisher(): TrackPublisher { + // The signalling layer is typed against DOM media types, while the device + // layer only knows the platform contract. On React Native the runtime + // objects reaching this boundary are react-native-webrtc tracks that the + // signalling stack already handles, so the widening cast is confined here. + const asSignallingTrack = (track: PlatformMediaStreamTrack | null) => track as MediaStreamTrack | null; + + return { + addTrack: (track, metadata, simulcastConfig, maxBandwidth) => + this.addTrack(asSignallingTrack(track) as MediaStreamTrack, metadata, simulcastConfig, maxBandwidth), + replaceTrack: (trackId, newTrack) => this.replaceTrack(trackId, asSignallingTrack(newTrack)), + removeTrack: (trackId) => this.removeTrack(trackId), + updateTrackMetadata: (trackId, metadata) => this.updateTrackMetadata(trackId, metadata), + getDisplayName: () => { + const peerMetadata = this.getLocalPeer()?.metadata?.peer as Record | undefined; + const displayName = peerMetadata?.displayName; + return typeof displayName === "string" ? displayName : undefined; + }, + resolveRemoteTrackId: (remoteOrLocalTrackId) => { + const tracks = this.getLocalPeer()?.tracks; + if (!tracks) return null; + if (tracks.get(remoteOrLocalTrackId)) return remoteOrLocalTrackId; + const trackByLocalId = [...tracks.values()].find(({ track }) => track?.id === remoteOrLocalTrackId); + return trackByLocalId?.trackId ?? null; + }, + isSignallingActive: () => this.status === "initialized", + onJoined: (listener) => { + this.on("joined", listener); + return () => this.off("joined", listener); + }, + onDisconnected: (listener) => { + this.on("disconnected", listener); + return () => this.off("disconnected", listener); + }, + }; } /** Synchronously readable snapshot of the client's observable state. */ @@ -276,6 +438,7 @@ export class FishjamClient = { + publisher: TrackPublisher; + deviceManager: IDeviceManager; + store: StateStore>; + logger: Logger; + videoConstraints?: MediaTrackConstraints | boolean; + audioConstraints?: MediaTrackConstraints | boolean; + bandwidthLimits: BandwidthLimits; + videoStreamConfig?: StreamConfig; + audioStreamConfig?: StreamConfig; +}; + +/** + * Wraps whichever `IDeviceManager` the client config provides — + * `WebDeviceManager`, a native implementation, or a custom one; the + * orchestrator only ever sees the interface. Owns device initialization, + * hardware enumeration, and the per-source controllers, and mirrors all of it + * into the client state store. Not created for clients configured without a + * device manager (signalling-only mode). + */ +export class DeviceOrchestrator { + public readonly camera: TrackDeviceController; + public readonly microphone: TrackDeviceController; + + private deviceList: DeviceItem[] = []; + private availableCameras: DeviceItem[] = []; + private availableMicrophones: DeviceItem[] = []; + + private isInitialized = false; + private initializationPromise: Promise | null = null; + private readonly deviceChangeCleanup: () => void; + + public constructor(private readonly deps: DeviceOrchestratorDeps) { + const commonControllerDeps = { + publisher: deps.publisher, + deviceManager: deps.deviceManager, + logger: deps.logger, + getPeerStatus: () => deps.store.getState().peerStatus, + getInitialStream: () => this.getInitialStream(), + invalidateInitialStream: () => { + this.initializationPromise = null; + }, + onStateChanged: () => this.syncStore(), + }; + + this.camera = new TrackDeviceController({ + ...commonControllerDeps, + type: "video", + constraints: deps.videoConstraints, + bandwidthLimits: deps.bandwidthLimits, + streamConfig: deps.videoStreamConfig, + getAvailableDevices: () => this.availableCameras, + onSelectedDeviceChanged: (device) => this.persistLastDevice("video", device), + }); + + this.microphone = new TrackDeviceController({ + ...commonControllerDeps, + type: "audio", + constraints: deps.audioConstraints, + bandwidthLimits: deps.bandwidthLimits, + streamConfig: deps.audioStreamConfig, + getAvailableDevices: () => this.availableMicrophones, + onSelectedDeviceChanged: (device) => this.persistLastDevice("audio", device), + }); + + this.deviceChangeCleanup = deps.deviceManager.onDeviceChange(() => { + void this.refreshDeviceList().catch((error) => deps.logger.error("Failed to refresh device list", error)); + }); + + this.syncStore(); + } + + public async initializeDevices(settings?: InitializeDevicesSettings): Promise { + if (this.isInitialized) { + return { stream: null, errors: null, status: "already_initialized" }; + } + if (this.initializationPromise) { + return this.initializationPromise; + } + + const persistence = this.deps.deviceManager.persistence; + const lastUsed = { + audio: (await persistence?.getLastDevice("audio")) ?? null, + video: (await persistence?.getLastDevice("video")) ?? null, + }; + + const constraints = { + video: + settings?.enableVideo !== false && prepareConstraints(lastUsed.video?.deviceId, this.deps.videoConstraints), + audio: + settings?.enableAudio !== false && prepareConstraints(lastUsed.audio?.deviceId, this.deps.audioConstraints), + }; + + const initialize = async (): Promise => { + let media = await getAvailableMedia(this.deps.deviceManager, constraints); + await this.refreshDeviceList(); + + if (media.stream) { + media = await recoverPersistedDevices( + this.deps.deviceManager, + media.stream, + media.errors, + this.deviceList, + constraints, + lastUsed, + ); + } + + const { stream, errors } = media; + + const videoDeviceId = stream?.getVideoTracks()[0]?.getSettings().deviceId; + const audioDeviceId = stream?.getAudioTracks()[0]?.getSettings().deviceId; + + const videoDevice = this.availableCameras.find((device) => device.deviceId === videoDeviceId); + const audioDevice = this.availableMicrophones.find((device) => device.deviceId === audioDeviceId); + + if (videoDevice) this.camera.setSelectedDevice(videoDevice); + if (audioDevice) this.microphone.setSelectedDevice(audioDevice); + + // Both controllers adopt the same stream; each only ever touches tracks + // of its own kind. + this.camera.adoptInitialStream(stream); + this.microphone.adoptInitialStream(stream); + this.camera.setError(errors.video); + this.microphone.setError(errors.audio); + + if (!stream) { + return { status: "failed", errors, stream: null }; + } else if (errors.video || errors.audio) { + return { status: "initialized_with_errors", errors, stream: null }; + } else { + return { status: "initialized", errors: null, stream }; + } + }; + + const initializationPromise = initialize().then( + (result) => { + this.isInitialized = true; + this.syncStore(); + return result; + }, + (error) => { + this.initializationPromise = null; + throw error; + }, + ); + this.initializationPromise = initializationPromise; + + return initializationPromise; + } + + public dispose(): void { + this.deviceChangeCleanup(); + this.camera.dispose(); + this.microphone.dispose(); + } + + private async getInitialStream(): Promise { + const result = await this.initializationPromise; + return result?.stream ?? null; + } + + private async refreshDeviceList(): Promise { + this.deviceList = await this.deps.deviceManager.enumerateDevices(); + this.availableCameras = this.deviceList.filter((device) => device.kind === "video"); + this.availableMicrophones = this.deviceList.filter((device) => device.kind === "audio"); + this.syncStore(); + } + + private persistLastDevice(type: DeviceType, device: DeviceItem): void { + void Promise.resolve(this.deps.deviceManager.persistence?.saveLastDevice(type, device)).catch((error) => + this.deps.logger.warn({ name: "Failed to persist last device", error }), + ); + } + + private syncStore(): void { + this.deps.store.update({ + camera: this.camera.snapshot(), + microphone: this.microphone.snapshot(), + availableCameras: this.availableCameras, + availableMicrophones: this.availableMicrophones, + cameraError: this.camera.error, + microphoneError: this.microphone.error, + devicesInitialized: this.isInitialized, + } as Partial>); + } +} diff --git a/packages/tsunami/src/errors/lifecycleErrors.ts b/packages/tsunami/src/errors/lifecycleErrors.ts index 0842fffd..eabf9b24 100644 --- a/packages/tsunami/src/errors/lifecycleErrors.ts +++ b/packages/tsunami/src/errors/lifecycleErrors.ts @@ -15,3 +15,17 @@ export class ClientDisposedError extends FishjamError { this.name = "ClientDisposedError"; } } + +/** + * Thrown when a device-related method is called on a client that was created + * without a device manager. Construct the client with a `deviceManager` to + * use the device API; a signalling-only client cannot acquire local media. + */ +export class DeviceManagerMissingError extends FishjamError { + public readonly recoverability: ErrorRecoverability = "fatal"; + + public constructor() { + super("This FishjamClient was created without a device manager, so the device API is unavailable"); + this.name = "DeviceManagerMissingError"; + } +} diff --git a/packages/tsunami/src/index.ts b/packages/tsunami/src/index.ts index c74d4380..32f6218d 100644 --- a/packages/tsunami/src/index.ts +++ b/packages/tsunami/src/index.ts @@ -3,19 +3,45 @@ * * @packageDocumentation */ -export type { DeviceItem, DeviceType, IDeviceManager, IDevicePersistence } from "./devices/deviceManager"; +export type { DeviceOrchestrator } from "./controllers/DeviceOrchestrator"; +export type { TrackDeviceController } from "./controllers/TrackDeviceController"; +export type { + DeviceItem, + DeviceType, + IDeviceManager, + IDevicePersistence, + PlatformMediaStream, + PlatformMediaStreamTrack, +} from "./devices/deviceManager"; +export { + classifyDeviceError, + DeviceError, + type DeviceErrorName, + DeviceNotFoundError, + DeviceOverconstrainedError, + DevicePermissionDeniedError, + UnknownDeviceError, +} from "./devices/errors"; export { LocalStorageDevicePersistence } from "./devices/LocalStorageDevicePersistence"; export { WebDeviceManager, type WebDeviceManagerOptions } from "./devices/WebDeviceManager"; export { type ErrorRecoverability, FishjamError } from "./errors/FishjamError"; -export { ClientDisposedError } from "./errors/lifecycleErrors"; +export { ClientDisposedError, DeviceManagerMissingError } from "./errors/lifecycleErrors"; export { FishjamClient, type FishjamClientConfig } from "./FishjamClient"; -export { type ClientState, createInitialClientState, type PeerStatus } from "./state/clientState"; +export type { + BandwidthLimits, + InitializeDevicesResult, + InitializeDevicesSettings, + InitializeDevicesStatus, + MiddlewareResult, + SimulcastBandwidthLimits, + StreamConfig, + TrackMiddleware, +} from "./mediaTypes"; +export { + type ClientState, + createInitialClientState, + type LocalDeviceState, + type PeerStatus, +} from "./state/clientState"; export { StateStore, type StateStoreOptions, type StoreListener } from "./state/StateStore"; export * from "@fishjam-cloud/ts-client"; - -export type MiddlewareResult = { - track: MediaStreamTrack; - onClear?: () => void; -}; - -export type TrackMiddleware = ((track: MediaStreamTrack) => MiddlewareResult | Promise) | null; diff --git a/packages/tsunami/src/state/clientState.ts b/packages/tsunami/src/state/clientState.ts index 0f04387b..7cd4bffa 100644 --- a/packages/tsunami/src/state/clientState.ts +++ b/packages/tsunami/src/state/clientState.ts @@ -1,6 +1,7 @@ import type { Component, GenericMetadata, Peer, ReconnectionStatus } from "@fishjam-cloud/ts-client"; import type { DeviceItem, PlatformMediaStream, PlatformMediaStreamTrack } from "../devices/deviceManager"; +import type { DeviceError } from "../devices/errors"; import type { TrackMiddleware } from "../mediaTypes"; /** @@ -41,8 +42,28 @@ export interface ClientState | null; remotePeers: Record>; components: Record; + + // local devices + camera: LocalDeviceState; + microphone: LocalDeviceState; + + // available hardware + availableCameras: DeviceItem[]; + availableMicrophones: DeviceItem[]; + cameraError: DeviceError | null; + microphoneError: DeviceError | null; + devicesInitialized: boolean; } +const createInitialDeviceState = (): LocalDeviceState => ({ + track: null, + stream: null, + isEnabled: true, + activeDevice: null, + selectedDevice: null, + middleware: null, +}); + export const createInitialClientState = (): ClientState< PeerMetadata, ServerMetadata @@ -52,4 +73,11 @@ export const createInitialClientState = (): Client localPeer: null, remotePeers: {}, components: {}, + camera: createInitialDeviceState(), + microphone: createInitialDeviceState(), + availableCameras: [], + availableMicrophones: [], + cameraError: null, + microphoneError: null, + devicesInitialized: false, });