From def3c9802eee45c86d2e73548f2cea8b1d2e2463 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 4 Jun 2026 12:58:00 -0400 Subject: [PATCH 01/68] refactor: rename constant --- src/room/data-stream/outgoing/OutgoingDataStreamManager.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 9443c7c673..f258783ea3 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -24,7 +24,7 @@ import type { import { numberToBigInt, splitUtf8 } from '../../utils'; import { ByteStreamWriter, TextStreamWriter } from './StreamWriter'; -const STREAM_CHUNK_SIZE = 15_000; +const STREAM_CHUNK_SIZE_BYTES = 15_000; /** * Manages sending custom user data via data channels. @@ -145,7 +145,7 @@ export default class OutgoingDataStreamManager { const writableStream = new WritableStream({ // Implement the sink async write(text) { - for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE)) { + for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { const chunk = new DataStream_Chunk({ content: textByteChunk, streamId, @@ -276,7 +276,7 @@ export default class OutgoingDataStreamManager { let byteOffset = 0; try { while (byteOffset < chunk.byteLength) { - const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE); + const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); const chunkPacket = new DataPacket({ destinationIdentities, value: { From 9f467c41123f6290591e07984f88259d05968dfe Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 4 Jun 2026 12:58:09 -0400 Subject: [PATCH 02/68] fix: make rpc use datastream.sendText, NOT streamText Using sendText means that future optimizations to send a single data stream packet can be enabled. --- src/room/rpc/client/RpcClientManager.ts | 5 +---- src/room/rpc/server/RpcServerManager.ts | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/room/rpc/client/RpcClientManager.ts b/src/room/rpc/client/RpcClientManager.ts index 019a3714b1..0b282e24f7 100644 --- a/src/room/rpc/client/RpcClientManager.ts +++ b/src/room/rpc/client/RpcClientManager.ts @@ -145,7 +145,7 @@ export default class RpcClientManager extends (EventEmitter as new () => TypedEm ) { if (remoteClientProtocol >= CLIENT_PROTOCOL_DATA_STREAM_RPC) { // Send payload as a data stream - a "version 2" rpc request. - const writer = await this.outgoingDataStreamManager.streamText({ + await this.outgoingDataStreamManager.sendText(payload, { topic: RPC_REQUEST_DATA_STREAM_TOPIC, destinationIdentities: [destinationIdentity], attributes: { @@ -155,9 +155,6 @@ export default class RpcClientManager extends (EventEmitter as new () => TypedEm [RpcRequestAttrs.RPC_REQUEST_VERSION]: `${RPC_VERSION_V2}`, }, }); - - await writer.write(payload); - await writer.close(); return; } diff --git a/src/room/rpc/server/RpcServerManager.ts b/src/room/rpc/server/RpcServerManager.ts index 33215c3d8a..a8f161d929 100644 --- a/src/room/rpc/server/RpcServerManager.ts +++ b/src/room/rpc/server/RpcServerManager.ts @@ -263,13 +263,11 @@ export default class RpcServerManager extends (EventEmitter as new () => TypedEm if (callerClientProtocol >= CLIENT_PROTOCOL_DATA_STREAM_RPC) { // Send response as a data stream - const writer = await this.outgoingDataStreamManager.streamText({ + await this.outgoingDataStreamManager.sendText(payload, { topic: RPC_RESPONSE_DATA_STREAM_TOPIC, destinationIdentities: [destinationIdentity], attributes: { [RpcRequestAttrs.RPC_REQUEST_ID]: requestId }, }); - await writer.write(payload); - await writer.close(); return; } From 632f5519c8c9253a28e8b8e9b17e2281fcd3a714 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 4 Jun 2026 14:04:36 -0400 Subject: [PATCH 03/68] fix: convert rpc tests from streamText -> sendText --- src/room/rpc/client/RpcClientManager.test.ts | 45 ++++++++------------ src/room/rpc/server/RpcServerManager.test.ts | 30 ++++--------- 2 files changed, 27 insertions(+), 48 deletions(-) diff --git a/src/room/rpc/client/RpcClientManager.test.ts b/src/room/rpc/client/RpcClientManager.test.ts index 12230397ea..8f9184c141 100644 --- a/src/room/rpc/client/RpcClientManager.test.ts +++ b/src/room/rpc/client/RpcClientManager.test.ts @@ -132,19 +132,13 @@ describe('RpcClientManager', () => { describe('v2 -> v2', () => { let rpcClientManager: RpcClientManager; - let mockStreamTextWriter: { - write: ReturnType; - close: ReturnType; - }; + let sendTextMock: ReturnType; let mockOutgoingDataStreamManager: OutgoingDataStreamManager; beforeEach(() => { - mockStreamTextWriter = { - write: vi.fn().mockResolvedValue(undefined), - close: vi.fn().mockResolvedValue(undefined), - }; + sendTextMock = vi.fn().mockResolvedValue(undefined); mockOutgoingDataStreamManager = { - streamText: vi.fn().mockResolvedValue(mockStreamTextWriter), + sendText: sendTextMock, } as unknown as OutgoingDataStreamManager; rpcClientManager = new RpcClientManager( @@ -171,7 +165,8 @@ describe('RpcClientManager', () => { }); // Verify the data stream was used with correct attributes - expect(mockOutgoingDataStreamManager.streamText).toHaveBeenCalledWith( + expect(mockOutgoingDataStreamManager.sendText).toHaveBeenCalledWith( + 'request-payload', expect.objectContaining({ topic: RPC_REQUEST_DATA_STREAM_TOPIC, destinationIdentities: ['destination-identity'], @@ -182,8 +177,6 @@ describe('RpcClientManager', () => { }), }), ); - expect(mockStreamTextWriter.write).toHaveBeenCalledWith('request-payload'); - expect(mockStreamTextWriter.close).toHaveBeenCalled(); // No packet should have been emitted expect(managerEvents.areThereBufferedEvents('sendDataPacket')).toBe(false); @@ -213,7 +206,8 @@ describe('RpcClientManager', () => { }); // Verify the data stream was used with correct attributes - expect(mockOutgoingDataStreamManager.streamText).toHaveBeenCalledWith( + expect(mockOutgoingDataStreamManager.sendText).toHaveBeenCalledWith( + longPayload, expect.objectContaining({ topic: RPC_REQUEST_DATA_STREAM_TOPIC, destinationIdentities: ['destination-identity'], @@ -224,8 +218,6 @@ describe('RpcClientManager', () => { }), }), ); - expect(mockStreamTextWriter.write).toHaveBeenCalledWith(longPayload); - expect(mockStreamTextWriter.close).toHaveBeenCalled(); // No packet should have been emitted expect(managerEvents.areThereBufferedEvents('sendDataPacket')).toBe(false); @@ -337,14 +329,14 @@ describe('RpcClientManager', () => { }); it('should not drop ack and response that arrive before publish completes', async () => { - // Hold the publish path open by blocking writer.close() until we explicitly resolve it. - let resolveClose!: () => void; - const closeBlocked = new Promise((resolve) => { - resolveClose = resolve; + // Hold the publish path open by blocking sendText() until we explicitly resolve it. + let resolveSend!: () => void; + const sendBlocked = new Promise((resolve) => { + resolveSend = resolve; }); - mockStreamTextWriter.close = vi.fn().mockReturnValue(closeBlocked); + sendTextMock.mockReturnValue(sendBlocked); - // Start performRpc but don't await its return yet. The synchronous prefix runs streamText. + // Start performRpc but don't await its return yet. The synchronous prefix runs sendText. const performRpcPromise = rpcClientManager.performRpc({ destinationIdentity: 'destination-identity', method: 'test-method', @@ -352,11 +344,10 @@ describe('RpcClientManager', () => { responseTimeout: 200, }); - // streamText was called synchronously; pull the request id out of the attributes. - const streamTextCalls = (mockOutgoingDataStreamManager.streamText as ReturnType) - .mock.calls; - expect(streamTextCalls.length).toBe(1); - const requestId = streamTextCalls[0][0].attributes[RpcRequestAttrs.RPC_REQUEST_ID]; + // sendText was called synchronously; pull the request id out of the attributes. + const sendTextCalls = sendTextMock.mock.calls; + expect(sendTextCalls.length).toBe(1); + const requestId = sendTextCalls[0][1].attributes[RpcRequestAttrs.RPC_REQUEST_ID]; // Deliver ack and response BEFORE close() unblocks - the publish has not yet returned. rpcClientManager.handleIncomingRpcAck(requestId); @@ -367,7 +358,7 @@ describe('RpcClientManager', () => { ); // Now allow the publish path to complete. - resolveClose(); + resolveSend(); const [, completionPromise] = await performRpcPromise; await expect(completionPromise).resolves.toStrictEqual('response-payload'); diff --git a/src/room/rpc/server/RpcServerManager.test.ts b/src/room/rpc/server/RpcServerManager.test.ts index a2a32a707a..8f26e495de 100644 --- a/src/room/rpc/server/RpcServerManager.test.ts +++ b/src/room/rpc/server/RpcServerManager.test.ts @@ -184,21 +184,11 @@ describe('RpcServerManager', () => { describe('v2 -> v2', () => { let rpcServerManager: RpcServerManager; let outgoingDataStreamManager: OutgoingDataStreamManager; - let mockStreamTextWriter: { - write: ReturnType; - close: ReturnType; - }; beforeEach(() => { outgoingDataStreamManager = new OutgoingDataStreamManager({} as unknown as RTCEngine, log); - mockStreamTextWriter = { - write: vi.fn().mockResolvedValue(undefined), - close: vi.fn().mockResolvedValue(undefined), - }; - vi.spyOn(outgoingDataStreamManager, 'streamText').mockResolvedValue( - mockStreamTextWriter as any, - ); + vi.spyOn(outgoingDataStreamManager, 'sendText').mockResolvedValue(undefined as any); rpcServerManager = new RpcServerManager( log, @@ -243,15 +233,14 @@ describe('RpcServerManager', () => { // The response should have been sent via data stream, not packet expect(managerEvents.areThereBufferedEvents('sendDataPacket')).toBe(false); - expect(outgoingDataStreamManager.streamText).toHaveBeenCalledWith( + expect(outgoingDataStreamManager.sendText).toHaveBeenCalledWith( + 'response payload', expect.objectContaining({ topic: RPC_RESPONSE_DATA_STREAM_TOPIC, destinationIdentities: ['caller-identity'], attributes: { [RpcRequestAttrs.RPC_REQUEST_ID]: requestId }, }), ); - expect(mockStreamTextWriter.write).toHaveBeenCalledWith('response payload'); - expect(mockStreamTextWriter.close).toHaveBeenCalled(); }); it('should receive a large rpc request (> 15kb) and send a large response via data stream from a participant', async () => { @@ -277,15 +266,14 @@ describe('RpcServerManager', () => { // The response should have been sent via data stream, not packet expect(managerEvents.areThereBufferedEvents('sendDataPacket')).toBe(false); - expect(outgoingDataStreamManager.streamText).toHaveBeenCalledWith( + expect(outgoingDataStreamManager.sendText).toHaveBeenCalledWith( + new Array(20_000).fill('B').join(''), expect.objectContaining({ topic: RPC_RESPONSE_DATA_STREAM_TOPIC, destinationIdentities: ['caller-identity'], attributes: { [RpcRequestAttrs.RPC_REQUEST_ID]: requestId }, }), ); - expect(mockStreamTextWriter.write).toHaveBeenCalledWith(new Array(20_000).fill('B').join('')); - expect(mockStreamTextWriter.close).toHaveBeenCalled(); }); it('should register an RPC method handler', async () => { @@ -317,7 +305,7 @@ describe('RpcServerManager', () => { // Response goes via data stream, not packet expect(managerEvents.areThereBufferedEvents('sendDataPacket')).toBe(false); - expect(outgoingDataStreamManager.streamText).toHaveBeenCalled(); + expect(outgoingDataStreamManager.sendText).toHaveBeenCalled(); }); it('should catch and transform unhandled errors in the RPC method handler', async () => { @@ -414,7 +402,7 @@ describe('RpcServerManager', () => { const errorResponse = errorEvent.packet.value.value.value.value; expect(errorResponse.code).toStrictEqual(RpcError.ErrorCode.UNSUPPORTED_METHOD); - expect(outgoingDataStreamManager.streamText).not.toHaveBeenCalled(); + expect(outgoingDataStreamManager.sendText).not.toHaveBeenCalled(); expect(managerEvents.areThereBufferedEvents('sendDataPacket')).toBe(false); }); }); @@ -425,7 +413,7 @@ describe('RpcServerManager', () => { {} as unknown as RTCEngine, log, ); - const streamTextSpy = vi.spyOn(outgoingDataStreamManager, 'streamText'); + const sendTextSpy = vi.spyOn(outgoingDataStreamManager, 'sendText'); const rpcServerManager = new RpcServerManager( log, @@ -457,7 +445,7 @@ describe('RpcServerManager', () => { assert(ackEvent.packet.value.case === 'rpcAck'); // Response should be a v1 RpcResponse packet, not a data stream - expect(streamTextSpy).not.toHaveBeenCalled(); + expect(sendTextSpy).not.toHaveBeenCalled(); const responseEvent = await managerEvents.waitFor('sendDataPacket'); assert(responseEvent.packet.value.case === 'rpcResponse'); const rpcResponse = responseEvent.packet.value.value; From b90d2eeb626b33f7f9204cb0db45d997cfa21a25 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 4 Jun 2026 14:04:20 -0400 Subject: [PATCH 04/68] feat: add initial support for single packet data streams via hacky mechanism --- .../incoming/IncomingDataStreamManager.ts | 54 ++++++++- .../outgoing/OutgoingDataStreamManager.ts | 113 +++++++++--------- src/room/data-stream/outgoing/header-utils.ts | 68 +++++++++++ src/room/utils.ts | 19 +++ 4 files changed, 198 insertions(+), 56 deletions(-) create mode 100644 src/room/data-stream/outgoing/header-utils.ts diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 0ac45e793e..ce24a04fd7 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -8,7 +8,8 @@ import { import log from '../../../logger'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { type ByteStreamInfo, type StreamController, type TextStreamInfo } from '../../types'; -import { bigIntToNumber } from '../../utils'; +import { bigIntToNumber, decodeBase64 } from '../../utils'; +import { INLINE_PAYLOAD_ATTRIBUTE } from '../constants'; import { type ByteStreamHandler, ByteStreamReader, @@ -156,6 +157,23 @@ export default class IncomingDataStreamManager { attributes: streamHeader.attributes, encryptionType, }; + + // Single-packet stream: the entire payload was smuggled into a reserved header attribute. + // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. + const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; + if (typeof inlinePayload !== 'undefined') { + delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; + streamHandlerCallback( + new ByteStreamReader( + info, + createInlineStream(streamHeader.streamId, decodeBase64(inlinePayload)), + bigIntToNumber(streamHeader.totalLength), + ), + { identity: participantIdentity }, + ); + return; + } + const stream = new ReadableStream({ start: (controller) => { streamController = controller; @@ -204,6 +222,22 @@ export default class IncomingDataStreamManager { attachedStreamIds: streamHeader.contentHeader.value.attachedStreamIds, }; + // Single-packet stream: the entire payload was smuggled into a reserved header attribute. + // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. + const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; + if (typeof inlinePayload !== 'undefined') { + delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; + streamHandlerCallback( + new TextStreamReader( + info, + createInlineStream(streamHeader.streamId, new TextEncoder().encode(inlinePayload)), + bigIntToNumber(streamHeader.totalLength), + ), + { identity: participantIdentity }, + ); + return; + } + const stream = new ReadableStream({ start: (controller) => { streamController = controller; @@ -295,3 +329,21 @@ export default class IncomingDataStreamManager { } } } + +/** + * Builds a `ReadableStream` that yields the given content as a single chunk and then immediately + * closes - used to surface an inline (single-packet) data stream as a fully-formed stream. + */ +function createInlineStream( + streamId: string, + content: Uint8Array, +): ReadableStream { + return new ReadableStream({ + start: (controller) => { + controller.enqueue( + new DataStream_Chunk({ streamId, chunkIndex: BigInt(0), content }), + ); + controller.close(); + }, + }); +} diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index f258783ea3..c689810c6a 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -1,11 +1,7 @@ import { Mutex } from '@livekit/mutex'; import { DataPacket, - DataStream_ByteHeader, DataStream_Chunk, - DataStream_Header, - DataStream_OperationType, - DataStream_TextHeader, DataStream_Trailer, Encryption_Type, } from '@livekit/protocol'; @@ -22,9 +18,13 @@ import type { TextStreamInfo, } from '../../types'; import { numberToBigInt, splitUtf8 } from '../../utils'; +import { INLINE_PAYLOAD_ATTRIBUTE, STREAM_CHUNK_SIZE_BYTES } from '../constants'; import { ByteStreamWriter, TextStreamWriter } from './StreamWriter'; - -const STREAM_CHUNK_SIZE_BYTES = 15_000; +import { + buildByteStreamHeader, + buildTextStreamHeader, + createStreamHeaderPacket, +} from './header-utils'; /** * Manages sending custom user data via data channels. @@ -50,6 +50,16 @@ export default class OutgoingDataStreamManager { const textInBytes = new TextEncoder().encode(text); const totalTextLength = textInBytes.byteLength; + // Fast path: when the full payload is known up front, there are no attachments, and the + // payload fits (with header overhead) under the MTU, smuggle it into a reserved header + // attribute and send a single `streamHeader` packet - no chunk/trailer packets. + if (!options?.attachments || options.attachments.length === 0) { + const inlineInfo = await this.trySendInlineText(streamId, text, totalTextLength, options); + if (inlineInfo) { + return inlineInfo; + } + } + const fileIds = options?.attachments?.map(() => crypto.randomUUID()); const progresses = new Array(fileIds ? fileIds.length + 1 : 1).fill(0); @@ -91,6 +101,44 @@ export default class OutgoingDataStreamManager { return writer.info; } + /** + * Attempts to send `text` as a single header packet with the payload smuggled into a reserved + * attribute. Returns the resulting {@link TextStreamInfo} if it fit under the MTU, or `undefined` + * if the caller should fall back to the regular chunked stream. + */ + private async trySendInlineText( + streamId: string, + text: string, + totalTextLength: number, + options?: SendTextOptions, + ): Promise { + const info: TextStreamInfo = { + id: streamId, + mimeType: 'text/plain', + timestamp: Date.now(), + topic: options?.topic ?? '', + size: totalTextLength, + attributes: options?.attributes, + encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled + ? Encryption_Type.GCM + : Encryption_Type.NONE, + }; + + const header = buildTextStreamHeader({ + ...info, + attributes: { ...info.attributes, [INLINE_PAYLOAD_ATTRIBUTE]: text }, + }); + const packet = createStreamHeaderPacket(header, options?.destinationIdentities); + + if (packet.toBinary().byteLength > STREAM_CHUNK_SIZE_BYTES) { + return null; + } + + await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); + options?.onProgress?.(1); + return info; + } + /** * @internal */ @@ -109,34 +157,9 @@ export default class OutgoingDataStreamManager { : Encryption_Type.NONE, attachedStreamIds: options?.attachedStreamIds, }; - const header = new DataStream_Header({ - streamId, - mimeType: info.mimeType, - topic: info.topic, - timestamp: numberToBigInt(info.timestamp), - totalLength: numberToBigInt(info.size), - attributes: info.attributes, - contentHeader: { - case: 'textHeader', - value: new DataStream_TextHeader({ - version: options?.version, - attachedStreamIds: info.attachedStreamIds, - replyToStreamId: options?.replyToStreamId, - operationType: - options?.type === 'update' - ? DataStream_OperationType.UPDATE - : DataStream_OperationType.CREATE, - }), - }, - }); + const header = buildTextStreamHeader(info, options); const destinationIdentities = options?.destinationIdentities; - const packet = new DataPacket({ - destinationIdentities, - value: { - case: 'streamHeader', - value: header, - }, - }); + const packet = createStreamHeaderPacket(header, destinationIdentities); await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); let chunkId = 0; @@ -239,28 +262,8 @@ export default class OutgoingDataStreamManager { : Encryption_Type.NONE, }; - const header = new DataStream_Header({ - totalLength: numberToBigInt(info.size), - mimeType: info.mimeType, - streamId, - topic: info.topic, - timestamp: numberToBigInt(Date.now()), - attributes: info.attributes, - contentHeader: { - case: 'byteHeader', - value: new DataStream_ByteHeader({ - name: info.name, - }), - }, - }); - - const packet = new DataPacket({ - destinationIdentities, - value: { - case: 'streamHeader', - value: header, - }, - }); + const header = buildByteStreamHeader(info); + const packet = createStreamHeaderPacket(header, destinationIdentities); await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); diff --git a/src/room/data-stream/outgoing/header-utils.ts b/src/room/data-stream/outgoing/header-utils.ts new file mode 100644 index 0000000000..e49a3ff108 --- /dev/null +++ b/src/room/data-stream/outgoing/header-utils.ts @@ -0,0 +1,68 @@ +import { + DataPacket, + DataStream_ByteHeader, + DataStream_Header, + DataStream_OperationType, + DataStream_TextHeader, +} from '@livekit/protocol'; +import type { ByteStreamInfo, StreamTextOptions, TextStreamInfo } from '../../types'; +import { numberToBigInt } from '../../utils'; + +/** Builds the `DataStream_Header` for a text stream from its info and stream options. */ +export function buildTextStreamHeader( + info: TextStreamInfo, + options?: Pick, +): DataStream_Header { + return new DataStream_Header({ + streamId: info.id, + mimeType: info.mimeType, + topic: info.topic, + timestamp: numberToBigInt(info.timestamp), + totalLength: numberToBigInt(info.size), + attributes: info.attributes, + contentHeader: { + case: 'textHeader', + value: new DataStream_TextHeader({ + version: options?.version, + attachedStreamIds: info.attachedStreamIds, + replyToStreamId: options?.replyToStreamId, + operationType: + options?.type === 'update' + ? DataStream_OperationType.UPDATE + : DataStream_OperationType.CREATE, + }), + }, + }); +} + +/** Builds the `DataStream_Header` for a byte stream from its info. */ +export function buildByteStreamHeader(info: ByteStreamInfo): DataStream_Header { + return new DataStream_Header({ + streamId: info.id, + mimeType: info.mimeType, + topic: info.topic, + timestamp: numberToBigInt(info.timestamp), + totalLength: numberToBigInt(info.size), + attributes: info.attributes, + contentHeader: { + case: 'byteHeader', + value: new DataStream_ByteHeader({ + name: info.name, + }), + }, + }); +} + +/** Wraps a `DataStream_Header` in a `DataPacket` ready to be sent over a data channel. */ +export function createStreamHeaderPacket( + header: DataStream_Header, + destinationIdentities?: Array, +): DataPacket { + return new DataPacket({ + destinationIdentities, + value: { + case: 'streamHeader', + value: header, + }, + }); +} diff --git a/src/room/utils.ts b/src/room/utils.ts index 52a586479b..5e3945124f 100644 --- a/src/room/utils.ts +++ b/src/room/utils.ts @@ -781,6 +781,25 @@ export function splitUtf8(s: string, n: number): NonSharedUint8Array[] { return result; } +/** Encodes a byte array as a base64 string (suitable for embedding binary data in a string field). */ +export function encodeBase64(bytes: Uint8Array): string { + let binary = ''; + for (let i = 0; i < bytes.byteLength; i++) { + binary += String.fromCharCode(bytes[i]!); + } + return btoa(binary); +} + +/** Decodes a base64 string (as produced by {@link encodeBase64}) back into a byte array. */ +export function decodeBase64(base64: string): Uint8Array { + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes; +} + export function extractMaxAgeFromRequestHeaders(headers: Headers): number | undefined { const cacheControl = headers.get('Cache-Control'); if (cacheControl) { From cb4a54ade3455e21260aa7ff88410cc9cbf2fbfc Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 4 Jun 2026 14:35:51 -0400 Subject: [PATCH 05/68] feat: add backwards compatibility back in for old multi packet v1 data streams --- src/room/Room.ts | 6 +++- .../incoming/IncomingDataStreamManager.ts | 4 +-- .../outgoing/OutgoingDataStreamManager.ts | 35 +++++++++++++++++-- src/room/rpc/client/RpcClientManager.test.ts | 1 + src/room/rpc/server/RpcServerManager.test.ts | 14 ++++++-- src/version.ts | 6 +++- 6 files changed, 56 insertions(+), 10 deletions(-) diff --git a/src/room/Room.ts b/src/room/Room.ts index b09f79bb8f..d33c16034f 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -268,7 +268,11 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.maybeCreateEngine(); this.incomingDataStreamManager = new IncomingDataStreamManager(); - this.outgoingDataStreamManager = new OutgoingDataStreamManager(this.engine, this.log); + this.outgoingDataStreamManager = new OutgoingDataStreamManager( + this.engine, + this.log, + this.getRemoteParticipantClientProtocol, + ); this.incomingDataTrackManager = new IncomingDataTrackManager({ e2eeManager: this.e2eeManager }); this.incomingDataTrackManager diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index ce24a04fd7..c0746a56ff 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -340,9 +340,7 @@ function createInlineStream( ): ReadableStream { return new ReadableStream({ start: (controller) => { - controller.enqueue( - new DataStream_Chunk({ streamId, chunkIndex: BigInt(0), content }), - ); + controller.enqueue(new DataStream_Chunk({ streamId, chunkIndex: BigInt(0), content })); controller.close(); }, }); diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index c689810c6a..270db427f2 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -6,6 +6,7 @@ import { Encryption_Type, } from '@livekit/protocol'; import { type StructuredLogger } from '../../../logger'; +import { CLIENT_PROTOCOL_DATA_STREAM_V2 } from '../../../version'; import type RTCEngine from '../../RTCEngine'; import { DataChannelKind } from '../../RTCEngine'; import { EngineEvent } from '../../events'; @@ -35,9 +36,18 @@ export default class OutgoingDataStreamManager { protected log: StructuredLogger; - constructor(engine: RTCEngine, log: StructuredLogger) { + /** Returns the advertised client protocol of a remote participant, used to decide whether a + * recipient can receive single-packet (inline) data streams. */ + protected getRemoteParticipantClientProtocol: (identity: string) => number; + + constructor( + engine: RTCEngine, + log: StructuredLogger, + getRemoteParticipantClientProtocol: (identity: string) => number, + ) { this.engine = engine; this.log = log; + this.getRemoteParticipantClientProtocol = getRemoteParticipantClientProtocol; } setupEngine(engine: RTCEngine) { @@ -101,10 +111,25 @@ export default class OutgoingDataStreamManager { return writer.info; } + /** + * Returns true only if every recipient is known to support single-packet (inline) data streams. + * Broadcasts (no explicit destination identities) can't be guaranteed, so they are not eligible. + */ + private canSendInline(destinationIdentities?: Array): boolean { + if (!destinationIdentities || destinationIdentities.length === 0) { + return false; + } + return destinationIdentities.every( + (identity) => + this.getRemoteParticipantClientProtocol(identity) >= CLIENT_PROTOCOL_DATA_STREAM_V2, + ); + } + /** * Attempts to send `text` as a single header packet with the payload smuggled into a reserved - * attribute. Returns the resulting {@link TextStreamInfo} if it fit under the MTU, or `undefined` - * if the caller should fall back to the regular chunked stream. + * attribute. Returns the resulting {@link TextStreamInfo} if it was sent inline, or `null` if the + * caller should fall back to the regular chunked stream (recipient doesn't support data streams + * v2, or the payload is too large to fit under the MTU). */ private async trySendInlineText( streamId: string, @@ -112,6 +137,10 @@ export default class OutgoingDataStreamManager { totalTextLength: number, options?: SendTextOptions, ): Promise { + if (!this.canSendInline(options?.destinationIdentities)) { + return null; + } + const info: TextStreamInfo = { id: streamId, mimeType: 'text/plain', diff --git a/src/room/rpc/client/RpcClientManager.test.ts b/src/room/rpc/client/RpcClientManager.test.ts index 8f9184c141..68b1298119 100644 --- a/src/room/rpc/client/RpcClientManager.test.ts +++ b/src/room/rpc/client/RpcClientManager.test.ts @@ -16,6 +16,7 @@ describe('RpcClientManager', () => { const outgoingDataStreamManager = new OutgoingDataStreamManager( {} as unknown as RTCEngine, log, + (_identity) => CLIENT_PROTOCOL_DEFAULT, ); rpcClientManager = new RpcClientManager( diff --git a/src/room/rpc/server/RpcServerManager.test.ts b/src/room/rpc/server/RpcServerManager.test.ts index 8f26e495de..be7bdd3a7e 100644 --- a/src/room/rpc/server/RpcServerManager.test.ts +++ b/src/room/rpc/server/RpcServerManager.test.ts @@ -2,7 +2,11 @@ import { RpcRequest } from '@livekit/protocol'; import { assert, beforeEach, describe, expect, it, vi } from 'vitest'; import log from '../../../logger'; import { subscribeToEvents } from '../../../utils/subscribeToEvents'; -import { CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DEFAULT } from '../../../version'; +import { + CLIENT_PROTOCOL_DATA_STREAM_RPC, + CLIENT_PROTOCOL_DATA_STREAM_V2, + CLIENT_PROTOCOL_DEFAULT, +} from '../../../version'; import type RTCEngine from '../../RTCEngine'; import OutgoingDataStreamManager from '../../data-stream/outgoing/OutgoingDataStreamManager'; import { RPC_RESPONSE_DATA_STREAM_TOPIC, RpcError, RpcRequestAttrs } from '../utils'; @@ -17,6 +21,7 @@ describe('RpcServerManager', () => { const outgoingDataStreamManager = new OutgoingDataStreamManager( {} as unknown as RTCEngine, log, + (_identity) => CLIENT_PROTOCOL_DEFAULT, ); rpcServerManager = new RpcServerManager( @@ -186,7 +191,11 @@ describe('RpcServerManager', () => { let outgoingDataStreamManager: OutgoingDataStreamManager; beforeEach(() => { - outgoingDataStreamManager = new OutgoingDataStreamManager({} as unknown as RTCEngine, log); + outgoingDataStreamManager = new OutgoingDataStreamManager( + {} as unknown as RTCEngine, + log, + (_identity) => CLIENT_PROTOCOL_DATA_STREAM_V2, + ); vi.spyOn(outgoingDataStreamManager, 'sendText').mockResolvedValue(undefined as any); @@ -412,6 +421,7 @@ describe('RpcServerManager', () => { const outgoingDataStreamManager = new OutgoingDataStreamManager( {} as unknown as RTCEngine, log, + (_identity) => CLIENT_PROTOCOL_DEFAULT, ); const sendTextSpy = vi.spyOn(outgoingDataStreamManager, 'sendText'); diff --git a/src/version.ts b/src/version.ts index ed123a1f06..75ad4d28d6 100644 --- a/src/version.ts +++ b/src/version.ts @@ -8,7 +8,11 @@ export const CLIENT_PROTOCOL_DEFAULT = 0; /** Replaces RPC v1 protocol with a v2 data streams based one to support unlimited request / * response payload length. */ export const CLIENT_PROTOCOL_DATA_STREAM_RPC = 1; +/** "Data streams v2": the client knows how to receive a single-packet data stream (a stream whose + * entire payload is smuggled into the header packet, with no chunk/trailer packets). Senders only + * use the single-packet optimization when the recipient advertises at least this protocol. */ +export const CLIENT_PROTOCOL_DATA_STREAM_V2 = 2; /** The client protocol version indicates what level of support that the client has for * client <-> client api interactions. */ -export const clientProtocol = CLIENT_PROTOCOL_DATA_STREAM_RPC; +export const clientProtocol = CLIENT_PROTOCOL_DATA_STREAM_V2; From 2d7936f3f60605bdc00e104247d9c1bff6fd86f3 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 4 Jun 2026 14:47:57 -0400 Subject: [PATCH 06/68] feat: check all participants in room before broadcasting a v2 data stream --- src/room/Room.ts | 5 +++++ .../outgoing/OutgoingDataStreamManager.ts | 19 ++++++++++++++----- src/room/rpc/client/RpcClientManager.test.ts | 1 + src/room/rpc/server/RpcServerManager.test.ts | 3 +++ 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/room/Room.ts b/src/room/Room.ts index d33c16034f..2f71100f89 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -272,6 +272,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.engine, this.log, this.getRemoteParticipantClientProtocol, + this.getAllRemoteParticipantIdentities, ); this.incomingDataTrackManager = new IncomingDataTrackManager({ e2eeManager: this.e2eeManager }); @@ -2513,6 +2514,10 @@ class Room extends (EventEmitter as new () => TypedEmitter) return this.remoteParticipants.get(identity)?.clientProtocol ?? CLIENT_PROTOCOL_DEFAULT; }; + private getAllRemoteParticipantIdentities = () => { + return Array.from(this.remoteParticipants.keys()); + }; + private registerRpcDataStreamHandler() { this.incomingDataStreamManager.registerTextStreamHandler( RPC_REQUEST_DATA_STREAM_TOPIC, diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 270db427f2..ecd800ff0d 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -40,14 +40,20 @@ export default class OutgoingDataStreamManager { * recipient can receive single-packet (inline) data streams. */ protected getRemoteParticipantClientProtocol: (identity: string) => number; + /** Returns the identities of every remote participant currently in the room, used to decide + * whether a broadcast (no explicit destinations) can be sent inline. */ + protected getAllRemoteParticipantIdentities: () => Array; + constructor( engine: RTCEngine, log: StructuredLogger, getRemoteParticipantClientProtocol: (identity: string) => number, + getAllRemoteParticipantIdentities: () => Array, ) { this.engine = engine; this.log = log; this.getRemoteParticipantClientProtocol = getRemoteParticipantClientProtocol; + this.getAllRemoteParticipantIdentities = getAllRemoteParticipantIdentities; } setupEngine(engine: RTCEngine) { @@ -113,13 +119,16 @@ export default class OutgoingDataStreamManager { /** * Returns true only if every recipient is known to support single-packet (inline) data streams. - * Broadcasts (no explicit destination identities) can't be guaranteed, so they are not eligible. + * For a targeted send this checks the named destination identities; for a broadcast (no explicit + * destinations) it checks every remote participant currently in the room. An empty room (nobody + * to receive) is considered eligible. */ private canSendInline(destinationIdentities?: Array): boolean { - if (!destinationIdentities || destinationIdentities.length === 0) { - return false; - } - return destinationIdentities.every( + const identities = + destinationIdentities && destinationIdentities.length > 0 + ? destinationIdentities + : this.getAllRemoteParticipantIdentities(); + return identities.every( (identity) => this.getRemoteParticipantClientProtocol(identity) >= CLIENT_PROTOCOL_DATA_STREAM_V2, ); diff --git a/src/room/rpc/client/RpcClientManager.test.ts b/src/room/rpc/client/RpcClientManager.test.ts index 68b1298119..3ff496bd44 100644 --- a/src/room/rpc/client/RpcClientManager.test.ts +++ b/src/room/rpc/client/RpcClientManager.test.ts @@ -17,6 +17,7 @@ describe('RpcClientManager', () => { {} as unknown as RTCEngine, log, (_identity) => CLIENT_PROTOCOL_DEFAULT, + () => [], ); rpcClientManager = new RpcClientManager( diff --git a/src/room/rpc/server/RpcServerManager.test.ts b/src/room/rpc/server/RpcServerManager.test.ts index be7bdd3a7e..e083712f67 100644 --- a/src/room/rpc/server/RpcServerManager.test.ts +++ b/src/room/rpc/server/RpcServerManager.test.ts @@ -22,6 +22,7 @@ describe('RpcServerManager', () => { {} as unknown as RTCEngine, log, (_identity) => CLIENT_PROTOCOL_DEFAULT, + () => [], ); rpcServerManager = new RpcServerManager( @@ -195,6 +196,7 @@ describe('RpcServerManager', () => { {} as unknown as RTCEngine, log, (_identity) => CLIENT_PROTOCOL_DATA_STREAM_V2, + () => [], ); vi.spyOn(outgoingDataStreamManager, 'sendText').mockResolvedValue(undefined as any); @@ -422,6 +424,7 @@ describe('RpcServerManager', () => { {} as unknown as RTCEngine, log, (_identity) => CLIENT_PROTOCOL_DEFAULT, + () => [], ); const sendTextSpy = vi.spyOn(outgoingDataStreamManager, 'sendText'); From a18007c6d4cc3b00571c73cf07077ccb93336386 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 5 Jun 2026 10:46:29 -0400 Subject: [PATCH 07/68] feat: add initial data stream benchmark app --- examples/data-stream-benchmark/README.md | 42 + examples/data-stream-benchmark/api.ts | 121 + examples/data-stream-benchmark/benchmark.ts | 526 ++++ examples/data-stream-benchmark/index.html | 36 + examples/data-stream-benchmark/package.json | 27 + examples/data-stream-benchmark/payload.ts | 68 + examples/data-stream-benchmark/pnpm-lock.yaml | 2462 +++++++++++++++++ .../data-stream-benchmark/pnpm-workspace.yaml | 8 + examples/data-stream-benchmark/styles.css | 148 + examples/data-stream-benchmark/tsconfig.json | 20 + examples/data-stream-benchmark/vite.config.js | 10 + 11 files changed, 3468 insertions(+) create mode 100644 examples/data-stream-benchmark/README.md create mode 100644 examples/data-stream-benchmark/api.ts create mode 100644 examples/data-stream-benchmark/benchmark.ts create mode 100644 examples/data-stream-benchmark/index.html create mode 100644 examples/data-stream-benchmark/package.json create mode 100644 examples/data-stream-benchmark/payload.ts create mode 100644 examples/data-stream-benchmark/pnpm-lock.yaml create mode 100644 examples/data-stream-benchmark/pnpm-workspace.yaml create mode 100644 examples/data-stream-benchmark/styles.css create mode 100644 examples/data-stream-benchmark/tsconfig.json create mode 100644 examples/data-stream-benchmark/vite.config.js diff --git a/examples/data-stream-benchmark/README.md b/examples/data-stream-benchmark/README.md new file mode 100644 index 0000000000..f4a4beafc1 --- /dev/null +++ b/examples/data-stream-benchmark/README.md @@ -0,0 +1,42 @@ +# Data Stream Benchmark + +Measures the end-to-end latency of LiveKit **v2 data streams** across a grid of network conditions +(X axis) and payload sizes (Y axis). + +Two participants (`bench-sender` and `bench-receiver`) join a shared room in the same browser tab. +For each box in the grid the sender sends a fixed number of data streams (default 10) of random, +realistic JSON data to the receiver. Each stream header carries a `checksum` (XOR of the payload +bytes) and a `sendTs` timestamp; the receiver verifies the checksum and computes the end-to-end +latency. Each cell shows the average latency over the checksum-matching streams, or `N/A`. + +## Running + +1. Create `.env.local` with `LIVEKIT_API_KEY`, `LIVEKIT_API_SECRET`, and `LIVEKIT_URL`. +2. Install dependencies: `pnpm install` +3. Start the server: `pnpm dev` +4. Open the local URL (typically http://localhost:5173). +5. Click **Connect**, then **Run Benchmark**. Click **Disconnect** when done. + +## Network conditioning (macOS only) + +The **X axis** (Edge / 3G / LTE / Wi-Fi / None) is driven by the macOS **Network Link Conditioner**. +The server (`api.ts`) toggles it via `osascript` through `POST /api/network-condition`. + +Prerequisites: + +- The **Network Link Conditioner** preference pane must be installed + (`/Library/PreferencePanes/Network Link Conditioner.prefPane`, from Apple's "Additional Tools for + Xcode"). +- The process running `vite`/node needs macOS **Accessibility** permission (System Settings → + Privacy & Security → Accessibility) so it can drive System Settings. +- The preset names must match the conditioner's menu exactly: `Edge`, `3G`, `LTE`, `Wi-Fi`. + +The benchmark always resets the conditioner to **off** when it finishes, errors, or you disconnect. + +## Notes + +- The v2 single-packet (inline) optimization only changes behavior for payloads under the ~15 KB + header budget (10 B–10 KB); 100 KB and 1 MB are multi-packet regardless. +- `Edge`/`3G` × `1 MB` can be slow; each send has a 60 s timeout (`SEND_TIMEOUT_MS` in + `benchmark.ts`) and slow boxes show `N/A`. Repeat count and timeout are constants at the top of + `benchmark.ts`. diff --git a/examples/data-stream-benchmark/api.ts b/examples/data-stream-benchmark/api.ts new file mode 100644 index 0000000000..850fdd7173 --- /dev/null +++ b/examples/data-stream-benchmark/api.ts @@ -0,0 +1,121 @@ +import { exec } from 'child_process'; +import dotenv from 'dotenv'; +import express from 'express'; +import { AccessToken } from 'livekit-server-sdk'; +import { promisify } from 'util'; +import type { Express } from 'express'; + +dotenv.config({ path: '.env.local' }); + +const execAsync = promisify(exec); + +const LIVEKIT_API_KEY = process.env.LIVEKIT_API_KEY; +const LIVEKIT_API_SECRET = process.env.LIVEKIT_API_SECRET; +const LIVEKIT_URL = process.env.LIVEKIT_URL; + +/** Network Link Conditioner presets we allow the client to request. */ +const ALLOWED_PRESETS = ['Edge', '3G', 'LTE', 'Wi-Fi', 'Very Bad Network']; + +const app = express(); +app.use(express.json()); + +app.post('/api/get-token', async (req, res) => { + const { identity, roomName } = req.body; + + if (!LIVEKIT_API_KEY || !LIVEKIT_API_SECRET) { + res.status(500).json({ error: 'Server misconfigured' }); + return; + } + + const token = new AccessToken(LIVEKIT_API_KEY, LIVEKIT_API_SECRET, { + identity, + }); + token.addGrant({ + room: roomName, + roomJoin: true, + canPublish: true, + canSubscribe: true, + }); + + res.json({ + token: await token.toJwt(), + url: LIVEKIT_URL, + }); +}); + +/** + * Enables the macOS Network Link Conditioner at the given preset. The preset name is read from the + * `mode` environment variable via `system attribute "mode"`. + */ +const ENABLE_SCRIPT = `osascript <<'EOF' +set mode to system attribute "mode" +do shell script "open '/Library/PreferencePanes/Network Link Conditioner.prefPane'" +delay 2 +tell application "System Settings" to activate +tell application "System Events" + tell process "System Settings" + tell window "Network Link Conditioner" + tell scroll area 1 of group 3 of splitter group 1 of group 1 + tell group 1 + click pop up button 1 + delay 0.3 + click menu item mode of menu 1 of pop up button 1 + end tell + delay 0.3 + click button "ON" + end tell + end tell + end tell +end tell +EOF`; + +/** Disables the macOS Network Link Conditioner, returning to the host network speed. */ +const DISABLE_SCRIPT = `osascript <<'EOF' +do shell script "open '/Library/PreferencePanes/Network Link Conditioner.prefPane'" +delay 2 +tell application "System Settings" to activate +tell application "System Events" + tell process "System Settings" + tell window "Network Link Conditioner" + tell scroll area 1 of group 3 of splitter group 1 of group 1 + click button "OFF" + end tell + end tell + end tell +end tell +EOF`; + +app.post('/api/network-condition', async (req, res) => { + const { preset } = req.body as { preset?: string }; + + if (preset === 'off') { + try { + await execAsync(DISABLE_SCRIPT, { timeout: 30_000 }); + res.json({ ok: true, preset: 'off' }); + } catch (error) { + res.status(500).json({ error: `Failed to disable network conditioner: ${String(error)}` }); + } + return; + } + + if (!preset || !ALLOWED_PRESETS.includes(preset)) { + res + .status(400) + .json({ error: `Invalid preset "${preset}". Allowed: ${ALLOWED_PRESETS.join(', ')}, off` }); + return; + } + + try { + await execAsync(ENABLE_SCRIPT, { + timeout: 30_000, + env: { ...process.env, mode: preset }, + }); + res.json({ ok: true, preset }); + } catch (error) { + res + .status(500) + .json({ error: `Failed to set network conditioner to ${preset}: ${String(error)}` }); + } +}); + +export const handler: Express = app; diff --git a/examples/data-stream-benchmark/benchmark.ts b/examples/data-stream-benchmark/benchmark.ts new file mode 100644 index 0000000000..2225fa333a --- /dev/null +++ b/examples/data-stream-benchmark/benchmark.ts @@ -0,0 +1,526 @@ +import { Room, RoomEvent } from '../../src/index'; +import { checksum, generatePayload } from './payload'; + +// --------------------------------------------------------------------------- +// Tunables +// --------------------------------------------------------------------------- + +/** Number of concurrent caller "threads" (async send loops) per box. */ +const CONCURRENCY = 4; +/** How long each box sends for. */ +const BOX_DURATION_MS = 5_000; +/** Grace period after the send window to let in-flight streams finish arriving. */ +const DRAIN_MS = 2_000; +/** Received count that maps to a fully-unfilled (white) cell; 0 received is fully filled. */ +const MAX_FILL_COUNT = BOX_DURATION_MS; +/** Cell fill hue (R,G,B); opacity scales with throughput. */ +const FILL_RGB = '52,152,219'; + +/** How many chunks to split up the data stream payload into. If `0`, send all at once with `sendText`. */ +const STREAM_CHUNK_SIZE_BYTES = 0; + +const TOPIC = 'benchmark'; +const SENDER_IDENTITY = 'bench-sender'; +const RECEIVER_IDENTITY = 'bench-receiver'; + +const SIZES: Array<{ label: string; bytes: number }> = [ + { label: '10 B', bytes: 10 }, + { label: '100 B', bytes: 100 }, + { label: '1 KB', bytes: 1_000 }, + { label: '15 KB', bytes: 15_000 }, + { label: '100 KB', bytes: 100_000 }, + { label: '500 KB', bytes: 500_000 }, + { label: '1 MB', bytes: 1_000_000 }, +]; + +// `value` is the preset passed to /api/network-condition ('off' disables the conditioner). +const PRESETS: Array<{ label: string; value: string }> = [ + { label: 'None', value: 'off' }, + { label: 'Wi-Fi', value: 'Wi-Fi' }, + { label: 'LTE', value: 'LTE' }, + { label: '3G', value: '3G' }, + { label: 'Edge', value: 'Edge' }, + { label: 'Very Bad Net.', value: 'Very Bad Network' }, +]; + +// --------------------------------------------------------------------------- +// State +// --------------------------------------------------------------------------- + +/** Per-box metrics, modeled on the old RPC benchmark's BenchmarkStats but adapted to one-directional + * data streams (latency is measured one-way: receiver clock − sender's `sendTs` attribute). A fresh + * instance is created per box and captured by that box's receiver handler, so late (post-snapshot) + * arrivals can't leak into the next box. */ +class BoxStats { + sent = 0; + + sendErrors = 0; + + received = 0; + + mismatch = 0; + + latencies: number[] = []; + + errors: Record = {}; + + recordSent() { + this.sent += 1; + } + + recordSendError(kind: string) { + this.sendErrors += 1; + this.errors[kind] = (this.errors[kind] ?? 0) + 1; + } + + recordReceived(latencyMs: number, checksumOk: boolean) { + this.received += 1; + this.latencies.push(latencyMs); + if (!checksumOk) { + this.mismatch += 1; + } + } + + /** Arrived and checksum-valid — the analog of the old benchmark's `successfulCalls`. */ + get valid() { + return this.received - this.mismatch; + } + + /** Sent but never received within the drain window. */ + get lost() { + return Math.max(0, this.sent - this.received); + } + + get successRate() { + return this.sent > 0 ? (100 * this.valid) / this.sent : 0; + } + + private sortedLatencies(): number[] { + return [...this.latencies].sort((a, b) => a - b); + } + + get avgLatency(): number { + if (this.latencies.length === 0) { + return 0; + } + return this.latencies.reduce((a, b) => a + b, 0) / this.latencies.length; + } + + percentile(p: number): number { + const s = this.sortedLatencies(); + if (s.length === 0) { + return 0; + } + const idx = Math.min(Math.floor((p / 100) * s.length), s.length - 1); + return s[idx]; + } + + /** Received streams per second over the send window. */ + throughput(elapsedSec: number): number { + return elapsedSec > 0 ? this.received / elapsedSec : 0; + } + + /** Received bytes per second over the send window. */ + bytesPerSec(sizeBytes: number, elapsedSec: number): number { + return elapsedSec > 0 ? (this.received * sizeBytes) / elapsedSec : 0; + } +} + +let senderRoom: Room | null = null; +let receiverRoom: Room | null = null; +let running = false; + +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +// --------------------------------------------------------------------------- +// UI helpers +// --------------------------------------------------------------------------- + +const $ = (id: string) => document.getElementById(id) as T; + +function log(message: string) { + const area = $('log'); + const ts = new Date().toLocaleTimeString(); + area.value += `[${ts}] ${message}\n`; + area.scrollTop = area.scrollHeight; + // eslint-disable-next-line no-console + console.log(message); +} + +function setStatus(message: string) { + $('status').textContent = message; +} + +function cell(rowIdx: number, colIdx: number) { + return $(`cell-${rowIdx}-${colIdx}`); +} + +function buildGrid() { + const table = $('grid'); + const header = ['Payload \\ Network', ...PRESETS.map((p) => p.label)] + .map((label) => `${label}`) + .join(''); + + const rows = SIZES.map((size, rowIdx) => { + const cells = PRESETS.map( + (_, colIdx) => + `
—
`, + ).join(''); + return `${size.label}${cells}`; + }).join(''); + + table.innerHTML = `${header}${rows}`; +} + +/** Renders a box: big throughput number, a compact multi-metric status line below, and a fill + * opacity from the received count. `statusHtml` may contain markup (e.g. a red mismatch token). */ +function renderCell( + rowIdx: number, + colIdx: number, + opts: { recv: string; statusHtml?: string; fill?: number; title?: string; running?: boolean }, +) { + const td = cell(rowIdx, colIdx); + const recvEl = td.querySelector('.recv') as HTMLElement; + const statusEl = td.querySelector('.status') as HTMLElement; + + recvEl.textContent = opts.recv; + statusEl.innerHTML = opts.statusHtml ?? ''; + td.className = opts.running ? 'cell running' : 'cell'; + td.title = opts.title ?? ''; + + if (opts.fill === undefined) { + td.style.backgroundColor = ''; + } else { + const alpha = Math.max(0, Math.min(1, opts.fill / MAX_FILL_COUNT)); + td.style.backgroundColor = `rgba(${FILL_RGB}, ${alpha.toFixed(3)})`; + } +} + +function setButtons(opts: { run: boolean; stop: boolean }) { + $('run').disabled = !opts.run; + $('stop').disabled = !opts.stop; +} + +// --------------------------------------------------------------------------- +// Networking helpers +// --------------------------------------------------------------------------- + +async function fetchToken( + identity: string, + roomName: string, +): Promise<{ token: string; url: string }> { + const response = await fetch('/api/get-token', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ identity, roomName }), + }); + if (!response.ok) { + throw new Error('Failed to fetch token'); + } + const data = await response.json(); + return { token: data.token, url: data.url }; +} + +async function setNetwork(preset: string): Promise { + const response = await fetch('/api/network-condition', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ preset }), + }); + if (!response.ok) { + const body = await response.text(); + throw new Error(`Failed to set network condition "${preset}": ${body}`); + } +} + +/** Waits until `room` sees a remote participant with the given identity (so sends can target it). */ +function waitForParticipant(room: Room, identity: string, timeoutMs = 15_000): Promise { + return new Promise((resolve, reject) => { + if (room.remoteParticipants.has(identity)) { + resolve(); + return; + } + const timer = setTimeout(() => { + room.off(RoomEvent.ParticipantConnected, onConnected); + reject(new Error(`timed out waiting for participant "${identity}"`)); + }, timeoutMs); + const onConnected = () => { + if (room.remoteParticipants.has(identity)) { + clearTimeout(timer); + room.off(RoomEvent.ParticipantConnected, onConnected); + resolve(); + } + }; + room.on(RoomEvent.ParticipantConnected, onConnected); + }); +} + +// --------------------------------------------------------------------------- +// Per-box connection lifecycle +// --------------------------------------------------------------------------- + +/** Connects a fresh sender/receiver pair into a new room and wires up the receiver metrics. */ +async function connectPair(stats: BoxStats): Promise { + const roomName = `ds-bench-${Math.random().toString(36).substring(7)}`; + receiverRoom = new Room(); + senderRoom = new Room(); + + // Register the receive handler before connecting so no stream is missed. The handler closes over + // this box's `stats` object, so post-snapshot stragglers can't bleed into the next box. + receiverRoom.registerTextStreamHandler(TOPIC, async (reader) => { + const attrs = reader.info.attributes ?? {}; + try { + const text = await reader.readAll(); + // One-way end-to-end latency — sender and receiver share this tab's clock. + const latency = Date.now() - Number(attrs.sendTs); + stats.recordReceived(latency, `${checksum(text)}` === attrs.checksum); + } catch { + // dropped/aborted mid-stream — not counted as received + } + }); + + const [senderToken, receiverToken] = await Promise.all([ + fetchToken(SENDER_IDENTITY, roomName), + fetchToken(RECEIVER_IDENTITY, roomName), + ]); + + await Promise.all([ + senderRoom.connect(senderToken.url, senderToken.token), + receiverRoom.connect(receiverToken.url, receiverToken.token), + ]); + + // The sender must see the receiver (with its advertised protocol) before sending, otherwise sends + // would fall back to the chunked path. + await waitForParticipant(senderRoom, RECEIVER_IDENTITY); +} + +async function disconnectPair(): Promise { + await Promise.allSettled([senderRoom?.disconnect(), receiverRoom?.disconnect()]); + senderRoom = null; + receiverRoom = null; +} + +// --------------------------------------------------------------------------- +// One box: fresh connect -> N concurrent senders for a fixed window -> disconnect +// --------------------------------------------------------------------------- + +async function runBox(rowIdx: number, colIdx: number, sizeBytes: number, label: string) { + const stats = new BoxStats(); + renderCell(rowIdx, colIdx, { recv: '…', running: true }); + + for (let i = 1; i <= 3; i += 1) { + try { + await connectPair(stats); + break; + } catch (err) { + log(`${label}: connect failed (try ${i}/3): ${String(err)}`); + await disconnectPair(); + if (i >= 3) { + renderCell(rowIdx, colIdx, { recv: 'conn err', fill: 0, title: String(err) }); + return; + } + } + } + + const callerLoop = async () => { + let resolve: (() => void) | null = null; + let timeoutHit = false; + setTimeout(() => { + timeoutHit = true; + resolve?.(); + }, BOX_DURATION_MS); + + const iteration = async () => { + const room = senderRoom; + if (!room) { + return; + } + + const payload = generatePayload(sizeBytes); + let promise; + if (STREAM_CHUNK_SIZE_BYTES > 0) { + // Stream payload data in STREAM_CHUNK_SIZE_BYTES chunks + const writer = await room.localParticipant.streamText({ + topic: TOPIC, + destinationIdentities: [RECEIVER_IDENTITY], + attributes: { sendTs: `${Date.now()}`, checksum: `${checksum(payload)}` }, + }); + for (let i = 0; i < Math.ceil(sizeBytes / STREAM_CHUNK_SIZE_BYTES); i += 1) { + await writer.write( + payload.slice(i * STREAM_CHUNK_SIZE_BYTES, (i + 1) * STREAM_CHUNK_SIZE_BYTES), + ); + } + promise = writer.close(); + } else { + // Send payload all in one go + promise = room.localParticipant.sendText(payload, { + topic: TOPIC, + destinationIdentities: [RECEIVER_IDENTITY], + attributes: { sendTs: `${Date.now()}`, checksum: `${checksum(payload)}` }, + }); + } + + promise + .then(async () => { + if (timeoutHit) { + return; + } + stats.recordSent(); + await iteration(); + }) + .catch(async (err) => { + if (timeoutHit) { + return; + } + + // Under a throttled link sends can fail transiently; back off briefly and keep going. + stats.recordSendError(errorKind(err)); + await sleep(50); + await iteration(); + }); + }; + iteration(); + + return new Promise((r) => { + resolve = r; + }); + }; + + await Promise.all(Array.from({ length: CONCURRENCY }, () => callerLoop())); + + // Let already-sent streams finish arriving before snapshotting and tearing down. + await sleep(DRAIN_MS); + await disconnectPair(); + + renderBoxStats(rowIdx, colIdx, label, stats, sizeBytes); +} + +/** Shortens an error into a stable bucket key for the error summary. */ +function errorKind(err: unknown): string { + if (err instanceof Error) { + return err.name && err.name !== 'Error' ? err.name : err.message.split('\n')[0].slice(0, 60); + } + return String(err).split('\n')[0].slice(0, 60); +} + +/** Renders the box cell (throughput + status line) and logs the full per-box summary. */ +function renderBoxStats( + rowIdx: number, + colIdx: number, + label: string, + stats: BoxStats, + sizeBytes: number, +) { + const elapsedSec = BOX_DURATION_MS / 1000; + const tput = stats.throughput(elapsedSec); + const kbPerSec = stats.bytesPerSec(sizeBytes, elapsedSec) / 1000; + const { sent, received, valid, mismatch, lost, sendErrors } = stats; + const p50 = stats.percentile(50); + const p95 = stats.percentile(95); + const p99 = stats.percentile(99); + const avg = stats.avgLatency; + const rate = stats.successRate; + + const mismatchTok = mismatch ? ` ✗${mismatch}` : ''; + const statusHtml = + `↑${sent} ✓${valid}${mismatchTok} ⊘${lost} · ${rate.toFixed(0)}%
` + + `p50 ${p50.toFixed(0)} p95 ${p95.toFixed(0)} p99 ${p99.toFixed(0)} ms`; + + renderCell(rowIdx, colIdx, { + recv: `${tput.toFixed(1)} ds/s`, + statusHtml, + fill: received, + title: + `sent ${sent}, received ${received}, valid ${valid}, mismatch ${mismatch}, ` + + `lost ${lost}, sendErrors ${sendErrors}`, + }); + + const errs = Object.keys(stats.errors).length ? ` · errors ${JSON.stringify(stats.errors)}` : ''; + log( + `${label}: sent ${sent} valid ${valid} (${rate.toFixed(1)}%) recv ${received} ✗${mismatch} ` + + `lost ${lost} sendErr ${sendErrors}`, + ); + log( + `${label}: lat avg ${avg.toFixed(1)} p50 ${p50.toFixed(1)} p95 ${p95.toFixed(1)} ` + + `p99 ${p99.toFixed(1)} ms · ${tput.toFixed(1)} ds/s · ${kbPerSec.toFixed(1)} KB/s${errs}`, + ); +} + +// --------------------------------------------------------------------------- +// Run / stop +// --------------------------------------------------------------------------- + +async function runBenchmark() { + if (running) { + return; + } + running = true; + setButtons({ run: false, stop: true }); + buildGrid(); + log( + `Starting benchmark: ${CONCURRENCY} concurrent senders, ${BOX_DURATION_MS / 1000}s per box, ` + + `fresh connection per box.`, + ); + + try { + for (let colIdx = 0; colIdx < PRESETS.length && running; colIdx += 1) { + const preset = PRESETS[colIdx]; + setStatus(`Setting network condition: ${preset.label}…`); + log(`Setting network condition: ${preset.label}`); + await setNetwork(preset.value); + + for (let rowIdx = 0; rowIdx < SIZES.length && running; rowIdx += 1) { + const size = SIZES[rowIdx]; + const label = `${preset.label} · ${size.label}`; + setStatus(`${label} — sending for ${BOX_DURATION_MS / 1000}s…`); + await runBox(rowIdx, colIdx, size.bytes, label); + } + } + log(running ? 'Benchmark complete.' : 'Benchmark stopped.'); + setStatus(running ? 'Benchmark complete' : 'Stopped'); + } catch (err) { + log(`Benchmark error: ${String(err)}`); + setStatus('Benchmark error'); + } finally { + // Always reset the conditioner and tear down any lingering connection. + try { + await setNetwork('off'); + } catch (err) { + log(`Failed to reset network: ${String(err)}`); + } + await disconnectPair(); + running = false; + setButtons({ run: true, stop: false }); + } +} + +async function stop() { + if (!running) { + return; + } + log('Stopping…'); + setStatus('Stopping…'); + running = false; + setButtons({ run: false, stop: false }); + + // Always reset the conditioner and tear down any lingering connection. + try { + await setNetwork('off'); + } catch (err) { + log(`Failed to reset network: ${String(err)}`); + } + + // Abort any in-flight box immediately; runBenchmark's finally handles the rest. + await disconnectPair(); +} + +// --------------------------------------------------------------------------- +// Wire up +// --------------------------------------------------------------------------- + +document.addEventListener('DOMContentLoaded', () => { + buildGrid(); + setButtons({ run: true, stop: false }); + $('run').addEventListener('click', runBenchmark); + $('stop').addEventListener('click', stop); +}); diff --git a/examples/data-stream-benchmark/index.html b/examples/data-stream-benchmark/index.html new file mode 100644 index 0000000000..7706afe463 --- /dev/null +++ b/examples/data-stream-benchmark/index.html @@ -0,0 +1,36 @@ + + + + + + LiveKit Data Stream Benchmark + + + +
+

LiveKit v2 Data Stream Benchmark

+ +
+ + +
+ +
Idle
+ +
+ +

+ Each box does a fresh connect → send for a fixed window with several concurrent senders → + disconnect. The large number is throughput (data streams received per + second). The status line below shows ↑sent ✓valid ✗mismatch ⊘lost · success % and + one-way latency percentiles (p50/p95/p99). Box fill scales with throughput; full per-box + stats are printed to the log. +

+ +
+ +
+
+ + + diff --git a/examples/data-stream-benchmark/package.json b/examples/data-stream-benchmark/package.json new file mode 100644 index 0000000000..1d2f1599b5 --- /dev/null +++ b/examples/data-stream-benchmark/package.json @@ -0,0 +1,27 @@ +{ + "name": "livekit-data-stream-benchmark", + "version": "1.0.0", + "description": "Benchmark of LiveKit v2 data streams across network conditions and payload sizes", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc && vite build", + "preview": "vite preview" + }, + "dependencies": { + "cors": "^2.8.5", + "dotenv": "^16.4.5", + "express": "^5.2.1", + "livekit-server-sdk": "^2.7.0", + "vite": "^5.4.21", + "vite-plugin-mix": "^0.4.0" + }, + "devDependencies": { + "@types/cors": "^2.8.17", + "@types/express": "^5.0.0", + "@types/node": "^20.0.0", + "concurrently": "^8.2.0", + "tsx": "^4.7.0", + "typescript": "^5.4.5" + } +} diff --git a/examples/data-stream-benchmark/payload.ts b/examples/data-stream-benchmark/payload.ts new file mode 100644 index 0000000000..0408efb795 --- /dev/null +++ b/examples/data-stream-benchmark/payload.ts @@ -0,0 +1,68 @@ +/** + * Realistic structured JSON test data for the data-stream benchmark. Each line is a self-contained + * JSON object resembling production payloads (user profiles, events, metrics, logs, etc.). All lines + * are ASCII so that one character equals one byte, which lets us slice payloads to an exact byte + * length without worrying about UTF-8 boundaries. + */ +const TEST_DATA_LINES: string[] = [ + '{"id":"usr_a1b2c3","name":"Alice Chen","email":"alice.chen@example.com","role":"engineer","department":"platform","projects":["livekit-core","media-pipeline","signaling"],"metrics":{"commits":342,"reviews":128,"deployments":57},"location":"San Francisco, CA","joined":"2022-03-15T08:30:00Z"}', + '{"event":"room.participant_joined","timestamp":"2025-01-15T14:22:33.456Z","room_sid":"RM_xK9mPq2nR4","participant_sid":"PA_j7hLw3vYm1","identity":"speaker-042","metadata":{"display_name":"Dr. Sarah Mitchell","avatar_url":"https://cdn.example.com/avatars/sm042.jpg","hand_raised":false}}', + '{"sensor_id":"temp-rack-07b","readings":[{"ts":1705312800,"value":23.4,"unit":"celsius"},{"ts":1705312860,"value":23.6,"unit":"celsius"},{"ts":1705312920,"value":24.1,"unit":"celsius"},{"ts":1705312980,"value":23.8,"unit":"celsius"}],"status":"nominal","location":"datacenter-west-3"}', + '{"order_id":"ORD-2025-00847","customer":{"id":"cust_9f8e7d","name":"Bob Williams","tier":"premium"},"items":[{"sku":"WDG-1042","name":"Wireless Adapter Pro","qty":2,"price":49.99},{"sku":"CBL-3001","name":"USB-C Cable 2m","qty":5,"price":12.99}],"total":164.93,"currency":"USD","status":"processing"}', + '{"trace_id":"abc123def456","spans":[{"name":"http.request","duration_ms":245,"status":"ok","attributes":{"http.method":"POST","http.url":"/api/v2/rooms","http.status_code":201}},{"name":"db.query","duration_ms":12,"status":"ok","attributes":{"db.system":"postgresql","db.statement":"INSERT INTO rooms"}}]}', + '{"log_level":"warn","service":"media-router","instance":"mr-us-east-07","message":"Track subscription delayed due to network congestion","context":{"room_sid":"RM_pQ8nL2mK5x","track_sid":"TR_w4jR7vN9y3","participant_sid":"PA_k2mX5bH8r1","delay_ms":1847,"retry_count":3,"bandwidth_estimate_bps":2450000}}', + '{"config":{"video":{"codecs":["VP8","H264","AV1"],"simulcast":{"enabled":true,"layers":[{"rid":"f","maxBitrate":2500000,"maxFramerate":30},{"rid":"h","maxBitrate":800000,"maxFramerate":15},{"rid":"q","maxBitrate":200000,"maxFramerate":7}]},"dynacast":true},"audio":{"codecs":["opus"],"dtx":true,"red":true,"stereo":false}}}', + '{"benchmark":{"test":"data-stream-throughput","iteration":1547,"payload_bytes":15360,"latency_ms":23.7,"path":"inline","timestamp":"2025-06-20T10:15:33.891Z","sender":"bench-sender","receiver":"bench-receiver","room":"benchmark-room-8f3a"}}', + '{"user_id":"u_7k3m9p","session":{"id":"sess_abc123","started":"2025-01-15T09:00:00Z","duration_minutes":47,"pages_viewed":12,"actions":[{"type":"click","target":"#start-call","ts":1705308120},{"type":"input","target":"#chat-message","ts":1705308245},{"type":"click","target":"#share-screen","ts":1705308390}],"device":{"browser":"Chrome 121","os":"macOS 14.2","screen":"2560x1440"}}}', + '{"pipeline_id":"pipe_rtc_042","stages":[{"name":"capture","codec":"VP8","resolution":"1920x1080","fps":30,"bitrate_kbps":2500},{"name":"encode","profile":"constrained-baseline","hardware_accel":true,"latency_ms":4.2},{"name":"packetize","mtu":1200,"fec_enabled":true,"nack_enabled":true},{"name":"transport","protocol":"UDP","ice_candidates":3,"dtls_setup":"actpass"}]}', + '{"cluster":{"id":"lk-us-east-1","region":"us-east-1","nodes":[{"id":"node-01","type":"media","status":"healthy","load":0.67,"rooms":42,"participants":318,"cpu_pct":54.2,"mem_pct":71.8},{"id":"node-02","type":"media","status":"healthy","load":0.43,"rooms":31,"participants":201,"cpu_pct":38.1,"mem_pct":55.4}],"total_rooms":73,"total_participants":519}}', + '{"deployment":{"id":"deploy_20250115_003","service":"livekit-server","version":"1.8.2","environment":"production","region":"eu-west-1","status":"completed","started_at":"2025-01-15T03:00:00Z","completed_at":"2025-01-15T03:12:47Z","changes":["fix: ice restart race condition","feat: improved simulcast layer selection","perf: reduce memory allocation in media forwarding"],"rollback_available":true,"health_check":"passing"}}', + '{"analytics":{"room_id":"RM_daily_standup_042","period":"2025-01-15T09:00:00Z/2025-01-15T09:30:00Z","participants":{"total":8,"max_concurrent":7,"avg_duration_minutes":22.4},"media":{"audio":{"total_minutes":156.8,"avg_bitrate_kbps":32,"packet_loss_pct":0.02},"video":{"total_minutes":134.2,"avg_bitrate_kbps":1850,"packet_loss_pct":0.08,"avg_fps":28.7}},"quality_score":4.7}}', + '{"ticket":{"id":"TICKET-8472","title":"Intermittent audio dropout in large rooms","priority":"high","status":"in_progress","assignee":"eng-media-team","reporter":"support-agent-12","created":"2025-01-14T16:30:00Z","updated":"2025-01-15T11:22:00Z","labels":["audio","production","p1"],"comments_count":7,"related_incidents":["INC-2025-0042","INC-2025-0039"]}}', +]; + +let cachedBase: string | null = null; + +/** + * Builds (once) a large ASCII string by repeatedly joining the test data lines until it comfortably + * exceeds the largest payload we benchmark, so payloads can be sliced out of it. + */ +function getBase(): string { + if (cachedBase !== null) { + return cachedBase; + } + const minLength = 1_100_000; + const parts: string[] = []; + let length = 0; + let idx = 0; + while (length < minLength) { + const line = TEST_DATA_LINES[idx % TEST_DATA_LINES.length]; + parts.push(line); + length += line.length + 1; // +1 for the '\n' join separator + idx += 1; + } + cachedBase = parts.join('\n'); + return cachedBase; +} + +/** + * Returns a JSON-ish payload of exactly `targetBytes` bytes, sliced from a random offset of the + * shared base string (ASCII, so byte length === character length). + */ +export function generatePayload(targetBytes: number): string { + const base = getBase(); + if (targetBytes >= base.length) { + throw new Error(`requested payload (${targetBytes}) larger than base data (${base.length})`); + } + const maxOffset = base.length - targetBytes; + const offset = Math.floor(Math.random() * maxOffset); + return base.slice(offset, offset + targetBytes); +} + +export function checksum(str: string) { + let sum = 0; + for (let i = 0; i < str.length; i += 1) { + sum += str.charCodeAt(i); + } + return sum; +} diff --git a/examples/data-stream-benchmark/pnpm-lock.yaml b/examples/data-stream-benchmark/pnpm-lock.yaml new file mode 100644 index 0000000000..9c96ebc90e --- /dev/null +++ b/examples/data-stream-benchmark/pnpm-lock.yaml @@ -0,0 +1,2462 @@ +lockfileVersion: '9.0' + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + + .: + dependencies: + cors: + specifier: ^2.8.5 + version: 2.8.6 + dotenv: + specifier: ^16.4.5 + version: 16.6.1 + express: + specifier: ^5.2.1 + version: 5.2.1 + livekit-server-sdk: + specifier: ^2.7.0 + version: 2.15.0 + vite: + specifier: ^5.4.21 + version: 5.4.21(@types/node@20.19.41) + vite-plugin-mix: + specifier: ^0.4.0 + version: 0.4.0(vite@5.4.21(@types/node@20.19.41)) + devDependencies: + '@types/cors': + specifier: ^2.8.17 + version: 2.8.19 + '@types/express': + specifier: ^5.0.0 + version: 5.0.6 + '@types/node': + specifier: ^20.0.0 + version: 20.19.41 + concurrently: + specifier: ^8.2.0 + version: 8.2.2 + tsx: + specifier: ^4.7.0 + version: 4.21.0 + typescript: + specifier: ^5.4.5 + version: 5.9.3 + +packages: + + '@babel/runtime@7.28.6': + resolution: {integrity: sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==} + engines: {node: '>=6.9.0'} + + '@bufbuild/protobuf@1.10.1': + resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==} + + '@esbuild/aix-ppc64@0.21.5': + resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [aix] + + '@esbuild/aix-ppc64@0.27.3': + resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.21.5': + resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm64@0.27.3': + resolution: {integrity: sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.21.5': + resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} + engines: {node: '>=12'} + cpu: [arm] + os: [android] + + '@esbuild/android-arm@0.27.3': + resolution: {integrity: sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.21.5': + resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} + engines: {node: '>=12'} + cpu: [x64] + os: [android] + + '@esbuild/android-x64@0.27.3': + resolution: {integrity: sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.21.5': + resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} + engines: {node: '>=12'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-arm64@0.27.3': + resolution: {integrity: sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.21.5': + resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} + engines: {node: '>=12'} + cpu: [x64] + os: [darwin] + + '@esbuild/darwin-x64@0.27.3': + resolution: {integrity: sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.21.5': + resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} + engines: {node: '>=12'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-arm64@0.27.3': + resolution: {integrity: sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.21.5': + resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.27.3': + resolution: {integrity: sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.21.5': + resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} + engines: {node: '>=12'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm64@0.27.3': + resolution: {integrity: sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.21.5': + resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} + engines: {node: '>=12'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-arm@0.27.3': + resolution: {integrity: sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.21.5': + resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} + engines: {node: '>=12'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-ia32@0.27.3': + resolution: {integrity: sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.21.5': + resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} + engines: {node: '>=12'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-loong64@0.27.3': + resolution: {integrity: sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.21.5': + resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} + engines: {node: '>=12'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-mips64el@0.27.3': + resolution: {integrity: sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.21.5': + resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} + engines: {node: '>=12'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-ppc64@0.27.3': + resolution: {integrity: sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.21.5': + resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} + engines: {node: '>=12'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-riscv64@0.27.3': + resolution: {integrity: sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.21.5': + resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} + engines: {node: '>=12'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-s390x@0.27.3': + resolution: {integrity: sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.21.5': + resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} + engines: {node: '>=12'} + cpu: [x64] + os: [linux] + + '@esbuild/linux-x64@0.27.3': + resolution: {integrity: sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.27.3': + resolution: {integrity: sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.21.5': + resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} + engines: {node: '>=12'} + cpu: [x64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.27.3': + resolution: {integrity: sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.27.3': + resolution: {integrity: sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.21.5': + resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} + engines: {node: '>=12'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.27.3': + resolution: {integrity: sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/openharmony-arm64@0.27.3': + resolution: {integrity: sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + + '@esbuild/sunos-x64@0.21.5': + resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} + engines: {node: '>=12'} + cpu: [x64] + os: [sunos] + + '@esbuild/sunos-x64@0.27.3': + resolution: {integrity: sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.21.5': + resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} + engines: {node: '>=12'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-arm64@0.27.3': + resolution: {integrity: sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.21.5': + resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} + engines: {node: '>=12'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-ia32@0.27.3': + resolution: {integrity: sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.21.5': + resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} + engines: {node: '>=12'} + cpu: [x64] + os: [win32] + + '@esbuild/win32-x64@0.27.3': + resolution: {integrity: sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + + '@livekit/protocol@1.44.0': + resolution: {integrity: sha512-/vfhDUGcUKO8Q43r6i+5FrDhl5oZjm/X3U4x2Iciqvgn5C8qbj+57YPcWSJ1kyIZm5Cm6AV2nAPjMm3ETD/iyg==} + + '@rollup/rollup-android-arm-eabi@4.59.0': + resolution: {integrity: sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==} + cpu: [arm] + os: [android] + + '@rollup/rollup-android-arm64@4.59.0': + resolution: {integrity: sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==} + cpu: [arm64] + os: [android] + + '@rollup/rollup-darwin-arm64@4.59.0': + resolution: {integrity: sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==} + cpu: [arm64] + os: [darwin] + + '@rollup/rollup-darwin-x64@4.59.0': + resolution: {integrity: sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==} + cpu: [x64] + os: [darwin] + + '@rollup/rollup-freebsd-arm64@4.59.0': + resolution: {integrity: sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==} + cpu: [arm64] + os: [freebsd] + + '@rollup/rollup-freebsd-x64@4.59.0': + resolution: {integrity: sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==} + cpu: [x64] + os: [freebsd] + + '@rollup/rollup-linux-arm-gnueabihf@4.59.0': + resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} + cpu: [arm] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm-musleabihf@4.59.0': + resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} + cpu: [arm] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-arm64-gnu@4.59.0': + resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} + cpu: [arm64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-arm64-musl@4.59.0': + resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} + cpu: [arm64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-loong64-gnu@4.59.0': + resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} + cpu: [loong64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-loong64-musl@4.59.0': + resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} + cpu: [loong64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-ppc64-gnu@4.59.0': + resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} + cpu: [ppc64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-ppc64-musl@4.59.0': + resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} + cpu: [ppc64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-riscv64-gnu@4.59.0': + resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} + cpu: [riscv64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-riscv64-musl@4.59.0': + resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} + cpu: [riscv64] + os: [linux] + libc: [musl] + + '@rollup/rollup-linux-s390x-gnu@4.59.0': + resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} + cpu: [s390x] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-gnu@4.59.0': + resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} + cpu: [x64] + os: [linux] + libc: [glibc] + + '@rollup/rollup-linux-x64-musl@4.59.0': + resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} + cpu: [x64] + os: [linux] + libc: [musl] + + '@rollup/rollup-openbsd-x64@4.59.0': + resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} + cpu: [x64] + os: [openbsd] + + '@rollup/rollup-openharmony-arm64@4.59.0': + resolution: {integrity: sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==} + cpu: [arm64] + os: [openharmony] + + '@rollup/rollup-win32-arm64-msvc@4.59.0': + resolution: {integrity: sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==} + cpu: [arm64] + os: [win32] + + '@rollup/rollup-win32-ia32-msvc@4.59.0': + resolution: {integrity: sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==} + cpu: [ia32] + os: [win32] + + '@rollup/rollup-win32-x64-gnu@4.59.0': + resolution: {integrity: sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==} + cpu: [x64] + os: [win32] + + '@rollup/rollup-win32-x64-msvc@4.59.0': + resolution: {integrity: sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==} + cpu: [x64] + os: [win32] + + '@types/body-parser@1.19.6': + resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} + + '@types/connect@3.4.38': + resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + + '@types/cors@2.8.19': + resolution: {integrity: sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==} + + '@types/estree@1.0.8': + resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + + '@types/express-serve-static-core@5.1.1': + resolution: {integrity: sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A==} + + '@types/express@5.0.6': + resolution: {integrity: sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==} + + '@types/http-errors@2.0.5': + resolution: {integrity: sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==} + + '@types/node@20.19.41': + resolution: {integrity: sha512-ECymXOukMnOoVkC2bb1Vc/w/836DXncOg5m8Xj1RH7xSHZJWNYY6Zh7EH477vcnD5egKNNfy2RpNOmuChhFPgQ==} + + '@types/qs@6.14.0': + resolution: {integrity: sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==} + + '@types/range-parser@1.2.7': + resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} + + '@types/send@1.2.1': + resolution: {integrity: sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==} + + '@types/serve-static@2.2.0': + resolution: {integrity: sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==} + + '@vercel/nft@0.10.1': + resolution: {integrity: sha512-xhINCdohfeWg/70QLs3De/rfNFcO2+Sw4tL9oqgFl4zQzhogT3q0MjH6Hda5uM2KuFGndRPs6VkKJphAhWmymg==} + hasBin: true + + abbrev@1.1.1: + resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} + + accepts@2.0.0: + resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} + engines: {node: '>= 0.6'} + + acorn-class-fields@1.0.0: + resolution: {integrity: sha512-l+1FokF34AeCXGBHkrXFmml9nOIRI+2yBnBpO5MaVAaTIJ96irWLtcCxX+7hAp6USHFCe+iyyBB4ZhxV807wmA==} + engines: {node: '>=4.8.2'} + peerDependencies: + acorn: ^6 || ^7 || ^8 + + acorn-private-class-elements@1.0.0: + resolution: {integrity: sha512-zYNcZtxKgVCg1brS39BEou86mIao1EV7eeREG+6WMwKbuYTeivRRs6S2XdWnboRde6G9wKh2w+WBydEyJsJ6mg==} + engines: {node: '>=4.8.2'} + peerDependencies: + acorn: ^6.1.0 || ^7 || ^8 + + acorn-static-class-features@1.0.0: + resolution: {integrity: sha512-XZJECjbmMOKvMHiNzbiPXuXpLAJfN3dAKtfIYbk1eHiWdsutlek+gS7ND4B8yJ3oqvHo1NxfafnezVmq7NXK0A==} + engines: {node: '>=4.8.2'} + peerDependencies: + acorn: ^6.1.0 || ^7 || ^8 + + acorn@8.16.0: + resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} + engines: {node: '>=0.4.0'} + hasBin: true + + ansi-regex@2.1.1: + resolution: {integrity: sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==} + engines: {node: '>=0.10.0'} + + ansi-regex@5.0.1: + resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} + engines: {node: '>=8'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + aproba@1.2.0: + resolution: {integrity: sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==} + + are-we-there-yet@1.1.7: + resolution: {integrity: sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==} + deprecated: This package is no longer supported. + + balanced-match@1.0.2: + resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + + body-parser@2.2.2: + resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} + engines: {node: '>=18'} + + brace-expansion@1.1.12: + resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} + + braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} + engines: {node: '>=8'} + + bytes@3.1.2: + resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} + engines: {node: '>= 0.8'} + + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + + call-bound@1.0.4: + resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} + engines: {node: '>= 0.4'} + + camelcase-keys@9.1.3: + resolution: {integrity: sha512-Rircqi9ch8AnZscQcsA1C47NFdaO3wukpmIRzYcDOrmvgt78hM/sj5pZhZNec2NM12uk5vTwRHZ4anGcrC4ZTg==} + engines: {node: '>=16'} + + camelcase@8.0.0: + resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==} + engines: {node: '>=16'} + + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + + cliui@8.0.1: + resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} + engines: {node: '>=12'} + + code-point-at@1.1.0: + resolution: {integrity: sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==} + engines: {node: '>=0.10.0'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + concat-map@0.0.1: + resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} + + concurrently@8.2.2: + resolution: {integrity: sha512-1dP4gpXFhei8IOtlXRE/T/4H88ElHgTiUzh71YUmtjTEHMSRS2Z/fgOxHSxxusGHogsRfxNq1vyAwxSC+EVyDg==} + engines: {node: ^14.13.0 || >=16.0.0} + hasBin: true + + console-control-strings@1.1.0: + resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} + + content-disposition@1.0.1: + resolution: {integrity: sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==} + engines: {node: '>=18'} + + content-type@1.0.5: + resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} + engines: {node: '>= 0.6'} + + cookie-signature@1.2.2: + resolution: {integrity: sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==} + engines: {node: '>=6.6.0'} + + cookie@0.7.2: + resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} + engines: {node: '>= 0.6'} + + core-util-is@1.0.3: + resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} + + cors@2.8.6: + resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==} + engines: {node: '>= 0.10'} + + date-fns@2.30.0: + resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} + engines: {node: '>=0.11'} + + debug@3.2.7: + resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + + deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + + delegates@1.0.0: + resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} + + depd@2.0.0: + resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} + engines: {node: '>= 0.8'} + + detect-libc@1.0.3: + resolution: {integrity: sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==} + engines: {node: '>=0.10'} + hasBin: true + + dotenv@16.6.1: + resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} + engines: {node: '>=12'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + ee-first@1.1.1: + resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} + + emoji-regex@8.0.0: + resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} + + encodeurl@2.0.0: + resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} + engines: {node: '>= 0.8'} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + + es-object-atoms@1.1.1: + resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} + engines: {node: '>= 0.4'} + + esbuild@0.21.5: + resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} + engines: {node: '>=12'} + hasBin: true + + esbuild@0.27.3: + resolution: {integrity: sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==} + engines: {node: '>=18'} + hasBin: true + + escalade@3.2.0: + resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} + engines: {node: '>=6'} + + escape-html@1.0.3: + resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} + + estree-walker@0.6.1: + resolution: {integrity: sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==} + + etag@1.8.1: + resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} + engines: {node: '>= 0.6'} + + express@5.2.1: + resolution: {integrity: sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==} + engines: {node: '>= 18'} + + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + + fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} + engines: {node: '>=8'} + + finalhandler@2.1.1: + resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} + engines: {node: '>= 18.0.0'} + + forwarded@0.2.0: + resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} + engines: {node: '>= 0.6'} + + fresh@2.0.0: + resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} + engines: {node: '>= 0.8'} + + fs-minipass@1.2.7: + resolution: {integrity: sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==} + + fs.realpath@1.0.0: + resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} + + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + gauge@2.7.4: + resolution: {integrity: sha512-14x4kjc6lkD3ltw589k0NrPD6cCNTD6CWoVUNpB85+DrtONoZn+Rug6xZU5RvSC4+TZPxA5AnBibQYAvZn41Hg==} + deprecated: This package is no longer supported. + + get-caller-file@2.0.5: + resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} + engines: {node: 6.* || 8.* || >= 10.*} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + get-tsconfig@4.13.6: + resolution: {integrity: sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw==} + + glob@7.2.3: + resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-unicode@2.0.1: + resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==} + + hasown@2.0.2: + resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} + engines: {node: '>= 0.4'} + + http-errors@2.0.1: + resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} + engines: {node: '>= 0.8'} + + iconv-lite@0.4.24: + resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} + engines: {node: '>=0.10.0'} + + iconv-lite@0.7.2: + resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==} + engines: {node: '>=0.10.0'} + + ignore-walk@3.0.4: + resolution: {integrity: sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ==} + + inflight@1.0.6: + resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} + deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + + ipaddr.js@1.9.1: + resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} + engines: {node: '>= 0.10'} + + is-fullwidth-code-point@1.0.0: + resolution: {integrity: sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==} + engines: {node: '>=0.10.0'} + + is-fullwidth-code-point@3.0.0: + resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} + engines: {node: '>=8'} + + is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + + is-promise@4.0.0: + resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} + + isarray@1.0.0: + resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} + + jose@5.10.0: + resolution: {integrity: sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==} + + livekit-server-sdk@2.15.0: + resolution: {integrity: sha512-HmzjWnwEwwShu8yUf7VGFXdc+BuMJR5pnIY4qsdlhqI9d9wDgq+4cdTEHg0NEBaiGnc6PCOBiaTYgmIyVJ0S9w==} + engines: {node: '>=18'} + + lodash@4.17.23: + resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} + + map-obj@5.0.0: + resolution: {integrity: sha512-2L3MIgJynYrZ3TYMriLDLWocz15okFakV6J12HXvMXDHui2x/zgChzg1u9mFFGbbGWE+GsLpQByt4POb9Or+uA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + media-typer@1.1.0: + resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} + engines: {node: '>= 0.8'} + + merge-descriptors@2.0.0: + resolution: {integrity: sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==} + engines: {node: '>=18'} + + micromatch@4.0.8: + resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} + engines: {node: '>=8.6'} + + mime-db@1.54.0: + resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} + engines: {node: '>= 0.6'} + + mime-types@3.0.2: + resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} + engines: {node: '>=18'} + + minimatch@3.1.5: + resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + minipass@2.9.0: + resolution: {integrity: sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==} + + minizlib@1.3.3: + resolution: {integrity: sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==} + + mkdirp@0.5.6: + resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} + hasBin: true + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + nanoid@3.3.11: + resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + + needle@2.9.1: + resolution: {integrity: sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==} + engines: {node: '>= 4.4.x'} + hasBin: true + + negotiator@1.0.0: + resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} + engines: {node: '>= 0.6'} + + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + + node-pre-gyp@0.13.0: + resolution: {integrity: sha512-Md1D3xnEne8b/HGVQkZZwV27WUi1ZRuZBij24TNaZwUPU3ZAFtvT6xxJGaUVillfmMKnn5oD1HoGsp2Ftik7SQ==} + deprecated: 'Please upgrade to @mapbox/node-pre-gyp: the non-scoped node-pre-gyp package is deprecated and only the @mapbox scoped package will recieve updates in the future' + hasBin: true + + nopt@4.0.3: + resolution: {integrity: sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==} + hasBin: true + + npm-bundled@1.1.2: + resolution: {integrity: sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==} + + npm-normalize-package-bin@1.0.1: + resolution: {integrity: sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==} + + npm-packlist@1.4.8: + resolution: {integrity: sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==} + + npmlog@4.1.2: + resolution: {integrity: sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==} + deprecated: This package is no longer supported. + + number-is-nan@1.0.1: + resolution: {integrity: sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ==} + engines: {node: '>=0.10.0'} + + object-assign@4.1.1: + resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} + engines: {node: '>=0.10.0'} + + object-inspect@1.13.4: + resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} + engines: {node: '>= 0.4'} + + on-finished@2.4.1: + resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} + engines: {node: '>= 0.8'} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + os-homedir@1.0.2: + resolution: {integrity: sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==} + engines: {node: '>=0.10.0'} + + os-tmpdir@1.0.2: + resolution: {integrity: sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==} + engines: {node: '>=0.10.0'} + + osenv@0.1.5: + resolution: {integrity: sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==} + deprecated: This package is no longer supported. + + parseurl@1.3.3: + resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} + engines: {node: '>= 0.8'} + + path-is-absolute@1.0.1: + resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} + engines: {node: '>=0.10.0'} + + path-to-regexp@8.3.0: + resolution: {integrity: sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==} + + picocolors@1.1.1: + resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + + picomatch@2.3.1: + resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} + engines: {node: '>=8.6'} + + postcss@8.5.6: + resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} + engines: {node: ^10 || ^12 || >=14} + + process-nextick-args@2.0.1: + resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + + proxy-addr@2.0.7: + resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} + engines: {node: '>= 0.10'} + + qs@6.15.0: + resolution: {integrity: sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==} + engines: {node: '>=0.6'} + + quick-lru@6.1.2: + resolution: {integrity: sha512-AAFUA5O1d83pIHEhJwWCq/RQcRukCkn/NSm2QsTEMle5f2hP0ChI2+3Xb051PZCkLryI/Ir1MVKviT2FIloaTQ==} + engines: {node: '>=12'} + + range-parser@1.2.1: + resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} + engines: {node: '>= 0.6'} + + raw-body@3.0.2: + resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} + engines: {node: '>= 0.10'} + + rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + + readable-stream@2.3.8: + resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} + + require-directory@2.1.1: + resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} + engines: {node: '>=0.10.0'} + + resolve-from@5.0.0: + resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} + engines: {node: '>=8'} + + resolve-pkg-maps@1.0.0: + resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} + + rimraf@2.7.1: + resolution: {integrity: sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==} + deprecated: Rimraf versions prior to v4 are no longer supported + hasBin: true + + rollup-pluginutils@2.8.2: + resolution: {integrity: sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==} + + rollup@4.59.0: + resolution: {integrity: sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==} + engines: {node: '>=18.0.0', npm: '>=8.0.0'} + hasBin: true + + router@2.2.0: + resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} + engines: {node: '>= 18'} + + rxjs@7.8.2: + resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} + + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safer-buffer@2.1.2: + resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + + sax@1.4.4: + resolution: {integrity: sha512-1n3r/tGXO6b6VXMdFT54SHzT9ytu9yr7TaELowdYpMqY/Ao7EnlQGmAQ1+RatX7Tkkdm6hONI2owqNx2aZj5Sw==} + engines: {node: '>=11.0.0'} + + semver@5.7.2: + resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} + hasBin: true + + send@1.2.1: + resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} + engines: {node: '>= 18'} + + serve-static@2.2.1: + resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} + engines: {node: '>= 18'} + + set-blocking@2.0.0: + resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} + + setprototypeof@1.2.0: + resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} + + shell-quote@1.8.3: + resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==} + engines: {node: '>= 0.4'} + + side-channel-list@1.0.0: + resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} + engines: {node: '>= 0.4'} + + side-channel-map@1.0.1: + resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} + engines: {node: '>= 0.4'} + + side-channel-weakmap@1.0.2: + resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} + engines: {node: '>= 0.4'} + + side-channel@1.1.0: + resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} + engines: {node: '>= 0.4'} + + signal-exit@3.0.7: + resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + + source-map-js@1.2.1: + resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} + engines: {node: '>=0.10.0'} + + spawn-command@0.0.2: + resolution: {integrity: sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==} + + statuses@2.0.2: + resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} + engines: {node: '>= 0.8'} + + string-width@1.0.2: + resolution: {integrity: sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==} + engines: {node: '>=0.10.0'} + + string-width@4.2.3: + resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} + engines: {node: '>=8'} + + string_decoder@1.1.1: + resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} + + strip-ansi@3.0.1: + resolution: {integrity: sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==} + engines: {node: '>=0.10.0'} + + strip-ansi@6.0.1: + resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} + engines: {node: '>=8'} + + strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + supports-color@8.1.1: + resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} + engines: {node: '>=10'} + + tar@4.4.19: + resolution: {integrity: sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA==} + engines: {node: '>=4.5'} + deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me + + to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + + toidentifier@1.0.1: + resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} + engines: {node: '>=0.6'} + + tree-kill@1.2.2: + resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} + hasBin: true + + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + + tsx@4.21.0: + resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} + engines: {node: '>=18.0.0'} + hasBin: true + + type-fest@4.41.0: + resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} + engines: {node: '>=16'} + + type-is@2.0.1: + resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==} + engines: {node: '>= 0.6'} + + typescript@5.9.3: + resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} + engines: {node: '>=14.17'} + hasBin: true + + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + + unpipe@1.0.0: + resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} + engines: {node: '>= 0.8'} + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + vary@1.1.2: + resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} + engines: {node: '>= 0.8'} + + vite-plugin-mix@0.4.0: + resolution: {integrity: sha512-9X8hiwhl0RbtEXBB0XqnQ5suheAtP3VHn794WcWwjU5ziYYWdlqpMh/2J8APpx/YdpvQ2CZT7dlcGGd/31ya3w==} + peerDependencies: + vite: ^3 + + vite@5.4.21: + resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + peerDependencies: + '@types/node': ^18.0.0 || >=20.0.0 + less: '*' + lightningcss: ^1.21.0 + sass: '*' + sass-embedded: '*' + stylus: '*' + sugarss: '*' + terser: ^5.4.0 + peerDependenciesMeta: + '@types/node': + optional: true + less: + optional: true + lightningcss: + optional: true + sass: + optional: true + sass-embedded: + optional: true + stylus: + optional: true + sugarss: + optional: true + terser: + optional: true + + wide-align@1.1.5: + resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==} + + wrap-ansi@7.0.0: + resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} + engines: {node: '>=10'} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + + y18n@5.0.8: + resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} + engines: {node: '>=10'} + + yallist@3.1.1: + resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + + yargs@17.7.2: + resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} + engines: {node: '>=12'} + +snapshots: + + '@babel/runtime@7.28.6': {} + + '@bufbuild/protobuf@1.10.1': {} + + '@esbuild/aix-ppc64@0.21.5': + optional: true + + '@esbuild/aix-ppc64@0.27.3': + optional: true + + '@esbuild/android-arm64@0.21.5': + optional: true + + '@esbuild/android-arm64@0.27.3': + optional: true + + '@esbuild/android-arm@0.21.5': + optional: true + + '@esbuild/android-arm@0.27.3': + optional: true + + '@esbuild/android-x64@0.21.5': + optional: true + + '@esbuild/android-x64@0.27.3': + optional: true + + '@esbuild/darwin-arm64@0.21.5': + optional: true + + '@esbuild/darwin-arm64@0.27.3': + optional: true + + '@esbuild/darwin-x64@0.21.5': + optional: true + + '@esbuild/darwin-x64@0.27.3': + optional: true + + '@esbuild/freebsd-arm64@0.21.5': + optional: true + + '@esbuild/freebsd-arm64@0.27.3': + optional: true + + '@esbuild/freebsd-x64@0.21.5': + optional: true + + '@esbuild/freebsd-x64@0.27.3': + optional: true + + '@esbuild/linux-arm64@0.21.5': + optional: true + + '@esbuild/linux-arm64@0.27.3': + optional: true + + '@esbuild/linux-arm@0.21.5': + optional: true + + '@esbuild/linux-arm@0.27.3': + optional: true + + '@esbuild/linux-ia32@0.21.5': + optional: true + + '@esbuild/linux-ia32@0.27.3': + optional: true + + '@esbuild/linux-loong64@0.21.5': + optional: true + + '@esbuild/linux-loong64@0.27.3': + optional: true + + '@esbuild/linux-mips64el@0.21.5': + optional: true + + '@esbuild/linux-mips64el@0.27.3': + optional: true + + '@esbuild/linux-ppc64@0.21.5': + optional: true + + '@esbuild/linux-ppc64@0.27.3': + optional: true + + '@esbuild/linux-riscv64@0.21.5': + optional: true + + '@esbuild/linux-riscv64@0.27.3': + optional: true + + '@esbuild/linux-s390x@0.21.5': + optional: true + + '@esbuild/linux-s390x@0.27.3': + optional: true + + '@esbuild/linux-x64@0.21.5': + optional: true + + '@esbuild/linux-x64@0.27.3': + optional: true + + '@esbuild/netbsd-arm64@0.27.3': + optional: true + + '@esbuild/netbsd-x64@0.21.5': + optional: true + + '@esbuild/netbsd-x64@0.27.3': + optional: true + + '@esbuild/openbsd-arm64@0.27.3': + optional: true + + '@esbuild/openbsd-x64@0.21.5': + optional: true + + '@esbuild/openbsd-x64@0.27.3': + optional: true + + '@esbuild/openharmony-arm64@0.27.3': + optional: true + + '@esbuild/sunos-x64@0.21.5': + optional: true + + '@esbuild/sunos-x64@0.27.3': + optional: true + + '@esbuild/win32-arm64@0.21.5': + optional: true + + '@esbuild/win32-arm64@0.27.3': + optional: true + + '@esbuild/win32-ia32@0.21.5': + optional: true + + '@esbuild/win32-ia32@0.27.3': + optional: true + + '@esbuild/win32-x64@0.21.5': + optional: true + + '@esbuild/win32-x64@0.27.3': + optional: true + + '@livekit/protocol@1.44.0': + dependencies: + '@bufbuild/protobuf': 1.10.1 + + '@rollup/rollup-android-arm-eabi@4.59.0': + optional: true + + '@rollup/rollup-android-arm64@4.59.0': + optional: true + + '@rollup/rollup-darwin-arm64@4.59.0': + optional: true + + '@rollup/rollup-darwin-x64@4.59.0': + optional: true + + '@rollup/rollup-freebsd-arm64@4.59.0': + optional: true + + '@rollup/rollup-freebsd-x64@4.59.0': + optional: true + + '@rollup/rollup-linux-arm-gnueabihf@4.59.0': + optional: true + + '@rollup/rollup-linux-arm-musleabihf@4.59.0': + optional: true + + '@rollup/rollup-linux-arm64-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-arm64-musl@4.59.0': + optional: true + + '@rollup/rollup-linux-loong64-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-loong64-musl@4.59.0': + optional: true + + '@rollup/rollup-linux-ppc64-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-ppc64-musl@4.59.0': + optional: true + + '@rollup/rollup-linux-riscv64-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-riscv64-musl@4.59.0': + optional: true + + '@rollup/rollup-linux-s390x-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-x64-gnu@4.59.0': + optional: true + + '@rollup/rollup-linux-x64-musl@4.59.0': + optional: true + + '@rollup/rollup-openbsd-x64@4.59.0': + optional: true + + '@rollup/rollup-openharmony-arm64@4.59.0': + optional: true + + '@rollup/rollup-win32-arm64-msvc@4.59.0': + optional: true + + '@rollup/rollup-win32-ia32-msvc@4.59.0': + optional: true + + '@rollup/rollup-win32-x64-gnu@4.59.0': + optional: true + + '@rollup/rollup-win32-x64-msvc@4.59.0': + optional: true + + '@types/body-parser@1.19.6': + dependencies: + '@types/connect': 3.4.38 + '@types/node': 20.19.41 + + '@types/connect@3.4.38': + dependencies: + '@types/node': 20.19.41 + + '@types/cors@2.8.19': + dependencies: + '@types/node': 20.19.41 + + '@types/estree@1.0.8': {} + + '@types/express-serve-static-core@5.1.1': + dependencies: + '@types/node': 20.19.41 + '@types/qs': 6.14.0 + '@types/range-parser': 1.2.7 + '@types/send': 1.2.1 + + '@types/express@5.0.6': + dependencies: + '@types/body-parser': 1.19.6 + '@types/express-serve-static-core': 5.1.1 + '@types/serve-static': 2.2.0 + + '@types/http-errors@2.0.5': {} + + '@types/node@20.19.41': + dependencies: + undici-types: 6.21.0 + + '@types/qs@6.14.0': {} + + '@types/range-parser@1.2.7': {} + + '@types/send@1.2.1': + dependencies: + '@types/node': 20.19.41 + + '@types/serve-static@2.2.0': + dependencies: + '@types/http-errors': 2.0.5 + '@types/node': 20.19.41 + + '@vercel/nft@0.10.1': + dependencies: + acorn: 8.16.0 + acorn-class-fields: 1.0.0(acorn@8.16.0) + acorn-static-class-features: 1.0.0(acorn@8.16.0) + bindings: 1.5.0 + estree-walker: 0.6.1 + glob: 7.2.3 + graceful-fs: 4.2.11 + micromatch: 4.0.8 + mkdirp: 0.5.6 + node-gyp-build: 4.8.4 + node-pre-gyp: 0.13.0 + resolve-from: 5.0.0 + rollup-pluginutils: 2.8.2 + transitivePeerDependencies: + - supports-color + + abbrev@1.1.1: {} + + accepts@2.0.0: + dependencies: + mime-types: 3.0.2 + negotiator: 1.0.0 + + acorn-class-fields@1.0.0(acorn@8.16.0): + dependencies: + acorn: 8.16.0 + acorn-private-class-elements: 1.0.0(acorn@8.16.0) + + acorn-private-class-elements@1.0.0(acorn@8.16.0): + dependencies: + acorn: 8.16.0 + + acorn-static-class-features@1.0.0(acorn@8.16.0): + dependencies: + acorn: 8.16.0 + acorn-private-class-elements: 1.0.0(acorn@8.16.0) + + acorn@8.16.0: {} + + ansi-regex@2.1.1: {} + + ansi-regex@5.0.1: {} + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + aproba@1.2.0: {} + + are-we-there-yet@1.1.7: + dependencies: + delegates: 1.0.0 + readable-stream: 2.3.8 + + balanced-match@1.0.2: {} + + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + + body-parser@2.2.2: + dependencies: + bytes: 3.1.2 + content-type: 1.0.5 + debug: 4.4.3 + http-errors: 2.0.1 + iconv-lite: 0.7.2 + on-finished: 2.4.1 + qs: 6.15.0 + raw-body: 3.0.2 + type-is: 2.0.1 + transitivePeerDependencies: + - supports-color + + brace-expansion@1.1.12: + dependencies: + balanced-match: 1.0.2 + concat-map: 0.0.1 + + braces@3.0.3: + dependencies: + fill-range: 7.1.1 + + bytes@3.1.2: {} + + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + + call-bound@1.0.4: + dependencies: + call-bind-apply-helpers: 1.0.2 + get-intrinsic: 1.3.0 + + camelcase-keys@9.1.3: + dependencies: + camelcase: 8.0.0 + map-obj: 5.0.0 + quick-lru: 6.1.2 + type-fest: 4.41.0 + + camelcase@8.0.0: {} + + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + chownr@1.1.4: {} + + cliui@8.0.1: + dependencies: + string-width: 4.2.3 + strip-ansi: 6.0.1 + wrap-ansi: 7.0.0 + + code-point-at@1.1.0: {} + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + concat-map@0.0.1: {} + + concurrently@8.2.2: + dependencies: + chalk: 4.1.2 + date-fns: 2.30.0 + lodash: 4.17.23 + rxjs: 7.8.2 + shell-quote: 1.8.3 + spawn-command: 0.0.2 + supports-color: 8.1.1 + tree-kill: 1.2.2 + yargs: 17.7.2 + + console-control-strings@1.1.0: {} + + content-disposition@1.0.1: {} + + content-type@1.0.5: {} + + cookie-signature@1.2.2: {} + + cookie@0.7.2: {} + + core-util-is@1.0.3: {} + + cors@2.8.6: + dependencies: + object-assign: 4.1.1 + vary: 1.1.2 + + date-fns@2.30.0: + dependencies: + '@babel/runtime': 7.28.6 + + debug@3.2.7: + dependencies: + ms: 2.1.3 + + debug@4.4.3: + dependencies: + ms: 2.1.3 + + deep-extend@0.6.0: {} + + delegates@1.0.0: {} + + depd@2.0.0: {} + + detect-libc@1.0.3: {} + + dotenv@16.6.1: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + ee-first@1.1.1: {} + + emoji-regex@8.0.0: {} + + encodeurl@2.0.0: {} + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + + es-object-atoms@1.1.1: + dependencies: + es-errors: 1.3.0 + + esbuild@0.21.5: + optionalDependencies: + '@esbuild/aix-ppc64': 0.21.5 + '@esbuild/android-arm': 0.21.5 + '@esbuild/android-arm64': 0.21.5 + '@esbuild/android-x64': 0.21.5 + '@esbuild/darwin-arm64': 0.21.5 + '@esbuild/darwin-x64': 0.21.5 + '@esbuild/freebsd-arm64': 0.21.5 + '@esbuild/freebsd-x64': 0.21.5 + '@esbuild/linux-arm': 0.21.5 + '@esbuild/linux-arm64': 0.21.5 + '@esbuild/linux-ia32': 0.21.5 + '@esbuild/linux-loong64': 0.21.5 + '@esbuild/linux-mips64el': 0.21.5 + '@esbuild/linux-ppc64': 0.21.5 + '@esbuild/linux-riscv64': 0.21.5 + '@esbuild/linux-s390x': 0.21.5 + '@esbuild/linux-x64': 0.21.5 + '@esbuild/netbsd-x64': 0.21.5 + '@esbuild/openbsd-x64': 0.21.5 + '@esbuild/sunos-x64': 0.21.5 + '@esbuild/win32-arm64': 0.21.5 + '@esbuild/win32-ia32': 0.21.5 + '@esbuild/win32-x64': 0.21.5 + + esbuild@0.27.3: + optionalDependencies: + '@esbuild/aix-ppc64': 0.27.3 + '@esbuild/android-arm': 0.27.3 + '@esbuild/android-arm64': 0.27.3 + '@esbuild/android-x64': 0.27.3 + '@esbuild/darwin-arm64': 0.27.3 + '@esbuild/darwin-x64': 0.27.3 + '@esbuild/freebsd-arm64': 0.27.3 + '@esbuild/freebsd-x64': 0.27.3 + '@esbuild/linux-arm': 0.27.3 + '@esbuild/linux-arm64': 0.27.3 + '@esbuild/linux-ia32': 0.27.3 + '@esbuild/linux-loong64': 0.27.3 + '@esbuild/linux-mips64el': 0.27.3 + '@esbuild/linux-ppc64': 0.27.3 + '@esbuild/linux-riscv64': 0.27.3 + '@esbuild/linux-s390x': 0.27.3 + '@esbuild/linux-x64': 0.27.3 + '@esbuild/netbsd-arm64': 0.27.3 + '@esbuild/netbsd-x64': 0.27.3 + '@esbuild/openbsd-arm64': 0.27.3 + '@esbuild/openbsd-x64': 0.27.3 + '@esbuild/openharmony-arm64': 0.27.3 + '@esbuild/sunos-x64': 0.27.3 + '@esbuild/win32-arm64': 0.27.3 + '@esbuild/win32-ia32': 0.27.3 + '@esbuild/win32-x64': 0.27.3 + + escalade@3.2.0: {} + + escape-html@1.0.3: {} + + estree-walker@0.6.1: {} + + etag@1.8.1: {} + + express@5.2.1: + dependencies: + accepts: 2.0.0 + body-parser: 2.2.2 + content-disposition: 1.0.1 + content-type: 1.0.5 + cookie: 0.7.2 + cookie-signature: 1.2.2 + debug: 4.4.3 + depd: 2.0.0 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + finalhandler: 2.1.1 + fresh: 2.0.0 + http-errors: 2.0.1 + merge-descriptors: 2.0.0 + mime-types: 3.0.2 + on-finished: 2.4.1 + once: 1.4.0 + parseurl: 1.3.3 + proxy-addr: 2.0.7 + qs: 6.15.0 + range-parser: 1.2.1 + router: 2.2.0 + send: 1.2.1 + serve-static: 2.2.1 + statuses: 2.0.2 + type-is: 2.0.1 + vary: 1.1.2 + transitivePeerDependencies: + - supports-color + + file-uri-to-path@1.0.0: {} + + fill-range@7.1.1: + dependencies: + to-regex-range: 5.0.1 + + finalhandler@2.1.1: + dependencies: + debug: 4.4.3 + encodeurl: 2.0.0 + escape-html: 1.0.3 + on-finished: 2.4.1 + parseurl: 1.3.3 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + + forwarded@0.2.0: {} + + fresh@2.0.0: {} + + fs-minipass@1.2.7: + dependencies: + minipass: 2.9.0 + + fs.realpath@1.0.0: {} + + fsevents@2.3.3: + optional: true + + function-bind@1.1.2: {} + + gauge@2.7.4: + dependencies: + aproba: 1.2.0 + console-control-strings: 1.1.0 + has-unicode: 2.0.1 + object-assign: 4.1.1 + signal-exit: 3.0.7 + string-width: 1.0.2 + strip-ansi: 3.0.1 + wide-align: 1.1.5 + + get-caller-file@2.0.5: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.1 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.2 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.1 + + get-tsconfig@4.13.6: + dependencies: + resolve-pkg-maps: 1.0.0 + + glob@7.2.3: + dependencies: + fs.realpath: 1.0.0 + inflight: 1.0.6 + inherits: 2.0.4 + minimatch: 3.1.5 + once: 1.4.0 + path-is-absolute: 1.0.1 + + gopd@1.2.0: {} + + graceful-fs@4.2.11: {} + + has-flag@4.0.0: {} + + has-symbols@1.1.0: {} + + has-unicode@2.0.1: {} + + hasown@2.0.2: + dependencies: + function-bind: 1.1.2 + + http-errors@2.0.1: + dependencies: + depd: 2.0.0 + inherits: 2.0.4 + setprototypeof: 1.2.0 + statuses: 2.0.2 + toidentifier: 1.0.1 + + iconv-lite@0.4.24: + dependencies: + safer-buffer: 2.1.2 + + iconv-lite@0.7.2: + dependencies: + safer-buffer: 2.1.2 + + ignore-walk@3.0.4: + dependencies: + minimatch: 3.1.5 + + inflight@1.0.6: + dependencies: + once: 1.4.0 + wrappy: 1.0.2 + + inherits@2.0.4: {} + + ini@1.3.8: {} + + ipaddr.js@1.9.1: {} + + is-fullwidth-code-point@1.0.0: + dependencies: + number-is-nan: 1.0.1 + + is-fullwidth-code-point@3.0.0: {} + + is-number@7.0.0: {} + + is-promise@4.0.0: {} + + isarray@1.0.0: {} + + jose@5.10.0: {} + + livekit-server-sdk@2.15.0: + dependencies: + '@bufbuild/protobuf': 1.10.1 + '@livekit/protocol': 1.44.0 + camelcase-keys: 9.1.3 + jose: 5.10.0 + + lodash@4.17.23: {} + + map-obj@5.0.0: {} + + math-intrinsics@1.1.0: {} + + media-typer@1.1.0: {} + + merge-descriptors@2.0.0: {} + + micromatch@4.0.8: + dependencies: + braces: 3.0.3 + picomatch: 2.3.1 + + mime-db@1.54.0: {} + + mime-types@3.0.2: + dependencies: + mime-db: 1.54.0 + + minimatch@3.1.5: + dependencies: + brace-expansion: 1.1.12 + + minimist@1.2.8: {} + + minipass@2.9.0: + dependencies: + safe-buffer: 5.2.1 + yallist: 3.1.1 + + minizlib@1.3.3: + dependencies: + minipass: 2.9.0 + + mkdirp@0.5.6: + dependencies: + minimist: 1.2.8 + + ms@2.1.3: {} + + nanoid@3.3.11: {} + + needle@2.9.1: + dependencies: + debug: 3.2.7 + iconv-lite: 0.4.24 + sax: 1.4.4 + transitivePeerDependencies: + - supports-color + + negotiator@1.0.0: {} + + node-gyp-build@4.8.4: {} + + node-pre-gyp@0.13.0: + dependencies: + detect-libc: 1.0.3 + mkdirp: 0.5.6 + needle: 2.9.1 + nopt: 4.0.3 + npm-packlist: 1.4.8 + npmlog: 4.1.2 + rc: 1.2.8 + rimraf: 2.7.1 + semver: 5.7.2 + tar: 4.4.19 + transitivePeerDependencies: + - supports-color + + nopt@4.0.3: + dependencies: + abbrev: 1.1.1 + osenv: 0.1.5 + + npm-bundled@1.1.2: + dependencies: + npm-normalize-package-bin: 1.0.1 + + npm-normalize-package-bin@1.0.1: {} + + npm-packlist@1.4.8: + dependencies: + ignore-walk: 3.0.4 + npm-bundled: 1.1.2 + npm-normalize-package-bin: 1.0.1 + + npmlog@4.1.2: + dependencies: + are-we-there-yet: 1.1.7 + console-control-strings: 1.1.0 + gauge: 2.7.4 + set-blocking: 2.0.0 + + number-is-nan@1.0.1: {} + + object-assign@4.1.1: {} + + object-inspect@1.13.4: {} + + on-finished@2.4.1: + dependencies: + ee-first: 1.1.1 + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + os-homedir@1.0.2: {} + + os-tmpdir@1.0.2: {} + + osenv@0.1.5: + dependencies: + os-homedir: 1.0.2 + os-tmpdir: 1.0.2 + + parseurl@1.3.3: {} + + path-is-absolute@1.0.1: {} + + path-to-regexp@8.3.0: {} + + picocolors@1.1.1: {} + + picomatch@2.3.1: {} + + postcss@8.5.6: + dependencies: + nanoid: 3.3.11 + picocolors: 1.1.1 + source-map-js: 1.2.1 + + process-nextick-args@2.0.1: {} + + proxy-addr@2.0.7: + dependencies: + forwarded: 0.2.0 + ipaddr.js: 1.9.1 + + qs@6.15.0: + dependencies: + side-channel: 1.1.0 + + quick-lru@6.1.2: {} + + range-parser@1.2.1: {} + + raw-body@3.0.2: + dependencies: + bytes: 3.1.2 + http-errors: 2.0.1 + iconv-lite: 0.7.2 + unpipe: 1.0.0 + + rc@1.2.8: + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + + readable-stream@2.3.8: + dependencies: + core-util-is: 1.0.3 + inherits: 2.0.4 + isarray: 1.0.0 + process-nextick-args: 2.0.1 + safe-buffer: 5.1.2 + string_decoder: 1.1.1 + util-deprecate: 1.0.2 + + require-directory@2.1.1: {} + + resolve-from@5.0.0: {} + + resolve-pkg-maps@1.0.0: {} + + rimraf@2.7.1: + dependencies: + glob: 7.2.3 + + rollup-pluginutils@2.8.2: + dependencies: + estree-walker: 0.6.1 + + rollup@4.59.0: + dependencies: + '@types/estree': 1.0.8 + optionalDependencies: + '@rollup/rollup-android-arm-eabi': 4.59.0 + '@rollup/rollup-android-arm64': 4.59.0 + '@rollup/rollup-darwin-arm64': 4.59.0 + '@rollup/rollup-darwin-x64': 4.59.0 + '@rollup/rollup-freebsd-arm64': 4.59.0 + '@rollup/rollup-freebsd-x64': 4.59.0 + '@rollup/rollup-linux-arm-gnueabihf': 4.59.0 + '@rollup/rollup-linux-arm-musleabihf': 4.59.0 + '@rollup/rollup-linux-arm64-gnu': 4.59.0 + '@rollup/rollup-linux-arm64-musl': 4.59.0 + '@rollup/rollup-linux-loong64-gnu': 4.59.0 + '@rollup/rollup-linux-loong64-musl': 4.59.0 + '@rollup/rollup-linux-ppc64-gnu': 4.59.0 + '@rollup/rollup-linux-ppc64-musl': 4.59.0 + '@rollup/rollup-linux-riscv64-gnu': 4.59.0 + '@rollup/rollup-linux-riscv64-musl': 4.59.0 + '@rollup/rollup-linux-s390x-gnu': 4.59.0 + '@rollup/rollup-linux-x64-gnu': 4.59.0 + '@rollup/rollup-linux-x64-musl': 4.59.0 + '@rollup/rollup-openbsd-x64': 4.59.0 + '@rollup/rollup-openharmony-arm64': 4.59.0 + '@rollup/rollup-win32-arm64-msvc': 4.59.0 + '@rollup/rollup-win32-ia32-msvc': 4.59.0 + '@rollup/rollup-win32-x64-gnu': 4.59.0 + '@rollup/rollup-win32-x64-msvc': 4.59.0 + fsevents: 2.3.3 + + router@2.2.0: + dependencies: + debug: 4.4.3 + depd: 2.0.0 + is-promise: 4.0.0 + parseurl: 1.3.3 + path-to-regexp: 8.3.0 + transitivePeerDependencies: + - supports-color + + rxjs@7.8.2: + dependencies: + tslib: 2.8.1 + + safe-buffer@5.1.2: {} + + safe-buffer@5.2.1: {} + + safer-buffer@2.1.2: {} + + sax@1.4.4: {} + + semver@5.7.2: {} + + send@1.2.1: + dependencies: + debug: 4.4.3 + encodeurl: 2.0.0 + escape-html: 1.0.3 + etag: 1.8.1 + fresh: 2.0.0 + http-errors: 2.0.1 + mime-types: 3.0.2 + ms: 2.1.3 + on-finished: 2.4.1 + range-parser: 1.2.1 + statuses: 2.0.2 + transitivePeerDependencies: + - supports-color + + serve-static@2.2.1: + dependencies: + encodeurl: 2.0.0 + escape-html: 1.0.3 + parseurl: 1.3.3 + send: 1.2.1 + transitivePeerDependencies: + - supports-color + + set-blocking@2.0.0: {} + + setprototypeof@1.2.0: {} + + shell-quote@1.8.3: {} + + side-channel-list@1.0.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + + side-channel-map@1.0.1: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + + side-channel-weakmap@1.0.2: + dependencies: + call-bound: 1.0.4 + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + object-inspect: 1.13.4 + side-channel-map: 1.0.1 + + side-channel@1.1.0: + dependencies: + es-errors: 1.3.0 + object-inspect: 1.13.4 + side-channel-list: 1.0.0 + side-channel-map: 1.0.1 + side-channel-weakmap: 1.0.2 + + signal-exit@3.0.7: {} + + source-map-js@1.2.1: {} + + spawn-command@0.0.2: {} + + statuses@2.0.2: {} + + string-width@1.0.2: + dependencies: + code-point-at: 1.1.0 + is-fullwidth-code-point: 1.0.0 + strip-ansi: 3.0.1 + + string-width@4.2.3: + dependencies: + emoji-regex: 8.0.0 + is-fullwidth-code-point: 3.0.0 + strip-ansi: 6.0.1 + + string_decoder@1.1.1: + dependencies: + safe-buffer: 5.1.2 + + strip-ansi@3.0.1: + dependencies: + ansi-regex: 2.1.1 + + strip-ansi@6.0.1: + dependencies: + ansi-regex: 5.0.1 + + strip-json-comments@2.0.1: {} + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + supports-color@8.1.1: + dependencies: + has-flag: 4.0.0 + + tar@4.4.19: + dependencies: + chownr: 1.1.4 + fs-minipass: 1.2.7 + minipass: 2.9.0 + minizlib: 1.3.3 + mkdirp: 0.5.6 + safe-buffer: 5.2.1 + yallist: 3.1.1 + + to-regex-range@5.0.1: + dependencies: + is-number: 7.0.0 + + toidentifier@1.0.1: {} + + tree-kill@1.2.2: {} + + tslib@2.8.1: {} + + tsx@4.21.0: + dependencies: + esbuild: 0.27.3 + get-tsconfig: 4.13.6 + optionalDependencies: + fsevents: 2.3.3 + + type-fest@4.41.0: {} + + type-is@2.0.1: + dependencies: + content-type: 1.0.5 + media-typer: 1.1.0 + mime-types: 3.0.2 + + typescript@5.9.3: {} + + undici-types@6.21.0: {} + + unpipe@1.0.0: {} + + util-deprecate@1.0.2: {} + + vary@1.1.2: {} + + vite-plugin-mix@0.4.0(vite@5.4.21(@types/node@20.19.41)): + dependencies: + '@vercel/nft': 0.10.1 + vite: 5.4.21(@types/node@20.19.41) + transitivePeerDependencies: + - supports-color + + vite@5.4.21(@types/node@20.19.41): + dependencies: + esbuild: 0.21.5 + postcss: 8.5.6 + rollup: 4.59.0 + optionalDependencies: + '@types/node': 20.19.41 + fsevents: 2.3.3 + + wide-align@1.1.5: + dependencies: + string-width: 1.0.2 + + wrap-ansi@7.0.0: + dependencies: + ansi-styles: 4.3.0 + string-width: 4.2.3 + strip-ansi: 6.0.1 + + wrappy@1.0.2: {} + + y18n@5.0.8: {} + + yallist@3.1.1: {} + + yargs-parser@21.1.1: {} + + yargs@17.7.2: + dependencies: + cliui: 8.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + require-directory: 2.1.1 + string-width: 4.2.3 + y18n: 5.0.8 + yargs-parser: 21.1.1 diff --git a/examples/data-stream-benchmark/pnpm-workspace.yaml b/examples/data-stream-benchmark/pnpm-workspace.yaml new file mode 100644 index 0000000000..7057c6516c --- /dev/null +++ b/examples/data-stream-benchmark/pnpm-workspace.yaml @@ -0,0 +1,8 @@ +# This example installs standalone (its own pnpm workspace root), separate from the repo-root +# workspace. Some pinned (dev) deps trip pnpm's supply-chain trust-downgrade check +# (ERR_PNPM_TRUST_DOWNGRADE) under a strict global `trustPolicy: no-downgrade`. Exclude those +# specific packages here so `pnpm install` / `pnpm dev` work in this directory without changing +# global or repo-root config, while keeping the trust check active for everything else. +trustPolicyExclude: + - 'vite@5.4.21' + - 'undici-types@6.21.0' diff --git a/examples/data-stream-benchmark/styles.css b/examples/data-stream-benchmark/styles.css new file mode 100644 index 0000000000..d25e9056d2 --- /dev/null +++ b/examples/data-stream-benchmark/styles.css @@ -0,0 +1,148 @@ +body { + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, + 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; + background-color: #f0f2f5; + color: #333; + line-height: 1.6; + margin: 0; + padding: 0; +} + +.container { + max-width: 1600px; + margin: 40px auto; + padding: 30px; + background-color: #ffffff; + box-shadow: 0 0 20px rgba(0, 0, 0, 0.1); + border-radius: 12px; +} + +h1 { + text-align: center; + color: #2c3e50; + margin-bottom: 24px; + font-weight: 600; +} + +.controls { + display: flex; + gap: 12px; + justify-content: center; + margin-bottom: 16px; +} + +.status { + text-align: center; + font-weight: 500; + color: #555; + margin-bottom: 20px; + min-height: 1.5em; +} + +.grid { + width: 100%; + border-collapse: collapse; + font-variant-numeric: tabular-nums; +} + +.grid th, +.grid td { + border: 1px solid #dce1e6; + padding: 10px 8px; + text-align: center; + font-size: 14px; +} + +.grid thead th { + background-color: #2c3e50; + color: #fff; + font-weight: 600; +} + +.grid th.row-head { + background-color: #f1f4f7; + font-weight: 600; + white-space: nowrap; +} + +.cell { + color: #2c3e50; +} + +.cell .recv { + font-size: 16px; + font-weight: 600; + font-variant-numeric: tabular-nums; +} + +.cell .status { + margin-top: 2px; + font-size: 12px; + line-height: 1.35; + color: #34495e; + font-variant-numeric: tabular-nums; + white-space: nowrap; +} + +.cell .status .bad { + color: #c0392b; + font-weight: 600; +} + +.cell.running .recv { + color: #2980b9; + font-style: italic; + font-weight: 400; +} + +.hint { + font-size: 13px; + color: #7f8c8d; + margin: 16px 0; +} + +#log-area { + margin-top: 10px; +} + +#log { + box-sizing: border-box; + width: 100%; + height: 240px; + padding: 10px; + border: 1px solid #ddd; + border-radius: 4px; + font-family: monospace; + font-size: 13px; + resize: vertical; +} + +.btn { + padding: 10px 20px; + background-color: #3498db; + color: white; + border: none; + border-radius: 5px; + font-size: 15px; + cursor: pointer; + transition: + background-color 0.3s, + transform 0.1s; + font-weight: 500; +} + +.btn:hover { + background-color: #2980b9; + transform: translateY(-2px); +} + +.btn:active { + transform: translateY(0); +} + +.btn:disabled { + background-color: #bdc3c7; + color: #7f8c8d; + cursor: not-allowed; + transform: none; +} diff --git a/examples/data-stream-benchmark/tsconfig.json b/examples/data-stream-benchmark/tsconfig.json new file mode 100644 index 0000000000..ad5b077b7e --- /dev/null +++ b/examples/data-stream-benchmark/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, + "module": "ESNext" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, + "outDir": "build", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "strict": true /* Enable all strict type-checking options. */, + "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, + "skipLibCheck": true /* Skip type checking of declaration files. */, + "noUnusedLocals": true, + "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */, + "moduleResolution": "node", + "resolveJsonModule": true + }, + "include": ["../../src/**/*", "benchmark.ts", "payload.ts", "api.ts"], + "exclude": ["**/*.test.ts", "build/**/*"] +} diff --git a/examples/data-stream-benchmark/vite.config.js b/examples/data-stream-benchmark/vite.config.js new file mode 100644 index 0000000000..8f82d19f31 --- /dev/null +++ b/examples/data-stream-benchmark/vite.config.js @@ -0,0 +1,10 @@ +import { defineConfig } from 'vite'; +import mix from 'vite-plugin-mix'; + +export default defineConfig({ + plugins: [ + mix.default({ + handler: './api.ts', + }), + ], +}); From 578dbd86e5cf8ddf8be1d6a7fb32e6df90896afd Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 5 Jun 2026 15:43:18 -0400 Subject: [PATCH 08/68] feat: add missing constants file --- src/room/data-stream/constants.ts | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 src/room/data-stream/constants.ts diff --git a/src/room/data-stream/constants.ts b/src/room/data-stream/constants.ts new file mode 100644 index 0000000000..d78d70cf66 --- /dev/null +++ b/src/room/data-stream/constants.ts @@ -0,0 +1,29 @@ +/** + * Reserved data-stream header attribute used to "smuggle" a small payload directly inside a single + * `streamHeader` packet, avoiding the separate `streamChunk`/`streamTrailer` packets a normal data + * stream requires. For text streams the value is the raw string; for byte streams it is base64. + * + * @internal + */ +export const INLINE_PAYLOAD_ATTRIBUTE = 'lk.inline_payload'; + +/** + * Maximum size of a single data-stream chunk in bytes, and the budget used to decide whether a + * payload can be sent inline as a single header packet. Kept below the ~16k data-channel MTU to + * leave headroom for protocol framing and E2EE overhead. + * + * @internal + */ +export const STREAM_CHUNK_SIZE_BYTES = 15_000; + +/** + * Reserved data-stream header attribute signaling that the payload (inline or chunked) is + * compressed. Self-describing: the sender sets it when it compresses, and the receiver decompresses + * iff it is present. The only supported value today is {@link COMPRESSION_GZIP}. + * + * @internal + */ +export const COMPRESSION_ATTRIBUTE = 'lk.compression'; + +/** Value of {@link COMPRESSION_ATTRIBUTE} for gzip-compressed payloads. @internal */ +export const COMPRESSION_GZIP = 'gzip'; From 7c35a52a8f1066e7dc52e709ecc5a521dc04f86c Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 5 Jun 2026 15:43:51 -0400 Subject: [PATCH 09/68] feat: add initial data stream compression implementation --- .../data-stream/compression-roundtrip.test.ts | 145 ++++++++ src/room/data-stream/compression.test.ts | 74 +++++ src/room/data-stream/compression.ts | 87 +++++ .../incoming/IncomingDataStreamManager.ts | 134 +++++++- .../outgoing/OutgoingDataStreamManager.ts | 314 ++++++++++++------ src/room/types.ts | 2 + 6 files changed, 650 insertions(+), 106 deletions(-) create mode 100644 src/room/data-stream/compression-roundtrip.test.ts create mode 100644 src/room/data-stream/compression.test.ts create mode 100644 src/room/data-stream/compression.ts diff --git a/src/room/data-stream/compression-roundtrip.test.ts b/src/room/data-stream/compression-roundtrip.test.ts new file mode 100644 index 0000000000..5c95cbf6d1 --- /dev/null +++ b/src/room/data-stream/compression-roundtrip.test.ts @@ -0,0 +1,145 @@ +import { type DataPacket, Encryption_Type } from '@livekit/protocol'; +import { describe, expect, it, vi } from 'vitest'; +import log from '../../logger'; +import { CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DATA_STREAM_V2 } from '../../version'; +import type RTCEngine from '../RTCEngine'; +import IncomingDataStreamManager from './incoming/IncomingDataStreamManager'; +import type { ByteStreamReader, TextStreamReader } from './incoming/StreamReader'; +import OutgoingDataStreamManager from './outgoing/OutgoingDataStreamManager'; + +const RECEIVER = 'bob'; +const hasCompression = typeof CompressionStream !== 'undefined'; + +/** An OutgoingDataStreamManager whose engine captures every sent packet. */ +function createSender(recipientProtocol = CLIENT_PROTOCOL_DATA_STREAM_V2) { + const sentPackets: DataPacket[] = []; + const engine = { + sendDataPacket: vi.fn(async (packet: DataPacket) => { + sentPackets.push(packet); + }), + e2eeManager: undefined, + once: vi.fn(), + off: vi.fn(), + } as unknown as RTCEngine; + const manager = new OutgoingDataStreamManager( + engine, + log, + () => recipientProtocol, + () => [RECEIVER], + ); + return { manager, sentPackets }; +} + +/** Replays captured outgoing packets into a receiver and returns the resulting text. */ +async function receiveText(packets: DataPacket[], topic: string): Promise { + const incoming = new IncomingDataStreamManager(); + incoming.setConnected(true); + const readerPromise = new Promise((resolve) => { + incoming.registerTextStreamHandler(topic, (reader) => resolve(reader)); + }); + for (const packet of packets) { + incoming.handleDataStreamPacket(packet, Encryption_Type.NONE); + } + return readerPromise; +} + +async function receiveBytes(packets: DataPacket[], topic: string): Promise { + const incoming = new IncomingDataStreamManager(); + incoming.setConnected(true); + const readerPromise = new Promise((resolve) => { + incoming.registerByteStreamHandler(topic, (reader) => resolve(reader)); + }); + for (const packet of packets) { + incoming.handleDataStreamPacket(packet, Encryption_Type.NONE); + } + return readerPromise; +} + +function flatten(chunks: Array): Uint8Array { + const total = chunks.reduce((n, c) => n + c.byteLength, 0); + const out = new Uint8Array(total); + let offset = 0; + for (const c of chunks) { + out.set(c, offset); + offset += c.byteLength; + } + return out; +} + +describe.skipIf(!hasCompression)('data stream compression round-trip', () => { + it('round-trips a small compressible inline text payload', async () => { + const { manager, sentPackets } = createSender(); + const payload = 'compress me '.repeat(50); // > raw, compresses well, fits inline + + await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); + + expect(sentPackets).toHaveLength(1); // single inline header packet + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe(payload); + }); + + it('round-trips a large chunked compressed text payload (multi-packet)', async () => { + const { manager, sentPackets } = createSender(); + // High entropy → too big to inline even compressed → chunked + compressed. + let payload = ''; + while (payload.length < 60_000) { + payload += Math.random().toString(36).slice(2); + } + + await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); + + expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe(payload); + }); + + it('round-trips chunked compressed text with multibyte UTF-8 (reframing on char boundaries)', async () => { + const { manager, sentPackets } = createSender(); + // Emoji + CJK so gzip/output chunk boundaries fall mid-character. + const payload = '日本語🚀テスト '.repeat(4_000); + + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); + await writer.write(payload); + await writer.close(); + + expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe(payload); + }); + + it('round-trips a chunked compressed byte stream', async () => { + const { manager, sentPackets } = createSender(); + const payload = new Uint8Array(50_000); + for (let i = 0; i < payload.length; i += 1) { + payload[i] = i % 256; + } + + const writer = await manager.streamBytes({ topic: 'b', destinationIdentities: [RECEIVER] }); + await writer.write(payload); + await writer.close(); + + expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); + const reader = await receiveBytes(sentPackets, 'b'); + expect(Array.from(flatten(await reader.readAll()))).toEqual(Array.from(payload)); + }); + + it('does not compress for a pre-v2 recipient (uncompressed round-trip)', async () => { + const { manager, sentPackets } = createSender(CLIENT_PROTOCOL_DATA_STREAM_RPC); + const payload = 'plain text '.repeat(2_000); + + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); + await writer.write(payload); + await writer.close(); + + // No compression attribute on the header. + const header = sentPackets.find((p) => p.value.case === 'streamHeader'); + const headerValue = header!.value.value as Extract< + DataPacket['value'], + { case: 'streamHeader' } + >['value']; + expect(headerValue.attributes['lk.compression']).toBeUndefined(); + + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe(payload); + }); +}); diff --git a/src/room/data-stream/compression.test.ts b/src/room/data-stream/compression.test.ts new file mode 100644 index 0000000000..91792a9fc2 --- /dev/null +++ b/src/room/data-stream/compression.test.ts @@ -0,0 +1,74 @@ +import { describe, expect, it } from 'vitest'; +import { gzipCompress, gzipDecompress, gzipDecompressStream } from './compression'; + +function bytes(str: string): Uint8Array { + return new TextEncoder().encode(str); +} + +function text(buf: Uint8Array): string { + return new TextDecoder().decode(buf); +} + +/** A readable stream that emits the given byte arrays in order. */ +function streamOf(...parts: Uint8Array[]): ReadableStream { + return new ReadableStream({ + start(controller) { + for (const part of parts) { + controller.enqueue(part); + } + controller.close(); + }, + }); +} + +async function collect(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks.push(value); + } + const total = chunks.reduce((n, c) => n + c.byteLength, 0); + const out = new Uint8Array(total); + let offset = 0; + for (const c of chunks) { + out.set(c, offset); + offset += c.byteLength; + } + return out; +} + +describe('data-stream gzip helpers', () => { + it('round-trips a buffered payload', async () => { + const original = bytes('the quick brown fox '.repeat(500)); + const restored = await gzipDecompress(await gzipCompress(original)); + expect(text(restored)).toBe(text(original)); + }); + + it('actually compresses repetitive data', async () => { + const original = bytes('A'.repeat(50_000)); + const compressed = await gzipCompress(original); + expect(compressed.byteLength).toBeLessThan(original.byteLength); + }); + + it('round-trips an empty payload', async () => { + const restored = await gzipDecompress(await gzipCompress(new Uint8Array(0))); + expect(restored.byteLength).toBe(0); + }); + + it('streams decompression of a payload split across many compressed input chunks', async () => { + const original = bytes('hello compressed world '.repeat(2_000)); + const compressed = await gzipCompress(original); + + // Feed the compressed bytes in small slices to exercise incremental decompression. + const slices: Uint8Array[] = []; + for (let i = 0; i < compressed.byteLength; i += 100) { + slices.push(compressed.slice(i, i + 100)); + } + const restored = await collect(gzipDecompressStream(streamOf(...slices))); + expect(text(restored)).toBe(text(original)); + }); +}); diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts new file mode 100644 index 0000000000..b340a5a9de --- /dev/null +++ b/src/room/data-stream/compression.ts @@ -0,0 +1,87 @@ +/** + * gzip compression helpers for data streams, built on the platform `CompressionStream` / + * `DecompressionStream`. The buffered variants are for the inline (single-packet) case where the + * payload is small and bounded; {@link gzipDecompressStream} streams for the chunked (multi-packet) + * case, consuming compressed input and producing decompressed output incrementally rather than + * buffering it all. + * + * These operate on bytes (not strings) so a single set of helpers serves both text and byte streams; + * the `TextEncoder`/`TextDecoder` boundary lives at the manager/reader edges. + * + * Like the rest of the SDK, these drive `getWriter()`/`getReader()` directly instead of + * `pipeThrough`, which sidesteps the `CompressionStream` lib-type mismatches. + * + * @internal + */ + +/** gzip-compresses a byte array in full. Use for inline payloads; prefer the streaming path for the + * chunked case. */ +export async function gzipCompress(data: Uint8Array): Promise { + const cs = new CompressionStream('gzip'); + const writer = cs.writable.getWriter(); + writer.write(data as NonSharedUint8Array); + writer.close(); + return collect(cs.readable); +} + +/** gunzips a byte array in full (inverse of {@link gzipCompress}). */ +export async function gzipDecompress(data: Uint8Array): Promise { + const ds = new DecompressionStream('gzip'); + const writer = ds.writable.getWriter(); + writer.write(data as NonSharedUint8Array); + writer.close(); + return collect(ds.readable); +} + +/** + * Streams a gzip-compressed byte stream through decompression, feeding compressed chunks into the + * `DecompressionStream` as they arrive and exposing the decompressed output as a readable. Source + * errors are forwarded by aborting the decompression input. + */ +export function gzipDecompressStream( + input: ReadableStream, +): ReadableStream { + const ds = new DecompressionStream('gzip'); + const writer = ds.writable.getWriter(); + // Drive compressed input into the decompressor in the background; the IIFE handles its own errors + // by aborting the writable (which surfaces on the readable side), so this never rejects. + const pipe = (async () => { + const reader = input.getReader(); + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + await writer.write(value as NonSharedUint8Array); + } + await writer.close(); + } catch (err) { + await writer.abort(err).catch(() => {}); + } + })(); + pipe.catch(() => {}); + return ds.readable; +} + +/** Concatenates all chunks of a byte stream into one array. */ +async function collect(stream: ReadableStream): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks.push(value); + total += value.byteLength; + } + const result = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; +} diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index c0746a56ff..f1707d4357 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -8,8 +8,9 @@ import { import log from '../../../logger'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { type ByteStreamInfo, type StreamController, type TextStreamInfo } from '../../types'; -import { bigIntToNumber, decodeBase64 } from '../../utils'; -import { INLINE_PAYLOAD_ATTRIBUTE } from '../constants'; +import { bigIntToNumber, decodeBase64, numberToBigInt } from '../../utils'; +import { gzipDecompress, gzipDecompressStream } from '../compression'; +import { COMPRESSION_ATTRIBUTE, COMPRESSION_GZIP, INLINE_PAYLOAD_ATTRIBUTE } from '../constants'; import { type ByteStreamHandler, ByteStreamReader, @@ -158,15 +159,19 @@ export default class IncomingDataStreamManager { encryptionType, }; + const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; + // Single-packet stream: the entire payload was smuggled into a reserved header attribute. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; if (typeof inlinePayload !== 'undefined') { delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; + delete info.attributes![COMPRESSION_ATTRIBUTE]; + const bytes = decodeBase64(inlinePayload); streamHandlerCallback( new ByteStreamReader( info, - createInlineStream(streamHeader.streamId, decodeBase64(inlinePayload)), + createInlineStream(streamHeader.streamId, compressed ? gzipDecompress(bytes) : bytes), bigIntToNumber(streamHeader.totalLength), ), { identity: participantIdentity }, @@ -174,7 +179,11 @@ export default class IncomingDataStreamManager { return; } - const stream = new ReadableStream({ + if (compressed) { + delete info.attributes![COMPRESSION_ATTRIBUTE]; + } + + const stream = new ReadableStream({ start: (controller) => { streamController = controller; @@ -194,7 +203,12 @@ export default class IncomingDataStreamManager { }, }); streamHandlerCallback( - new ByteStreamReader(info, stream, bigIntToNumber(streamHeader.totalLength)), + new ByteStreamReader( + info, + compressed ? decompressedChunkStream(stream, streamHeader.streamId, 'byte') : stream, + // Compressed streams report no total length; completion is driven by the trailer. + compressed ? undefined : bigIntToNumber(streamHeader.totalLength), + ), { identity: participantIdentity, }, @@ -222,15 +236,22 @@ export default class IncomingDataStreamManager { attachedStreamIds: streamHeader.contentHeader.value.attachedStreamIds, }; + const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; + // Single-packet stream: the entire payload was smuggled into a reserved header attribute. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; if (typeof inlinePayload !== 'undefined') { delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; + delete info.attributes![COMPRESSION_ATTRIBUTE]; + // Compressed text is base64(gzip(utf-8)); uncompressed text is the raw string. + const content = compressed + ? gzipDecompress(decodeBase64(inlinePayload)) + : new TextEncoder().encode(inlinePayload); streamHandlerCallback( new TextStreamReader( info, - createInlineStream(streamHeader.streamId, new TextEncoder().encode(inlinePayload)), + createInlineStream(streamHeader.streamId, content), bigIntToNumber(streamHeader.totalLength), ), { identity: participantIdentity }, @@ -238,6 +259,10 @@ export default class IncomingDataStreamManager { return; } + if (compressed) { + delete info.attributes![COMPRESSION_ATTRIBUTE]; + } + const stream = new ReadableStream({ start: (controller) => { streamController = controller; @@ -258,7 +283,12 @@ export default class IncomingDataStreamManager { }, }); streamHandlerCallback( - new TextStreamReader(info, stream, bigIntToNumber(streamHeader.totalLength)), + new TextStreamReader( + info, + compressed ? decompressedChunkStream(stream, streamHeader.streamId, 'text') : stream, + // Compressed streams report no total length; completion is driven by the trailer. + compressed ? undefined : bigIntToNumber(streamHeader.totalLength), + ), { identity: participantIdentity }, ); } @@ -332,16 +362,96 @@ export default class IncomingDataStreamManager { /** * Builds a `ReadableStream` that yields the given content as a single chunk and then immediately - * closes - used to surface an inline (single-packet) data stream as a fully-formed stream. + * closes - used to surface an inline (single-packet) data stream as a fully-formed stream. `content` + * may be a promise (e.g. async gzip decompression); a rejection errors the stream. */ function createInlineStream( streamId: string, - content: Uint8Array, + content: Uint8Array | Promise, ): ReadableStream { return new ReadableStream({ - start: (controller) => { - controller.enqueue(new DataStream_Chunk({ streamId, chunkIndex: BigInt(0), content })); - controller.close(); + start: async (controller) => { + try { + const bytes = await content; + controller.enqueue( + new DataStream_Chunk({ streamId, chunkIndex: BigInt(0), content: bytes }), + ); + controller.close(); + } catch (err) { + controller.error(err); + } }, }); } + +/** + * Transforms a raw stream of (compressed) `DataStream_Chunk`s into a stream of decompressed + * `DataStream_Chunk`s, so the existing `ByteStreamReader`/`TextStreamReader` consume it unchanged. + * For text, the decompressed bytes are reframed on UTF-8 character boundaries (via + * `TextDecoderStream`) so each synthesized chunk decodes independently. Errors on the source stream + * (e.g. encryption mismatch, abnormal end) propagate downstream to the reader. + */ +function decompressedChunkStream( + raw: ReadableStream, + streamId: string, + kind: 'byte' | 'text', +): ReadableStream { + const decompressed = gzipDecompressStream(chunkContentStream(raw)).getReader(); + // For text, reframe decompressed bytes on UTF-8 character boundaries (the decoder buffers partial + // multibyte sequences across reads) and re-encode each whole-character fragment, so the reader's + // per-chunk fatal decode always sees valid input. + const decoder = kind === 'text' ? new TextDecoder() : undefined; + const encoder = kind === 'text' ? new TextEncoder() : undefined; + let chunkIndex = 0; + + return new ReadableStream({ + async pull(controller) { + for (;;) { + const { done, value } = await decompressed.read(); + if (done) { + const tail = decoder?.decode(); + if (tail) { + controller.enqueue(makeChunk(streamId, chunkIndex++, encoder!.encode(tail))); + } + controller.close(); + return; + } + const content = decoder ? encoder!.encode(decoder.decode(value, { stream: true })) : value; + if (content.byteLength === 0) { + continue; // partial multibyte char buffered; pull more + } + controller.enqueue(makeChunk(streamId, chunkIndex++, content)); + return; + } + }, + cancel: (reason) => { + decompressed.cancel(reason); + }, + }); +} + +/** Maps a stream of `DataStream_Chunk`s to a stream of their raw (compressed) `content` bytes. */ +function chunkContentStream(raw: ReadableStream): ReadableStream { + const reader = raw.getReader(); + return new ReadableStream({ + async pull(controller) { + const { done, value } = await reader.read(); + if (done) { + controller.close(); + return; + } + controller.enqueue(value.content); + }, + cancel: (reason) => { + reader.cancel(reason); + }, + }); +} + +function makeChunk(streamId: string, chunkIndex: number, content: Uint8Array): DataStream_Chunk { + return new DataStream_Chunk({ + streamId, + chunkIndex: numberToBigInt(chunkIndex), + content: content as NonSharedUint8Array, + }); +} diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index ecd800ff0d..901b994a95 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -18,8 +18,14 @@ import type { StreamTextOptions, TextStreamInfo, } from '../../types'; -import { numberToBigInt, splitUtf8 } from '../../utils'; -import { INLINE_PAYLOAD_ATTRIBUTE, STREAM_CHUNK_SIZE_BYTES } from '../constants'; +import { encodeBase64, isCompressionStreamSupported, numberToBigInt, splitUtf8 } from '../../utils'; +import { gzipCompress } from '../compression'; +import { + COMPRESSION_ATTRIBUTE, + COMPRESSION_GZIP, + INLINE_PAYLOAD_ATTRIBUTE, + STREAM_CHUNK_SIZE_BYTES, +} from '../constants'; import { ByteStreamWriter, TextStreamWriter } from './StreamWriter'; import { buildByteStreamHeader, @@ -118,12 +124,12 @@ export default class OutgoingDataStreamManager { } /** - * Returns true only if every recipient is known to support single-packet (inline) data streams. - * For a targeted send this checks the named destination identities; for a broadcast (no explicit - * destinations) it checks every remote participant currently in the room. An empty room (nobody - * to receive) is considered eligible. + * Returns true only if every recipient is known to support data streams v2 (single-packet inline + * streams and compression). For a targeted send this checks the named destination identities; for + * a broadcast (no explicit destinations) it checks every remote participant currently in the room. + * An empty room (nobody to receive) is considered eligible. */ - private canSendInline(destinationIdentities?: Array): boolean { + private allRecipientsSupportV2(destinationIdentities?: Array): boolean { const identities = destinationIdentities && destinationIdentities.length > 0 ? destinationIdentities @@ -134,6 +140,11 @@ export default class OutgoingDataStreamManager { ); } + /** Whether to gzip-compress a stream: all recipients support v2 and the runtime can compress. */ + private shouldCompress(destinationIdentities?: Array): boolean { + return this.allRecipientsSupportV2(destinationIdentities) && isCompressionStreamSupported(); + } + /** * Attempts to send `text` as a single header packet with the payload smuggled into a reserved * attribute. Returns the resulting {@link TextStreamInfo} if it was sent inline, or `null` if the @@ -146,7 +157,7 @@ export default class OutgoingDataStreamManager { totalTextLength: number, options?: SendTextOptions, ): Promise { - if (!this.canSendInline(options?.destinationIdentities)) { + if (!this.allRecipientsSupportV2(options?.destinationIdentities)) { return null; } @@ -162,10 +173,23 @@ export default class OutgoingDataStreamManager { : Encryption_Type.NONE, }; - const header = buildTextStreamHeader({ - ...info, - attributes: { ...info.attributes, [INLINE_PAYLOAD_ATTRIBUTE]: text }, - }); + // Compress when the runtime supports it, but only keep the result if it actually shrinks the + // payload (gzip adds ~18 bytes of framing, so tiny strings get larger). Uncompressed inline + // payloads stay as the raw string; compressed ones are base64'd and flagged via an attribute. + const inlineAttributes: Record = { + ...info.attributes, + [INLINE_PAYLOAD_ATTRIBUTE]: text, + }; + if (isCompressionStreamSupported()) { + const raw = new TextEncoder().encode(text); + const compressed = await gzipCompress(raw); + if (compressed.byteLength < raw.byteLength) { + inlineAttributes[INLINE_PAYLOAD_ATTRIBUTE] = encodeBase64(compressed); + inlineAttributes[COMPRESSION_ATTRIBUTE] = COMPRESSION_GZIP; + } + } + + const header = buildTextStreamHeader({ ...info, attributes: inlineAttributes }); const packet = createStreamHeaderPacket(header, options?.destinationIdentities); if (packet.toBinary().byteLength > STREAM_CHUNK_SIZE_BYTES) { @@ -182,71 +206,76 @@ export default class OutgoingDataStreamManager { */ async streamText(options?: StreamTextOptions): Promise { const streamId = options?.streamId ?? crypto.randomUUID(); + const destinationIdentities = options?.destinationIdentities; + const compressOption = options?.compress ?? true; + const compress = compressOption && this.shouldCompress(destinationIdentities); const info: TextStreamInfo = { id: streamId, mimeType: 'text/plain', timestamp: Date.now(), topic: options?.topic ?? '', - size: options?.totalSize, - attributes: options?.attributes, + // Compressed streams have an unknown total length up front, so it is left undefined and the + // trailer signals completion (see receiver validation). + // + // FIXME: make this instead the size before compression maybe? + size: compress ? undefined : options?.totalSize, + attributes: compress + ? { ...options?.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_GZIP } + : options?.attributes, encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled ? Encryption_Type.GCM : Encryption_Type.NONE, attachedStreamIds: options?.attachedStreamIds, }; const header = buildTextStreamHeader(info, options); - const destinationIdentities = options?.destinationIdentities; const packet = createStreamHeaderPacket(header, destinationIdentities); await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); let chunkId = 0; const engine = this.engine; - const writableStream = new WritableStream({ - // Implement the sink - async write(text) { - for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { - const chunk = new DataStream_Chunk({ - content: textByteChunk, - streamId, - chunkIndex: numberToBigInt(chunkId), - }); - const chunkPacket = new DataPacket({ - destinationIdentities, - value: { - case: 'streamChunk', - value: chunk, - }, - }); - await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); - - chunkId += 1; - } - }, - async close() { - const trailer = new DataStream_Trailer({ - streamId, - }); - const trailerPacket = new DataPacket({ - destinationIdentities, - value: { - case: 'streamTrailer', - value: trailer, + const writableStream = compress + ? createCompressedChunkWritable(streamId, destinationIdentities, engine, (text) => + new TextEncoder().encode(text), + ) + : new WritableStream({ + // Uncompressed path: split each write on UTF-8 boundaries so every chunk decodes + // independently on the receiver (required for pre-v2 receivers). + async write(text) { + for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { + const chunk = new DataStream_Chunk({ + content: textByteChunk, + streamId, + chunkIndex: numberToBigInt(chunkId), + }); + const chunkPacket = new DataPacket({ + destinationIdentities, + value: { + case: 'streamChunk', + value: chunk, + }, + }); + await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); + + chunkId += 1; + } + }, + async close() { + await sendStreamTrailer(streamId, destinationIdentities, engine); + }, + abort(err) { + console.log('Sink error:', err); + // TODO handle aborts to signal something to receiver side }, }); - await engine.sendDataPacket(trailerPacket, DataChannelKind.RELIABLE); - }, - abort(err) { - console.log('Sink error:', err); - // TODO handle aborts to signal something to receiver side - }, - }); let onEngineClose = async () => { await writer.close(); }; + // FIXME: make this a global event to ensure "max listener" warning won't get logged for lots of + // in flight data streams. engine.once(EngineEvent.Closing, onEngineClose); const writer = new TextStreamWriter(writableStream, info, () => @@ -286,14 +315,21 @@ export default class OutgoingDataStreamManager { async streamBytes(options?: StreamBytesOptions) { const streamId = options?.streamId ?? crypto.randomUUID(); const destinationIdentities = options?.destinationIdentities; + const compressOption = options?.compress ?? true; + const compress = compressOption && this.shouldCompress(destinationIdentities); const info: ByteStreamInfo = { id: streamId, mimeType: options?.mimeType ?? 'application/octet-stream', topic: options?.topic ?? '', timestamp: Date.now(), - attributes: options?.attributes, - size: options?.totalSize, + attributes: compress + ? { ...options?.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_GZIP } + : options?.attributes, + // Compressed streams have an unknown total length up front; left undefined (see receiver). + // + // FIXME: make this instead the size before compression maybe? + size: compress ? undefined : options?.totalSize, name: options?.name ?? 'unknown', encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled ? Encryption_Type.GCM @@ -310,53 +346,143 @@ export default class OutgoingDataStreamManager { const engine = this.engine; const logLocal = this.log; - const writableStream = new WritableStream({ - async write(chunk) { - const unlock = await writeMutex.lock(); - - let byteOffset = 0; - try { - while (byteOffset < chunk.byteLength) { - const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); - const chunkPacket = new DataPacket({ - destinationIdentities, - value: { - case: 'streamChunk', - value: new DataStream_Chunk({ - content: subChunk, - streamId, - chunkIndex: numberToBigInt(chunkId), - }), - }, - }); - await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); - chunkId += 1; - byteOffset += subChunk.byteLength; - } - } finally { - unlock(); - } - }, - async close() { - const trailer = new DataStream_Trailer({ + const writableStream = compress + ? createCompressedChunkWritable( streamId, - }); - const trailerPacket = new DataPacket({ destinationIdentities, - value: { - case: 'streamTrailer', - value: trailer, + engine, + (chunk) => chunk, + ) + : new WritableStream({ + async write(chunk) { + const unlock = await writeMutex.lock(); + + let byteOffset = 0; + try { + while (byteOffset < chunk.byteLength) { + const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); + const chunkPacket = new DataPacket({ + destinationIdentities, + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + content: subChunk, + streamId, + chunkIndex: numberToBigInt(chunkId), + }), + }, + }); + await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); + chunkId += 1; + byteOffset += subChunk.byteLength; + } + } finally { + unlock(); + } + }, + async close() { + await sendStreamTrailer(streamId, destinationIdentities, engine); + }, + abort(err) { + logLocal.error('Sink error:', err); }, }); - await engine.sendDataPacket(trailerPacket, DataChannelKind.RELIABLE); - }, - abort(err) { - logLocal.error('Sink error:', err); - }, - }); const byteWriter = new ByteStreamWriter(writableStream, info); return byteWriter; } } + +/** Sends a `streamTrailer` packet, marking the end of a stream. */ +async function sendStreamTrailer( + streamId: string, + destinationIdentities: Array | undefined, + engine: RTCEngine, +): Promise { + const trailerPacket = new DataPacket({ + destinationIdentities, + value: { case: 'streamTrailer', value: new DataStream_Trailer({ streamId }) }, + }); + await engine.sendDataPacket(trailerPacket, DataChannelKind.RELIABLE); +} + +function concatBytes(a: Uint8Array, b: Uint8Array): Uint8Array { + const out = new Uint8Array(a.byteLength + b.byteLength); + out.set(a, 0); + out.set(b, a.byteLength); + return out; +} + +/** + * Builds a `WritableStream` whose writes are gzip-compressed (streaming, never buffering the whole + * payload) and whose compressed output is emitted as fixed-size `streamChunk` packets, followed by a + * `streamTrailer` on close. The compressed bytes are chunked on arbitrary boundaries — safe because + * the receiver decompresses the full concatenation — so this is only used for recipients that + * understand compressed streams. + */ +function createCompressedChunkWritable( + streamId: string, + destinationIdentities: Array | undefined, + engine: RTCEngine, + encode: (chunk: T) => Uint8Array, +): WritableStream { + const cs = new CompressionStream('gzip'); + const csWriter = cs.writable.getWriter(); + const reader = cs.readable.getReader(); + + let chunkId = 0; + let pending: Uint8Array = new Uint8Array(0); + + const sendChunk = async (content: Uint8Array) => { + const chunkPacket = new DataPacket({ + destinationIdentities, + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + content: content as NonSharedUint8Array, + streamId, + chunkIndex: numberToBigInt(chunkId), + }), + }, + }); + await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); + chunkId += 1; + }; + + // Drain compressed output in the background, emitting full-size chunks as they accumulate and + // flushing the remainder once the compressor closes. Reading here backpressures the compressor, + // which backpressures the writer, giving end-to-end flow control. + const pump = (async () => { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + pending = concatBytes(pending, value); + while (pending.byteLength >= STREAM_CHUNK_SIZE_BYTES) { + await sendChunk(pending.slice(0, STREAM_CHUNK_SIZE_BYTES)); + pending = pending.slice(STREAM_CHUNK_SIZE_BYTES); + } + } + if (pending.byteLength > 0) { + await sendChunk(pending); + pending = new Uint8Array(0); + } + })(); + + return new WritableStream({ + async write(chunk) { + await csWriter.write(encode(chunk) as NonSharedUint8Array); + }, + async close() { + await csWriter.close(); + await pump; + await sendStreamTrailer(streamId, destinationIdentities, engine); + }, + async abort(err) { + await csWriter.abort(err).catch(() => {}); + await reader.cancel(err).catch(() => {}); + }, + }); +} diff --git a/src/room/types.ts b/src/room/types.ts index 1eb8121679..fc91dde06f 100644 --- a/src/room/types.ts +++ b/src/room/types.ts @@ -26,6 +26,7 @@ export interface SendTextOptions { export interface StreamTextOptions { topic?: string; destinationIdentities?: Array; + compress?: boolean; type?: 'create' | 'update'; streamId?: string; version?: number; @@ -40,6 +41,7 @@ export type StreamBytesOptions = { topic?: string; attributes?: Record; destinationIdentities?: Array; + compress?: boolean; streamId?: string; mimeType?: string; totalSize?: number; From aac267bbe3a0b0f09700947c414d08d299875a80 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 5 Jun 2026 16:48:00 -0400 Subject: [PATCH 10/68] feat: migrate compression to get rid of "pump" and compress per packet / split into 15k MTU Doing this makes data arrive a bit faster since it's not waiting for compressed bytes to accumulate before sending out a "fully filled" packet --- .../data-stream/compression-roundtrip.test.ts | 34 ++++++++++ src/room/data-stream/compression.ts | 13 ++++ .../outgoing/OutgoingDataStreamManager.ts | 65 ++++++------------- 3 files changed, 67 insertions(+), 45 deletions(-) diff --git a/src/room/data-stream/compression-roundtrip.test.ts b/src/room/data-stream/compression-roundtrip.test.ts index 5c95cbf6d1..8035cbc2ff 100644 --- a/src/room/data-stream/compression-roundtrip.test.ts +++ b/src/room/data-stream/compression-roundtrip.test.ts @@ -123,6 +123,40 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { expect(Array.from(flatten(await reader.readAll()))).toEqual(Array.from(payload)); }); + it('round-trips text written across multiple writes (multi-member gzip)', async () => { + const { manager, sentPackets } = createSender(); + const parts = ['first part ', '日本語🚀 second ', 'x'.repeat(20_000), ' tail']; + + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); + for (const part of parts) { + await writer.write(part); + } + await writer.close(); + + expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe(parts.join('')); + }); + + it('round-trips bytes written across multiple writes (multi-member gzip)', async () => { + const { manager, sentPackets } = createSender(); + const parts = [ + new Uint8Array([1, 2, 3]), + new Uint8Array(20_000).fill(7), + new Uint8Array([9, 8, 7, 6]), + ]; + const expected = parts.flatMap((p) => Array.from(p)); + + const writer = await manager.streamBytes({ topic: 'b', destinationIdentities: [RECEIVER] }); + for (const part of parts) { + await writer.write(part); + } + await writer.close(); + + const reader = await receiveBytes(sentPackets, 'b'); + expect(Array.from(flatten(await reader.readAll()))).toEqual(expected); + }); + it('does not compress for a pre-v2 recipient (uncompressed round-trip)', async () => { const { manager, sentPackets } = createSender(CLIENT_PROTOCOL_DATA_STREAM_RPC); const payload = 'plain text '.repeat(2_000); diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index b340a5a9de..be8a829caf 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -24,6 +24,19 @@ export async function gzipCompress(data: Uint8Array): Promise { return collect(cs.readable); } +/** + * gzip-compresses a byte array, exposing the compressed output as a readable stream so callers can + * forward it incrementally instead of buffering the whole result. The input is written in full (the + * caller already holds it), but the output is produced and consumed chunk by chunk. + */ +export function gzipCompressStream(data: Uint8Array): ReadableStream { + const cs = new CompressionStream('gzip'); + const writer = cs.writable.getWriter(); + writer.write(data as NonSharedUint8Array); + writer.close(); + return cs.readable; +} + /** gunzips a byte array in full (inverse of {@link gzipCompress}). */ export async function gzipDecompress(data: Uint8Array): Promise { const ds = new DecompressionStream('gzip'); diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 901b994a95..9ada53fe50 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -19,7 +19,7 @@ import type { TextStreamInfo, } from '../../types'; import { encodeBase64, isCompressionStreamSupported, numberToBigInt, splitUtf8 } from '../../utils'; -import { gzipCompress } from '../compression'; +import { gzipCompress, gzipCompressStream } from '../compression'; import { COMPRESSION_ATTRIBUTE, COMPRESSION_GZIP, @@ -407,19 +407,14 @@ async function sendStreamTrailer( await engine.sendDataPacket(trailerPacket, DataChannelKind.RELIABLE); } -function concatBytes(a: Uint8Array, b: Uint8Array): Uint8Array { - const out = new Uint8Array(a.byteLength + b.byteLength); - out.set(a, 0); - out.set(b, a.byteLength); - return out; -} - /** - * Builds a `WritableStream` whose writes are gzip-compressed (streaming, never buffering the whole - * payload) and whose compressed output is emitted as fixed-size `streamChunk` packets, followed by a - * `streamTrailer` on close. The compressed bytes are chunked on arbitrary boundaries — safe because - * the receiver decompresses the full concatenation — so this is only used for recipients that - * understand compressed streams. + * Builds a `WritableStream` whose writes are gzip-compressed and emitted as `streamChunk` packets, + * followed by a `streamTrailer` on close. Each `write()` is compressed into its own gzip member and + * its compressed bytes are split into `STREAM_CHUNK_SIZE_BYTES` pieces sent immediately — including a + * final partial piece — so a write's data is delivered without waiting to fill a chunk or for the + * stream to close (low latency for incremental senders). The receiver decompresses the concatenation + * of members (a valid multi-member gzip stream), so this is only used for recipients that understand + * compressed streams. */ function createCompressedChunkWritable( streamId: string, @@ -427,12 +422,7 @@ function createCompressedChunkWritable( engine: RTCEngine, encode: (chunk: T) => Uint8Array, ): WritableStream { - const cs = new CompressionStream('gzip'); - const csWriter = cs.writable.getWriter(); - const reader = cs.readable.getReader(); - let chunkId = 0; - let pending: Uint8Array = new Uint8Array(0); const sendChunk = async (content: Uint8Array) => { const chunkPacket = new DataPacket({ @@ -450,39 +440,24 @@ function createCompressedChunkWritable( chunkId += 1; }; - // Drain compressed output in the background, emitting full-size chunks as they accumulate and - // flushing the remainder once the compressor closes. Reading here backpressures the compressor, - // which backpressures the writer, giving end-to-end flow control. - const pump = (async () => { - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - pending = concatBytes(pending, value); - while (pending.byteLength >= STREAM_CHUNK_SIZE_BYTES) { - await sendChunk(pending.slice(0, STREAM_CHUNK_SIZE_BYTES)); - pending = pending.slice(STREAM_CHUNK_SIZE_BYTES); - } - } - if (pending.byteLength > 0) { - await sendChunk(pending); - pending = new Uint8Array(0); - } - })(); - return new WritableStream({ async write(chunk) { - await csWriter.write(encode(chunk) as NonSharedUint8Array); + const reader = gzipCompressStream(encode(chunk)).getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + await Promise.all(new Array(Math.ceil(value.length / STREAM_CHUNK_SIZE_BYTES)).fill(null).map((_, i) => { + return sendChunk(value.slice(i * STREAM_CHUNK_SIZE_BYTES, (i+1) * STREAM_CHUNK_SIZE_BYTES)); + })); + } }, async close() { - await csWriter.close(); - await pump; await sendStreamTrailer(streamId, destinationIdentities, engine); }, - async abort(err) { - await csWriter.abort(err).catch(() => {}); - await reader.cancel(err).catch(() => {}); + abort() { + // Each write compresses independently, so there is no persistent compressor to tear down. }, }); } From c7acc6bef84ea5e97fc4531c18c14984c1454b8b Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 5 Jun 2026 16:56:38 -0400 Subject: [PATCH 11/68] fix: swap else if -> switch/case --- .../incoming/IncomingDataStreamManager.ts | 279 +++++++++--------- 1 file changed, 142 insertions(+), 137 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index f1707d4357..c4f76d2b6e 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -136,161 +136,166 @@ export default class IncomingDataStreamManager { participantIdentity: string, encryptionType: Encryption_Type, ) { - if (streamHeader.contentHeader.case === 'byteHeader') { - const streamHandlerCallback = this.byteStreamHandlers.get(streamHeader.topic); - if (!streamHandlerCallback) { - this.log.debug( - 'ignoring incoming byte stream due to no handler for topic', - streamHeader.topic, - ); - return; - } + switch (streamHeader.contentHeader.case) { + case 'byteHeader': { + const streamHandlerCallback = this.byteStreamHandlers.get(streamHeader.topic); + if (!streamHandlerCallback) { + this.log.debug( + 'ignoring incoming byte stream due to no handler for topic', + streamHeader.topic, + ); + return; + } + + let streamController: ReadableStreamDefaultController; + + const info: ByteStreamInfo = { + id: streamHeader.streamId, + name: streamHeader.contentHeader.value.name ?? 'unknown', + mimeType: streamHeader.mimeType, + size: streamHeader.totalLength ? Number(streamHeader.totalLength) : undefined, + topic: streamHeader.topic, + timestamp: bigIntToNumber(streamHeader.timestamp), + attributes: streamHeader.attributes, + encryptionType, + }; + + const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; + + // Single-packet stream: the entire payload was smuggled into a reserved header attribute. + // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. + const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; + if (typeof inlinePayload !== 'undefined') { + delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; + delete info.attributes![COMPRESSION_ATTRIBUTE]; + const bytes = decodeBase64(inlinePayload); + streamHandlerCallback( + new ByteStreamReader( + info, + createInlineStream(streamHeader.streamId, compressed ? gzipDecompress(bytes) : bytes), + bigIntToNumber(streamHeader.totalLength), + ), + { identity: participantIdentity }, + ); + return; + } - let streamController: ReadableStreamDefaultController; - - const info: ByteStreamInfo = { - id: streamHeader.streamId, - name: streamHeader.contentHeader.value.name ?? 'unknown', - mimeType: streamHeader.mimeType, - size: streamHeader.totalLength ? Number(streamHeader.totalLength) : undefined, - topic: streamHeader.topic, - timestamp: bigIntToNumber(streamHeader.timestamp), - attributes: streamHeader.attributes, - encryptionType, - }; - - const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; - - // Single-packet stream: the entire payload was smuggled into a reserved header attribute. - // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. - const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; - if (typeof inlinePayload !== 'undefined') { - delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; - delete info.attributes![COMPRESSION_ATTRIBUTE]; - const bytes = decodeBase64(inlinePayload); + if (compressed) { + delete info.attributes![COMPRESSION_ATTRIBUTE]; + } + + const stream = new ReadableStream({ + start: (controller) => { + streamController = controller; + + if (this.textStreamControllers.has(streamHeader.streamId)) { + throw new DataStreamError( + `A data stream read is already in progress for a stream with id ${streamHeader.streamId}.`, + DataStreamErrorReason.AlreadyOpened, + ); + } + + this.byteStreamControllers.set(streamHeader.streamId, { + info, + controller: streamController, + startTime: Date.now(), + sendingParticipantIdentity: participantIdentity, + }); + }, + }); streamHandlerCallback( new ByteStreamReader( info, - createInlineStream(streamHeader.streamId, compressed ? gzipDecompress(bytes) : bytes), - bigIntToNumber(streamHeader.totalLength), + compressed ? decompressedChunkStream(stream, streamHeader.streamId, 'byte') : stream, + // Compressed streams report no total length; completion is driven by the trailer. + compressed ? undefined : bigIntToNumber(streamHeader.totalLength), ), - { identity: participantIdentity }, + { + identity: participantIdentity, + }, ); - return; - } - - if (compressed) { - delete info.attributes![COMPRESSION_ATTRIBUTE]; + break; } + case 'textHeader': { + const streamHandlerCallback = this.textStreamHandlers.get(streamHeader.topic); + if (!streamHandlerCallback) { + this.log.debug( + 'ignoring incoming text stream due to no handler for topic', + streamHeader.topic, + ); + return; + } - const stream = new ReadableStream({ - start: (controller) => { - streamController = controller; + let streamController: ReadableStreamDefaultController; - if (this.textStreamControllers.has(streamHeader.streamId)) { - throw new DataStreamError( - `A data stream read is already in progress for a stream with id ${streamHeader.streamId}.`, - DataStreamErrorReason.AlreadyOpened, - ); - } + const info: TextStreamInfo = { + id: streamHeader.streamId, + mimeType: streamHeader.mimeType, + size: streamHeader.totalLength ? Number(streamHeader.totalLength) : undefined, + topic: streamHeader.topic, + timestamp: Number(streamHeader.timestamp), + attributes: streamHeader.attributes, + encryptionType, + attachedStreamIds: streamHeader.contentHeader.value.attachedStreamIds, + }; + + const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; + + // Single-packet stream: the entire payload was smuggled into a reserved header attribute. + // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. + const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; + if (typeof inlinePayload !== 'undefined') { + delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; + delete info.attributes![COMPRESSION_ATTRIBUTE]; + // Compressed text is base64(gzip(utf-8)); uncompressed text is the raw string. + const content = compressed + ? gzipDecompress(decodeBase64(inlinePayload)) + : new TextEncoder().encode(inlinePayload); + streamHandlerCallback( + new TextStreamReader( + info, + createInlineStream(streamHeader.streamId, content), + bigIntToNumber(streamHeader.totalLength), + ), + { identity: participantIdentity }, + ); + return; + } - this.byteStreamControllers.set(streamHeader.streamId, { - info, - controller: streamController, - startTime: Date.now(), - sendingParticipantIdentity: participantIdentity, - }); - }, - }); - streamHandlerCallback( - new ByteStreamReader( - info, - compressed ? decompressedChunkStream(stream, streamHeader.streamId, 'byte') : stream, - // Compressed streams report no total length; completion is driven by the trailer. - compressed ? undefined : bigIntToNumber(streamHeader.totalLength), - ), - { - identity: participantIdentity, - }, - ); - } else if (streamHeader.contentHeader.case === 'textHeader') { - const streamHandlerCallback = this.textStreamHandlers.get(streamHeader.topic); - if (!streamHandlerCallback) { - this.log.debug( - 'ignoring incoming text stream due to no handler for topic', - streamHeader.topic, - ); - return; - } + if (compressed) { + delete info.attributes![COMPRESSION_ATTRIBUTE]; + } - let streamController: ReadableStreamDefaultController; - - const info: TextStreamInfo = { - id: streamHeader.streamId, - mimeType: streamHeader.mimeType, - size: streamHeader.totalLength ? Number(streamHeader.totalLength) : undefined, - topic: streamHeader.topic, - timestamp: Number(streamHeader.timestamp), - attributes: streamHeader.attributes, - encryptionType, - attachedStreamIds: streamHeader.contentHeader.value.attachedStreamIds, - }; - - const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; - - // Single-packet stream: the entire payload was smuggled into a reserved header attribute. - // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. - const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; - if (typeof inlinePayload !== 'undefined') { - delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; - delete info.attributes![COMPRESSION_ATTRIBUTE]; - // Compressed text is base64(gzip(utf-8)); uncompressed text is the raw string. - const content = compressed - ? gzipDecompress(decodeBase64(inlinePayload)) - : new TextEncoder().encode(inlinePayload); + const stream = new ReadableStream({ + start: (controller) => { + streamController = controller; + + if (this.textStreamControllers.has(streamHeader.streamId)) { + throw new DataStreamError( + `A data stream read is already in progress for a stream with id ${streamHeader.streamId}.`, + DataStreamErrorReason.AlreadyOpened, + ); + } + + this.textStreamControllers.set(streamHeader.streamId, { + info, + controller: streamController, + startTime: Date.now(), + sendingParticipantIdentity: participantIdentity, + }); + }, + }); streamHandlerCallback( new TextStreamReader( info, - createInlineStream(streamHeader.streamId, content), - bigIntToNumber(streamHeader.totalLength), + compressed ? decompressedChunkStream(stream, streamHeader.streamId, 'text') : stream, + // Compressed streams report no total length; completion is driven by the trailer. + compressed ? undefined : bigIntToNumber(streamHeader.totalLength), ), { identity: participantIdentity }, ); - return; - } - - if (compressed) { - delete info.attributes![COMPRESSION_ATTRIBUTE]; + break; } - - const stream = new ReadableStream({ - start: (controller) => { - streamController = controller; - - if (this.textStreamControllers.has(streamHeader.streamId)) { - throw new DataStreamError( - `A data stream read is already in progress for a stream with id ${streamHeader.streamId}.`, - DataStreamErrorReason.AlreadyOpened, - ); - } - - this.textStreamControllers.set(streamHeader.streamId, { - info, - controller: streamController, - startTime: Date.now(), - sendingParticipantIdentity: participantIdentity, - }); - }, - }); - streamHandlerCallback( - new TextStreamReader( - info, - compressed ? decompressedChunkStream(stream, streamHeader.streamId, 'text') : stream, - // Compressed streams report no total length; completion is driven by the trailer. - compressed ? undefined : bigIntToNumber(streamHeader.totalLength), - ), - { identity: participantIdentity }, - ); } } From c406a2e1fa3f5b2ef65cd43d75f102c62b4e05d4 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 8 Jun 2026 14:59:44 -0400 Subject: [PATCH 12/68] feat: add initial take at streaming through CompressionStream for data streams --- examples/data-stream-benchmark/benchmark.ts | 4 +- .../IncomingDataStreamManager.test.ts | 95 +++++++++++++++ .../incoming/IncomingDataStreamManager.ts | 112 +++++++++++------- .../outgoing/OutgoingDataStreamManager.ts | 20 +++- 4 files changed, 185 insertions(+), 46 deletions(-) create mode 100644 src/room/data-stream/incoming/IncomingDataStreamManager.test.ts diff --git a/examples/data-stream-benchmark/benchmark.ts b/examples/data-stream-benchmark/benchmark.ts index 2225fa333a..9a3cfe9a22 100644 --- a/examples/data-stream-benchmark/benchmark.ts +++ b/examples/data-stream-benchmark/benchmark.ts @@ -16,8 +16,8 @@ const MAX_FILL_COUNT = BOX_DURATION_MS; /** Cell fill hue (R,G,B); opacity scales with throughput. */ const FILL_RGB = '52,152,219'; -/** How many chunks to split up the data stream payload into. If `0`, send all at once with `sendText`. */ -const STREAM_CHUNK_SIZE_BYTES = 0; +/** Chunk size to split up the data stream payload into. If `0`, send all at once with `sendText`. */ +const STREAM_CHUNK_SIZE_BYTES = 1000; const TOPIC = 'benchmark'; const SENDER_IDENTITY = 'bench-sender'; diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts new file mode 100644 index 0000000000..a6beb9ea2a --- /dev/null +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -0,0 +1,95 @@ +import { + DataPacket, + DataStream_ByteHeader, + DataStream_Header, + DataStream_TextHeader, + Encryption_Type, +} from '@livekit/protocol'; +import { describe, expect, it } from 'vitest'; +import { encodeBase64 } from '../../utils'; +import { INLINE_PAYLOAD_ATTRIBUTE } from '../constants'; +import IncomingDataStreamManager from './IncomingDataStreamManager'; +import type { ByteStreamReader, TextStreamReader } from './StreamReader'; + +function inlineTextHeaderPacket(streamId: string, topic: string, text: string) { + const header = new DataStream_Header({ + streamId, + topic, + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(new TextEncoder().encode(text).byteLength), + attributes: { [INLINE_PAYLOAD_ATTRIBUTE]: text, foo: 'bar' }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }); + return new DataPacket({ + participantIdentity: 'alice', + value: { case: 'streamHeader', value: header }, + }); +} + +function inlineByteHeaderPacket(streamId: string, topic: string, bytes: Uint8Array) { + const header = new DataStream_Header({ + streamId, + topic, + mimeType: 'application/octet-stream', + timestamp: 0n, + totalLength: BigInt(bytes.byteLength), + attributes: { [INLINE_PAYLOAD_ATTRIBUTE]: encodeBase64(bytes), foo: 'bar' }, + contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({ name: 'blob' }) }, + }); + return new DataPacket({ + participantIdentity: 'alice', + value: { case: 'streamHeader', value: header }, + }); +} + +describe('IncomingDataStreamManager inline streams', () => { + it('synthesizes a complete text stream from an inline header', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + manager.handleDataStreamPacket( + inlineTextHeaderPacket('stream-1', 'my-topic', 'hello world'), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + expect(await reader.readAll()).toBe('hello world'); + + // The reserved attribute is stripped, user attributes are preserved. + expect(reader.info.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeUndefined(); + expect(reader.info.attributes?.foo).toBe('bar'); + }); + + it('synthesizes a complete byte stream from an inline header', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const payload = new Uint8Array([0, 1, 2, 255, 128, 64]); + + const readerPromise = new Promise((resolve) => { + manager.registerByteStreamHandler('bytes-topic', (reader) => resolve(reader)); + }); + + manager.handleDataStreamPacket( + inlineByteHeaderPacket('stream-2', 'bytes-topic', payload), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + const chunks = await reader.readAll(); + const flattened = new Uint8Array(chunks.reduce((acc, c) => acc + c.byteLength, 0)); + let offset = 0; + for (const chunk of chunks) { + flattened.set(chunk, offset); + offset += chunk.byteLength; + } + expect(Array.from(flattened)).toEqual(Array.from(payload)); + expect(reader.info.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeUndefined(); + expect(reader.info.attributes?.foo).toBe('bar'); + }); +}); diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index c4f76d2b6e..13e4514f8c 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -9,7 +9,7 @@ import log from '../../../logger'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { type ByteStreamInfo, type StreamController, type TextStreamInfo } from '../../types'; import { bigIntToNumber, decodeBase64, numberToBigInt } from '../../utils'; -import { gzipDecompress, gzipDecompressStream } from '../compression'; +import { gzipDecompress } from '../compression'; import { COMPRESSION_ATTRIBUTE, COMPRESSION_GZIP, INLINE_PAYLOAD_ATTRIBUTE } from '../constants'; import { type ByteStreamHandler, @@ -392,63 +392,95 @@ function createInlineStream( /** * Transforms a raw stream of (compressed) `DataStream_Chunk`s into a stream of decompressed * `DataStream_Chunk`s, so the existing `ByteStreamReader`/`TextStreamReader` consume it unchanged. - * For text, the decompressed bytes are reframed on UTF-8 character boundaries (via - * `TextDecoderStream`) so each synthesized chunk decodes independently. Errors on the source stream - * (e.g. encryption mismatch, abnormal end) propagate downstream to the reader. + * + * The sender compresses each `write()` into its own gzip member and tags every chunk with that + * member's index in `chunk.version`. Browsers' `DecompressionStream` only accepts a single member + * per gzip stream, so we feed each member's chunks into its own `DecompressionStream` as they arrive + * (never buffering the whole member) and start a fresh one when the member index changes, draining + * decompressed output incrementally. For text, a streaming `TextDecoder` reframes the decompressed + * bytes on UTF-8 character boundaries across members so each synthesized chunk decodes independently. + * Errors on the source stream (e.g. encryption mismatch, abnormal end) propagate to the reader. */ function decompressedChunkStream( raw: ReadableStream, streamId: string, kind: 'byte' | 'text', ): ReadableStream { - const decompressed = gzipDecompressStream(chunkContentStream(raw)).getReader(); - // For text, reframe decompressed bytes on UTF-8 character boundaries (the decoder buffers partial - // multibyte sequences across reads) and re-encode each whole-character fragment, so the reader's - // per-chunk fatal decode always sees valid input. + const srcReader = raw.getReader(); const decoder = kind === 'text' ? new TextDecoder() : undefined; const encoder = kind === 'text' ? new TextEncoder() : undefined; - let chunkIndex = 0; - - return new ReadableStream({ - async pull(controller) { - for (;;) { - const { done, value } = await decompressed.read(); - if (done) { - const tail = decoder?.decode(); - if (tail) { - controller.enqueue(makeChunk(streamId, chunkIndex++, encoder!.encode(tail))); + let outIndex = 0; + + const enqueueDecompressed = ( + controller: ReadableStreamDefaultController, + bytes: Uint8Array, + ) => { + const content = decoder ? encoder!.encode(decoder.decode(bytes, { stream: true })) : bytes; + if (content.byteLength > 0) { + controller.enqueue(makeChunk(streamId, outIndex++, content)); + } + }; + + const pump = async (controller: ReadableStreamDefaultController) => { + let currentMember: number | undefined; + let dsWriter: WritableStreamDefaultWriter | null = null; + let drain: Promise | null = null; + + const openMember = () => { + const ds = new DecompressionStream('gzip'); + dsWriter = ds.writable.getWriter(); + const dsReader = ds.readable.getReader(); + // Drain this member's decompressed output concurrently with feeding its input. + drain = (async () => { + for (;;) { + const { done, value } = await dsReader.read(); + if (done) { + break; } - controller.close(); - return; - } - const content = decoder ? encoder!.encode(decoder.decode(value, { stream: true })) : value; - if (content.byteLength === 0) { - continue; // partial multibyte char buffered; pull more + enqueueDecompressed(controller, value); } - controller.enqueue(makeChunk(streamId, chunkIndex++, content)); - return; + })(); + }; + + // Close the current member's compressor input and wait for its remaining output to drain. + const closeMember = async () => { + if (dsWriter) { + await dsWriter.close(); + await drain; + dsWriter = null; + drain = null; } - }, - cancel: (reason) => { - decompressed.cancel(reason); - }, - }); -} + }; -/** Maps a stream of `DataStream_Chunk`s to a stream of their raw (compressed) `content` bytes. */ -function chunkContentStream(raw: ReadableStream): ReadableStream { - const reader = raw.getReader(); - return new ReadableStream({ - async pull(controller) { - const { done, value } = await reader.read(); + for (;;) { + const { done, value } = await srcReader.read(); if (done) { + await closeMember(); + const tail = decoder?.decode(); + if (tail) { + controller.enqueue(makeChunk(streamId, outIndex++, encoder!.encode(tail))); + } controller.close(); return; } - controller.enqueue(value.content); + // A change in member index means the previous gzip member is complete. + if (currentMember !== undefined && value.version !== currentMember) { + await closeMember(); + } + if (!dsWriter) { + openMember(); + } + currentMember = value.version; + await dsWriter!.write(value.content as NonSharedUint8Array); + } + }; + + return new ReadableStream({ + start: (controller) => { + pump(controller).catch((err) => controller.error(err)); }, cancel: (reason) => { - reader.cancel(reason); + srcReader.cancel(reason); }, }); } diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 9ada53fe50..d4b6c4962b 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -423,8 +423,12 @@ function createCompressedChunkWritable( encode: (chunk: T) => Uint8Array, ): WritableStream { let chunkId = 0; + // Each write() is compressed into its own gzip member. Browsers' DecompressionStream only accepts a + // single member per gzip stream, so we tag every chunk with its member index (in the chunk's spare + // `version` field); the receiver segments on it and decompresses each member independently. + let memberId = 0; - const sendChunk = async (content: Uint8Array) => { + const sendChunk = async (content: Uint8Array, version: number) => { const chunkPacket = new DataPacket({ destinationIdentities, value: { @@ -433,6 +437,7 @@ function createCompressedChunkWritable( content: content as NonSharedUint8Array, streamId, chunkIndex: numberToBigInt(chunkId), + version, }), }, }); @@ -442,15 +447,22 @@ function createCompressedChunkWritable( return new WritableStream({ async write(chunk) { + const member = memberId; + memberId += 1; const reader = gzipCompressStream(encode(chunk)).getReader(); while (true) { const { done, value } = await reader.read(); if (done) { break; } - await Promise.all(new Array(Math.ceil(value.length / STREAM_CHUNK_SIZE_BYTES)).fill(null).map((_, i) => { - return sendChunk(value.slice(i * STREAM_CHUNK_SIZE_BYTES, (i+1) * STREAM_CHUNK_SIZE_BYTES)); - })); + await Promise.all( + new Array(Math.ceil(value.length / STREAM_CHUNK_SIZE_BYTES)).fill(null).map((_, i) => { + return sendChunk( + value.slice(i * STREAM_CHUNK_SIZE_BYTES, (i + 1) * STREAM_CHUNK_SIZE_BYTES), + member, + ); + }), + ); } }, async close() { From 38b2a0cb844c732427ae5f0449627b9d6c063d04 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 9 Jun 2026 15:54:53 -0400 Subject: [PATCH 13/68] feat: commit hand written data streams compression approach Tag each data stream chunk with a "compression index" which if > 0 points to a given DecompressionStream which should be used for preprocessing received bytes. --- .../incoming/IncomingDataStreamManager.ts | 5 +- src/room/data-stream/incoming/StreamReader.ts | 179 ++++++++++++++---- .../outgoing/OutgoingDataStreamManager.ts | 82 +++++--- 3 files changed, 201 insertions(+), 65 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 13e4514f8c..a1ec44deea 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -288,9 +288,8 @@ export default class IncomingDataStreamManager { streamHandlerCallback( new TextStreamReader( info, - compressed ? decompressedChunkStream(stream, streamHeader.streamId, 'text') : stream, - // Compressed streams report no total length; completion is driven by the trailer. - compressed ? undefined : bigIntToNumber(streamHeader.totalLength), + stream, + bigIntToNumber(streamHeader.totalLength), ), { identity: participantIdentity }, ); diff --git a/src/room/data-stream/incoming/StreamReader.ts b/src/room/data-stream/incoming/StreamReader.ts index 4d5e0c2a4d..2e7985ce34 100644 --- a/src/room/data-stream/incoming/StreamReader.ts +++ b/src/room/data-stream/incoming/StreamReader.ts @@ -1,7 +1,7 @@ import type { DataStream_Chunk } from '@livekit/protocol'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import type { BaseStreamInfo, ByteStreamInfo, TextStreamInfo } from '../../types'; -import { bigIntToNumber } from '../../utils'; +import { bigIntToNumber, Future } from '../../utils'; export type BaseStreamReaderReadAllOpts = { /** An AbortSignal can be used to terminate reads early. */ @@ -151,7 +151,16 @@ export class ByteStreamReader extends BaseStreamReader { * A class to read chunks from a ReadableStream and provide them in a structured format. */ export class TextStreamReader extends BaseStreamReader { - private receivedChunks: Map; + /** Store a queue of chunks to be read. */ + private chunks: Array = []; + private chunkPublishedFuture = new Future<{ done: false, value: DataStream_Chunk } | { done: true, value?: undefined }, never>(); + private receivedChunks: Map; + private receivedChunkDecompressionStreams: Map; + reader: ReadableStreamDefaultReader; + }> = new Map(); + signal?: AbortSignal; @@ -204,54 +213,152 @@ export class TextStreamReader extends BaseStreamReader { const decoder = new TextDecoder('utf-8', { fatal: true }); const signal = this.signal; + const readNext = (reader: ReadableStreamDefaultReader, signal?: AbortSignal) => { + return new Promise>( + (resolve, reject) => { + if (signal) { + const onAbort = () => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + reader + .read() + .then(resolve, reject) + .finally(() => { + signal.removeEventListener('abort', onAbort); + }); + } else { + reader.read().then(resolve, reject); + } + }, + ); + }; + + const decodeChunkContents = (content: Uint8Array, chunkIndex: bigint) => { + let decodedResult; + try { + decodedResult = decoder.decode(content); + } catch (err) { + throw new DataStreamError( + `Cannot decode datastream chunk ${chunkIndex} as text: ${err}`, + DataStreamErrorReason.DecodeFailed, + ); + } + + return decodedResult; + }; + const cleanup = () => { reader.releaseLock(); this.signal = undefined; }; - return { - next: async (): Promise> => { + // Prefill DecompressionStreams ahead of the iterator for fastest decompression performance + (async () => { + let lastChunkIndex: bigint | null = null; + while (true) { try { if (signal?.aborted) { throw signal.reason; } - const result = await new Promise>( - (resolve, reject) => { - if (signal) { - const onAbort = () => reject(signal.reason); - signal.addEventListener('abort', onAbort, { once: true }); - reader - .read() - .then(resolve, reject) - .finally(() => { - signal.removeEventListener('abort', onAbort); - }); - } else { - reader.read().then(resolve, reject); - } - }, - ); + const result = await readNext(reader, signal); + // console.log('RESULT', result); if (result.done) { - this.validateBytesReceived(true); - return { done: true, value: undefined }; + this.chunkPublishedFuture.resolve?.({ done: true }); + break; } else { - this.handleChunkReceived(result.value); - - let decodedResult; - try { - decodedResult = decoder.decode(result.value.content); - } catch (err) { - throw new DataStreamError( - `Cannot decode datastream chunk ${result.value.chunkIndex} as text: ${err}`, - DataStreamErrorReason.DecodeFailed, - ); + this.chunks.unshift(result.value); + this.chunkPublishedFuture.resolve?.({ done: false, value: result.value }); + this.chunkPublishedFuture = new Future(); + + const compressionIndex = result.value.iv?.[0] ?? 0; + if (compressionIndex > 0) { + let state = this.receivedChunkDecompressionStreams.get(compressionIndex); + if (!state) { + const stream = new DecompressionStream('gzip'); + state = { + stream, + writer: stream.writable.getWriter(), + reader: stream.readable.getReader(), + }; + this.receivedChunkDecompressionStreams.set(compressionIndex, state); + } + // console.log('WRITE CMP', compressionIndex, result.value.content); + state.writer.write(result.value.content as BufferSource); } - return { - done: false, - value: decodedResult, - }; + lastChunkIndex = result.value.chunkIndex; + } + } catch (err) { + cleanup(); + throw err; + } + } + })(); + + return { + next: async (): Promise> => { + try { + if (signal?.aborted) { + throw signal.reason; + } + + // Step 1: Get next chunk, either already pre-fetched in this.chunks, or if not then + // wait for the next one to be generated + let chunk = this.chunks.pop(); + if (!chunk) { + const { done, value } = await this.chunkPublishedFuture.promise; + if (done) { + this.validateBytesReceived(true); + return { done: true, value: undefined }; + } + this.chunks.pop(); // FIXME: maybe do this in a loop? + chunk = value; + } + // console.log('CHUNK', chunk); + + this.handleChunkReceived(chunk); + + let chunkContent = chunk.content; + + // Step 2: optionally decompress bu pulling the proper length in bytes from the + // corresponding DecompressionStream + const compressionIndex = chunk.iv?.[0] ?? 0; + const uncompressedByteLength = (((chunk.iv?.[1] ?? 0) & 0xff) << 8) | ((chunk.iv?.[2] ?? 0) & 0xff); + // console.log('COMPRESSION RATIO:', chunkContent.length / uncompressedByteLength); + if (compressionIndex > 0) { + // Chunk was compressed, so read the next `uncompressedByteLength` bytes + const decompressionState = this.receivedChunkDecompressionStreams.get(compressionIndex); + if (decompressionState) { + let combinedBuffer = new Uint8Array(uncompressedByteLength); + let offset = 0; + while (true) { + const { done, value } = await decompressionState.reader.read(); + // console.log('CMP READ:', done, value); + if (done) { + break; + } + if (offset + value.length > combinedBuffer.length) { + throw new Error(`uncompressedByteLength value was too short, espected to be able to fit at least ${value.length} bytes at offset ${offset}, but only had ${combinedBuffer.length} bytes of space`); + } + combinedBuffer.set(value, offset); + offset += value.length; + if (offset >= combinedBuffer.length) { + // FIXME: store value.slice(offset - uncompressedByteLength) and return on next read + break; + } + } + chunkContent = combinedBuffer; + } } + + // Step 3: Decode raw result back into text + // console.log('CNT', chunkContent); + const decodedResult = decodeChunkContents(chunkContent, chunk.chunkIndex); + // console.log('OUTPUT', decodedResult); + + return { + done: false, + value: decodedResult, + }; } catch (err) { cleanup(); throw err; diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index d4b6c4962b..89f70bb9dc 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -208,7 +208,7 @@ export default class OutgoingDataStreamManager { const streamId = options?.streamId ?? crypto.randomUUID(); const destinationIdentities = options?.destinationIdentities; const compressOption = options?.compress ?? true; - const compress = compressOption && this.shouldCompress(destinationIdentities); + const compress = isCompressionStreamSupported() && compressOption && this.shouldCompress(destinationIdentities); const info: TextStreamInfo = { id: streamId, @@ -233,42 +233,72 @@ export default class OutgoingDataStreamManager { await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); let chunkId = 0; + let compressionIndex = 1; const engine = this.engine; - const writableStream = compress - ? createCompressedChunkWritable(streamId, destinationIdentities, engine, (text) => - new TextEncoder().encode(text), - ) - : new WritableStream({ - // Uncompressed path: split each write on UTF-8 boundaries so every chunk decodes - // independently on the receiver (required for pre-v2 receivers). - async write(text) { - for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { - const chunk = new DataStream_Chunk({ - content: textByteChunk, - streamId, - chunkIndex: numberToBigInt(chunkId), - }); + const writableStream = new WritableStream({ + // Uncompressed path: split each write on UTF-8 boundaries so every chunk decodes + // independently on the receiver (required for pre-v2 receivers). + async write(text) { + // console.log('WRITE', text); + const writeMutex = new Mutex(); + const sendChunkPacket = async (chunk: Uint8Array, uncompressedByteLength: number, compressionIndex: number = 0) => { + const unlock = await writeMutex.lock(); + + let byteOffset = 0; + try { + while (byteOffset < chunk.byteLength) { + const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); const chunkPacket = new DataPacket({ destinationIdentities, value: { case: 'streamChunk', - value: chunk, + value: new DataStream_Chunk({ + content: subChunk, + streamId, + chunkIndex: numberToBigInt(chunkId), + iv: new Uint8Array([ // FIXME: swap with dedicated fields! + compressionIndex, + (uncompressedByteLength >> 8) & 0xff, + uncompressedByteLength & 0xff, + ]), + }), }, }); + // console.log('PACKET', chunkPacket); await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); - chunkId += 1; + byteOffset += subChunk.byteLength; } - }, - async close() { - await sendStreamTrailer(streamId, destinationIdentities, engine); - }, - abort(err) { - console.log('Sink error:', err); - // TODO handle aborts to signal something to receiver side - }, - }); + } finally { + unlock(); + } + }; + + // Try to compress data if compression is supported + if (compress) { + const raw = new TextEncoder().encode(text); + const compressed = await gzipCompress(raw); + if (compressed.byteLength < raw.byteLength) { + await sendChunkPacket(compressed, raw.length, compressionIndex); + compressionIndex += 1; + return; + } + } + + // Fallback to old uncompressed path if compression isn't possible / doesn't make it smaller + for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { + await sendChunkPacket(textByteChunk, textByteChunk.length, 0); + } + }, + async close() { + await sendStreamTrailer(streamId, destinationIdentities, engine); + }, + abort(err) { + console.log('Sink error:', err); + // TODO handle aborts to signal something to receiver side + }, + }); let onEngineClose = async () => { await writer.close(); From 3f605f1b64c675ac893cbd1c66ef60bbd0880a22 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 10 Jun 2026 13:43:13 -0400 Subject: [PATCH 14/68] feat: try out using fflate to share a single compression stream across whole data stream fflate allows flushing the compressed bytes midway through so that they can be emitted "together" as a chunk in the output stream, which a DecompressionStream('deflate-raw') can properly consume. --- package.json | 1 + pnpm-lock.yaml | 8 + .../data-stream/compression-roundtrip.test.ts | 131 ++++++++++++- src/room/data-stream/compression.test.ts | 83 ++++++++- src/room/data-stream/compression.ts | 83 +++++++++ src/room/data-stream/constants.ts | 11 ++ .../incoming/IncomingDataStreamManager.ts | 120 +++++++++++- src/room/data-stream/incoming/StreamReader.ts | 174 ++++-------------- .../outgoing/OutgoingDataStreamManager.ts | 114 ++++++------ 9 files changed, 514 insertions(+), 211 deletions(-) diff --git a/package.json b/package.json index 2561dbc657..f6146429db 100644 --- a/package.json +++ b/package.json @@ -74,6 +74,7 @@ "@livekit/mutex": "1.1.1", "@livekit/protocol": "1.46.6", "events": "^3.3.0", + "fflate": "^0.8.3", "jose": "^6.1.0", "loglevel": "^1.9.2", "sdp-transform": "^2.15.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 247d0397cf..f45da23418 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: events: specifier: ^3.3.0 version: 3.3.0 + fflate: + specifier: ^0.8.3 + version: 0.8.3 jose: specifier: ^6.1.0 version: 6.2.3 @@ -2516,6 +2519,9 @@ packages: picomatch: optional: true + fflate@0.8.3: + resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} + file-entry-cache@8.0.0: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} @@ -6556,6 +6562,8 @@ snapshots: optionalDependencies: picomatch: 4.0.4 + fflate@0.8.3: {} + file-entry-cache@8.0.0: dependencies: flat-cache: 4.0.1 diff --git a/src/room/data-stream/compression-roundtrip.test.ts b/src/room/data-stream/compression-roundtrip.test.ts index 8035cbc2ff..8685b3816c 100644 --- a/src/room/data-stream/compression-roundtrip.test.ts +++ b/src/room/data-stream/compression-roundtrip.test.ts @@ -1,4 +1,4 @@ -import { type DataPacket, Encryption_Type } from '@livekit/protocol'; +import { type DataPacket, type DataStream_Chunk, Encryption_Type } from '@livekit/protocol'; import { describe, expect, it, vi } from 'vitest'; import log from '../../logger'; import { CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DATA_STREAM_V2 } from '../../version'; @@ -123,7 +123,7 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { expect(Array.from(flatten(await reader.readAll()))).toEqual(Array.from(payload)); }); - it('round-trips text written across multiple writes (multi-member gzip)', async () => { + it('round-trips text written across multiple writes (shared deflate context)', async () => { const { manager, sentPackets } = createSender(); const parts = ['first part ', '日本語🚀 second ', 'x'.repeat(20_000), ' tail']; @@ -157,6 +157,133 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { expect(Array.from(flatten(await reader.readAll()))).toEqual(expected); }); + it('marks chunked compressed text streams with the deflate-raw attribute', async () => { + const { manager, sentPackets } = createSender(); + + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); + await writer.write('compressed contents'); + await writer.close(); + + const header = sentPackets.find((p) => p.value.case === 'streamHeader'); + const headerValue = header!.value.value as Extract< + DataPacket['value'], + { case: 'streamHeader' } + >['value']; + expect(headerValue.attributes['lk.compression']).toBe('deflate-raw'); + // Chunk packets carry no smuggled metadata. + for (const packet of sentPackets) { + if (packet.value.case === 'streamChunk') { + expect(packet.value.value.iv).toBeUndefined(); + expect(packet.value.value.version).toBe(0); + } + } + }); + + it('emits each write to the receiver as its packets arrive, before the stream ends', async () => { + const { manager, sentPackets } = createSender(); + const incoming = new IncomingDataStreamManager(); + incoming.setConnected(true); + const readerPromise = new Promise((resolve) => { + incoming.registerTextStreamHandler('t', (reader) => resolve(reader)); + }); + + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); + let fed = 0; + const feedNewPackets = () => { + for (const packet of sentPackets.slice(fed)) { + incoming.handleDataStreamPacket(packet, Encryption_Type.NONE); + } + fed = sentPackets.length; + }; + + const writes = ['first write ', 'second write, repeating first write words ', 'third']; + let iterator: AsyncIterator | undefined; + for (const write of writes) { + await writer.write(write); + feedNewPackets(); + if (!iterator) { + const reader = await readerPromise; + iterator = reader[Symbol.asyncIterator](); + } + // The write's full text must be readable now - no trailer has been sent yet. + let got = ''; + while (got.length < write.length) { + const { done, value } = await iterator.next(); + expect(done).toBeFalsy(); + got += value; + } + expect(got).toBe(write); + } + + await writer.close(); + feedNewPackets(); + const end = await iterator!.next(); + expect(end.done).toBe(true); + }); + + it('ignores a duplicated chunk packet with a warning', async () => { + const { manager, sentPackets } = createSender(); + const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => {}); + + try { + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); + await writer.write('part one '); + await writer.write('part two'); + await writer.close(); + + // Replay the first chunk packet again right after the original. + const firstChunkIdx = sentPackets.findIndex((p) => p.value.case === 'streamChunk'); + const packets = [...sentPackets]; + packets.splice(firstChunkIdx + 1, 0, packets[firstChunkIdx]); + + const reader = await receiveText(packets, 't'); + expect(await reader.readAll()).toBe('part one part two'); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('duplicate chunk')); + } finally { + warnSpy.mockRestore(); + } + }); + + it('errors the stream when a chunk goes missing (gap in chunk indices)', async () => { + const { manager, sentPackets } = createSender(); + + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); + await writer.write('part one '); + await writer.write('part two'); + await writer.close(); + + // Drop the first chunk packet entirely. + const firstChunkIdx = sentPackets.findIndex((p) => p.value.case === 'streamChunk'); + const packets = sentPackets.filter((_, i) => i !== firstChunkIdx); + + const reader = await receiveText(packets, 't'); + await expect(reader.readAll()).rejects.toThrow(/[Mm]issing chunk/); + }); + + it('round-trips a long stream of many small writes (transcription pattern)', async () => { + const { manager, sentPackets } = createSender(); + const writes = Array.from( + { length: 500 }, + (_, i) => `transcription segment number ${i} with some repeated filler words. `, + ); + + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); + for (const write of writes) { + await writer.write(write); + } + await writer.close(); + + const expected = writes.join(''); + // Context takeover should compress the repetitive writes well below the raw size. + const compressedTotal = sentPackets + .filter((p) => p.value.case === 'streamChunk') + .reduce((sum, p) => sum + (p.value.value as DataStream_Chunk).content.byteLength, 0); + expect(compressedTotal).toBeLessThan(expected.length / 3); + + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe(expected); + }); + it('does not compress for a pre-v2 recipient (uncompressed round-trip)', async () => { const { manager, sentPackets } = createSender(CLIENT_PROTOCOL_DATA_STREAM_RPC); const payload = 'plain text '.repeat(2_000); diff --git a/src/room/data-stream/compression.test.ts b/src/room/data-stream/compression.test.ts index 91792a9fc2..e477b46efa 100644 --- a/src/room/data-stream/compression.test.ts +++ b/src/room/data-stream/compression.test.ts @@ -1,5 +1,11 @@ import { describe, expect, it } from 'vitest'; -import { gzipCompress, gzipDecompress, gzipDecompressStream } from './compression'; +import { + StreamingDeflate, + gzipCompress, + gzipDecompress, + gzipDecompressStream, + inflateRawStream, +} from './compression'; function bytes(str: string): Uint8Array { return new TextEncoder().encode(str); @@ -72,3 +78,78 @@ describe('data-stream gzip helpers', () => { expect(text(restored)).toBe(text(original)); }); }); + +describe('StreamingDeflate + inflateRawStream', () => { + it('round-trips multiple writes through a single decompressor', async () => { + const deflate = new StreamingDeflate(); + const writes = ['first write ', 'second write with 日本語🚀 ', 'third '.repeat(100)]; + const parts = writes.map((w) => deflate.compressWrite(bytes(w))); + parts.push(deflate.end()); + + const restored = await collect(inflateRawStream(streamOf(...parts))); + expect(text(restored)).toBe(writes.join('')); + }); + + it("emits each write's content before any further input arrives (timeliness)", async () => { + const deflate = new StreamingDeflate(); + + let inputController!: ReadableStreamDefaultController; + const input = new ReadableStream({ + start(controller) { + inputController = controller; + }, + }); + const reader = inflateRawStream(input).getReader(); + const decoder = new TextDecoder(); + + const writes = ['hello there, ', 'hello again - repeating myself, hello hello ', 'bye']; + for (const write of writes) { + inputController.enqueue(deflate.compressWrite(bytes(write))); + // The write's full content must come out without sending anything further. + let got = ''; + while (got.length < write.length) { + const { done, value } = await reader.read(); + expect(done).toBe(false); + got += decoder.decode(value!, { stream: true }); + } + expect(got).toBe(write); + } + + inputController.enqueue(deflate.end()); + inputController.close(); + const { done } = await reader.read(); + expect(done).toBe(true); + }); + + it('reuses the compression context across writes (later similar writes shrink)', () => { + const deflate = new StreamingDeflate(); + const sentence = 'the quick brown fox jumps over the lazy dog and keeps on jumping. '; + const first = deflate.compressWrite(bytes(sentence.repeat(10))); + const second = deflate.compressWrite(bytes(sentence.repeat(10))); + // The second write is pure back-references into the persisted window. + expect(second.byteLength).toBeLessThan(first.byteLength / 4); + }); + + it('round-trips incompressible input', async () => { + const deflate = new StreamingDeflate(); + const original = new Uint8Array(10_000); + for (let i = 0; i < original.length; i += 1) { + original[i] = Math.floor(Math.random() * 256); + } + const parts = [deflate.compressWrite(original), deflate.end()]; + const restored = await collect(inflateRawStream(streamOf(...parts))); + expect(Array.from(restored)).toEqual(Array.from(original)); + }); + + it('handles an empty write', async () => { + const deflate = new StreamingDeflate(); + const parts = [ + deflate.compressWrite(bytes('before ')), + deflate.compressWrite(new Uint8Array(0)), + deflate.compressWrite(bytes('after')), + deflate.end(), + ]; + const restored = await collect(inflateRawStream(streamOf(...parts))); + expect(text(restored)).toBe('before after'); + }); +}); diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index be8a829caf..89bca71d58 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -13,6 +13,89 @@ * * @internal */ +import { Deflate } from 'fflate'; + +/** + * Per-stream raw-deflate compressor for chunked data streams. One instance lives for the whole + * stream, so the LZ77 window persists across writes — repeated content in later writes compresses + * against earlier ones (the permessage-deflate "context takeover" model). Each + * {@link StreamingDeflate.compressWrite} output is flushed to a byte boundary, so a receiver + * feeding the bytes through a single raw-deflate decompressor can decode every write's content as + * soon as it arrives, without waiting for the stream to end. + * + * Built on fflate rather than `CompressionStream` because the platform compressor has no flush — + * it only emits buffered output on close, which would force a fresh (dictionary-reset) compressor + * per write. + */ +export class StreamingDeflate { + private pending: Array = []; + + private deflate = new Deflate((chunk) => { + this.pending.push(chunk); + }); + + /** Compresses one write. The returned bytes are byte-aligned (sync flush), so the receiver can + * decompress the write's full content immediately upon receiving them. */ + compressWrite(data: Uint8Array): Uint8Array { + this.deflate.push(data); + this.deflate.flush(); + return this.takePending(); + } + + /** Terminates the deflate stream with an empty final block. Call exactly once, after the last + * write; the returned bytes must be delivered so the receiver's decompressor can close cleanly. */ + end(): Uint8Array { + this.deflate.push(new Uint8Array(0), true); + return this.takePending(); + } + + private takePending(): Uint8Array { + if (this.pending.length === 1) { + const only = this.pending[0]; + this.pending = []; + return only; + } + const total = this.pending.reduce((sum, chunk) => sum + chunk.byteLength, 0); + const result = new Uint8Array(total); + let offset = 0; + for (const chunk of this.pending) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + this.pending = []; + return result; + } +} + +/** + * Streams raw-deflate input through a single `DecompressionStream('deflate-raw')` for the lifetime + * of the stream (inverse of {@link StreamingDeflate}). Inflate emits output greedily, so as long as + * the sender flushed at write boundaries, each write's content is produced as soon as its + * compressed bytes are written. Source errors are forwarded by aborting the decompression input. + */ +export function inflateRawStream(input: ReadableStream): ReadableStream { + const ds = new DecompressionStream('deflate-raw'); + const writer = ds.writable.getWriter(); + // Drive compressed input into the decompressor in the background; the IIFE handles its own errors + // by aborting the writable (which surfaces on the readable side), so this never rejects. + const pipe = (async () => { + const reader = input.getReader(); + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + await writer.write(value as NonSharedUint8Array); + } + await writer.close(); + } catch (err) { + await writer.abort(err).catch(() => {}); + } + })(); + pipe.catch(() => {}); + return ds.readable; +} /** gzip-compresses a byte array in full. Use for inline payloads; prefer the streaming path for the * chunked case. */ diff --git a/src/room/data-stream/constants.ts b/src/room/data-stream/constants.ts index d78d70cf66..f20480c156 100644 --- a/src/room/data-stream/constants.ts +++ b/src/room/data-stream/constants.ts @@ -27,3 +27,14 @@ export const COMPRESSION_ATTRIBUTE = 'lk.compression'; /** Value of {@link COMPRESSION_ATTRIBUTE} for gzip-compressed payloads. @internal */ export const COMPRESSION_GZIP = 'gzip'; + +/** + * Value of {@link COMPRESSION_ATTRIBUTE} for chunked streams compressed with a single raw-deflate + * context shared across the whole stream. The sender sync-flushes at every write boundary so the + * receiver can decompress each chunk as it arrives, and terminates the deflate stream with a final + * block before the trailer. Receivers concatenate chunk contents in `chunkIndex` order through one + * raw-deflate (windowBits -15) decompressor. + * + * @internal + */ +export const COMPRESSION_DEFLATE_RAW = 'deflate-raw'; diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index a1ec44deea..2c163ebb44 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -9,8 +9,13 @@ import log from '../../../logger'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { type ByteStreamInfo, type StreamController, type TextStreamInfo } from '../../types'; import { bigIntToNumber, decodeBase64, numberToBigInt } from '../../utils'; -import { gzipDecompress } from '../compression'; -import { COMPRESSION_ATTRIBUTE, COMPRESSION_GZIP, INLINE_PAYLOAD_ATTRIBUTE } from '../constants'; +import { gzipDecompress, inflateRawStream } from '../compression'; +import { + COMPRESSION_ATTRIBUTE, + COMPRESSION_DEFLATE_RAW, + COMPRESSION_GZIP, + INLINE_PAYLOAD_ATTRIBUTE, +} from '../constants'; import { type ByteStreamHandler, ByteStreamReader, @@ -239,7 +244,11 @@ export default class IncomingDataStreamManager { attachedStreamIds: streamHeader.contentHeader.value.attachedStreamIds, }; - const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; + // Inline payloads are compressed as one-shot gzip; chunked streams as a single raw-deflate + // stream spanning all chunks (see COMPRESSION_DEFLATE_RAW). + const inlineCompressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; + const streamCompressed = + info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_DEFLATE_RAW; // Single-packet stream: the entire payload was smuggled into a reserved header attribute. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. @@ -248,7 +257,7 @@ export default class IncomingDataStreamManager { delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; delete info.attributes![COMPRESSION_ATTRIBUTE]; // Compressed text is base64(gzip(utf-8)); uncompressed text is the raw string. - const content = compressed + const content = inlineCompressed ? gzipDecompress(decodeBase64(inlinePayload)) : new TextEncoder().encode(inlinePayload); streamHandlerCallback( @@ -262,7 +271,7 @@ export default class IncomingDataStreamManager { return; } - if (compressed) { + if (streamCompressed) { delete info.attributes![COMPRESSION_ATTRIBUTE]; } @@ -288,7 +297,9 @@ export default class IncomingDataStreamManager { streamHandlerCallback( new TextStreamReader( info, - stream, + streamCompressed ? inflateRawChunkStream(stream, streamHeader.streamId) : stream, + // `totalLength` is the pre-compression size, and the reader sees decompressed bytes, so + // it applies to both paths. bigIntToNumber(streamHeader.totalLength), ), { identity: participantIdentity }, @@ -388,6 +399,103 @@ function createInlineStream( }); } +/** + * Transforms a stream of deflate-raw-compressed text `DataStream_Chunk`s into a stream of + * decompressed chunks, so `TextStreamReader` consumes it unchanged. + * + * The sender runs a single raw-deflate context across the whole stream, sync-flushing at every + * write boundary, so the receiver feeds all chunk contents (in `chunkIndex` order) through ONE + * decompressor and gets each write's content emitted as soon as its chunks arrive. A streaming + * `TextDecoder` reframes the decompressed bytes on UTF-8 character boundaries (a write larger than + * the MTU spans several packets, which may split a codepoint) so each synthesized chunk decodes + * independently. Errors on the source stream (e.g. encryption mismatch, abnormal end) propagate to + * the reader. + */ +function inflateRawChunkStream( + raw: ReadableStream, + streamId: string, +): ReadableStream { + const srcReader = raw.getReader(); + + // Stage 1: unwrap chunk packets to compressed bytes, guarding ordering. A stateful decompressor + // silently corrupts on duplicated or out-of-order input, so duplicates are dropped (with a + // warning - in-order delivery is expected on the reliable channel, but reconnect handling may + // replay) and a gap is a hard error. + let lastChunkIndex = -1; + const compressedBytes = new ReadableStream({ + pull: async (controller) => { + for (;;) { + const { done, value } = await srcReader.read(); + if (done) { + controller.close(); + return; + } + const index = bigIntToNumber(value.chunkIndex); + if (index <= lastChunkIndex) { + log.warn( + `ignoring duplicate chunk ${index} for compressed data stream ${streamId} (last processed: ${lastChunkIndex})`, + ); + continue; + } + if (index > lastChunkIndex + 1) { + throw new DataStreamError( + `Missing chunk(s) ${lastChunkIndex + 1}..${index - 1} for compressed data stream ${streamId} - cannot continue decompressing`, + DataStreamErrorReason.Incomplete, + ); + } + lastChunkIndex = index; + controller.enqueue(value.content); + return; + } + }, + cancel: (reason) => srcReader.cancel(reason), + }); + + // Stage 2: one decompressor for the stream's lifetime. + const decompressedReader = inflateRawStream(compressedBytes).getReader(); + + // Stage 3: reframe decompressed bytes on UTF-8 boundaries and re-wrap as chunks. + const decoder = new TextDecoder('utf-8', { fatal: true }); + const encoder = new TextEncoder(); + let outIndex = 0; + const decodeOrThrow = (bytes?: Uint8Array): string => { + try { + return bytes ? decoder.decode(bytes, { stream: true }) : decoder.decode(); + } catch (err) { + throw new DataStreamError( + `Cannot decode compressed data stream ${streamId} as text: ${err}`, + DataStreamErrorReason.DecodeFailed, + ); + } + }; + + return new ReadableStream({ + pull: async (controller) => { + for (;;) { + const { done, value } = await decompressedReader.read(); + if (done) { + const tail = decodeOrThrow(); + if (tail.length > 0) { + controller.enqueue(makeChunk(streamId, outIndex++, encoder.encode(tail))); + } + controller.close(); + return; + } + const text = decodeOrThrow(value); + if (text.length > 0) { + controller.enqueue(makeChunk(streamId, outIndex++, encoder.encode(text))); + return; + } + // Everything so far was a partial codepoint; keep pulling. + } + }, + cancel: (reason) => { + decompressedReader.cancel(reason).catch(() => {}); + srcReader.cancel(reason).catch(() => {}); + }, + }); +} + /** * Transforms a raw stream of (compressed) `DataStream_Chunk`s into a stream of decompressed * `DataStream_Chunk`s, so the existing `ByteStreamReader`/`TextStreamReader` consume it unchanged. diff --git a/src/room/data-stream/incoming/StreamReader.ts b/src/room/data-stream/incoming/StreamReader.ts index 2e7985ce34..e95ea55fc6 100644 --- a/src/room/data-stream/incoming/StreamReader.ts +++ b/src/room/data-stream/incoming/StreamReader.ts @@ -1,7 +1,7 @@ import type { DataStream_Chunk } from '@livekit/protocol'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import type { BaseStreamInfo, ByteStreamInfo, TextStreamInfo } from '../../types'; -import { bigIntToNumber, Future } from '../../utils'; +import { bigIntToNumber } from '../../utils'; export type BaseStreamReaderReadAllOpts = { /** An AbortSignal can be used to terminate reads early. */ @@ -151,16 +151,7 @@ export class ByteStreamReader extends BaseStreamReader { * A class to read chunks from a ReadableStream and provide them in a structured format. */ export class TextStreamReader extends BaseStreamReader { - /** Store a queue of chunks to be read. */ - private chunks: Array = []; - private chunkPublishedFuture = new Future<{ done: false, value: DataStream_Chunk } | { done: true, value?: undefined }, never>(); private receivedChunks: Map; - private receivedChunkDecompressionStreams: Map; - reader: ReadableStreamDefaultReader; - }> = new Map(); - signal?: AbortSignal; @@ -210,155 +201,56 @@ export class TextStreamReader extends BaseStreamReader { // Suppress unhandled rejection on reader.closed — errors are // already propagated through reader.read() to the consumer. reader.closed.catch(() => {}); + // Each chunk decodes independently: the sender splits uncompressed writes on UTF-8 boundaries, + // and compressed streams are reframed on UTF-8 boundaries by the decompression transform. const decoder = new TextDecoder('utf-8', { fatal: true }); const signal = this.signal; - const readNext = (reader: ReadableStreamDefaultReader, signal?: AbortSignal) => { - return new Promise>( - (resolve, reject) => { - if (signal) { - const onAbort = () => reject(signal.reason); - signal.addEventListener('abort', onAbort, { once: true }); - reader - .read() - .then(resolve, reject) - .finally(() => { - signal.removeEventListener('abort', onAbort); - }); - } else { - reader.read().then(resolve, reject); - } - }, - ); - }; - - const decodeChunkContents = (content: Uint8Array, chunkIndex: bigint) => { - let decodedResult; - try { - decodedResult = decoder.decode(content); - } catch (err) { - throw new DataStreamError( - `Cannot decode datastream chunk ${chunkIndex} as text: ${err}`, - DataStreamErrorReason.DecodeFailed, - ); - } - - return decodedResult; - }; - const cleanup = () => { reader.releaseLock(); this.signal = undefined; }; - // Prefill DecompressionStreams ahead of the iterator for fastest decompression performance - (async () => { - let lastChunkIndex: bigint | null = null; - while (true) { - try { - if (signal?.aborted) { - throw signal.reason; - } - const result = await readNext(reader, signal); - // console.log('RESULT', result); - if (result.done) { - this.chunkPublishedFuture.resolve?.({ done: true }); - break; - } else { - this.chunks.unshift(result.value); - this.chunkPublishedFuture.resolve?.({ done: false, value: result.value }); - this.chunkPublishedFuture = new Future(); - - const compressionIndex = result.value.iv?.[0] ?? 0; - if (compressionIndex > 0) { - let state = this.receivedChunkDecompressionStreams.get(compressionIndex); - if (!state) { - const stream = new DecompressionStream('gzip'); - state = { - stream, - writer: stream.writable.getWriter(), - reader: stream.readable.getReader(), - }; - this.receivedChunkDecompressionStreams.set(compressionIndex, state); - } - // console.log('WRITE CMP', compressionIndex, result.value.content); - state.writer.write(result.value.content as BufferSource); - } - - lastChunkIndex = result.value.chunkIndex; - } - } catch (err) { - cleanup(); - throw err; - } - } - })(); - return { next: async (): Promise> => { try { if (signal?.aborted) { throw signal.reason; } - - // Step 1: Get next chunk, either already pre-fetched in this.chunks, or if not then - // wait for the next one to be generated - let chunk = this.chunks.pop(); - if (!chunk) { - const { done, value } = await this.chunkPublishedFuture.promise; - if (done) { - this.validateBytesReceived(true); - return { done: true, value: undefined }; - } - this.chunks.pop(); // FIXME: maybe do this in a loop? - chunk = value; - } - // console.log('CHUNK', chunk); - - this.handleChunkReceived(chunk); - - let chunkContent = chunk.content; - - // Step 2: optionally decompress bu pulling the proper length in bytes from the - // corresponding DecompressionStream - const compressionIndex = chunk.iv?.[0] ?? 0; - const uncompressedByteLength = (((chunk.iv?.[1] ?? 0) & 0xff) << 8) | ((chunk.iv?.[2] ?? 0) & 0xff); - // console.log('COMPRESSION RATIO:', chunkContent.length / uncompressedByteLength); - if (compressionIndex > 0) { - // Chunk was compressed, so read the next `uncompressedByteLength` bytes - const decompressionState = this.receivedChunkDecompressionStreams.get(compressionIndex); - if (decompressionState) { - let combinedBuffer = new Uint8Array(uncompressedByteLength); - let offset = 0; - while (true) { - const { done, value } = await decompressionState.reader.read(); - // console.log('CMP READ:', done, value); - if (done) { - break; - } - if (offset + value.length > combinedBuffer.length) { - throw new Error(`uncompressedByteLength value was too short, espected to be able to fit at least ${value.length} bytes at offset ${offset}, but only had ${combinedBuffer.length} bytes of space`); - } - combinedBuffer.set(value, offset); - offset += value.length; - if (offset >= combinedBuffer.length) { - // FIXME: store value.slice(offset - uncompressedByteLength) and return on next read - break; - } + const result = await new Promise>( + (resolve, reject) => { + if (signal) { + const onAbort = () => reject(signal.reason); + signal.addEventListener('abort', onAbort, { once: true }); + reader + .read() + .then(resolve, reject) + .finally(() => { + signal.removeEventListener('abort', onAbort); + }); + } else { + reader.read().then(resolve, reject); } - chunkContent = combinedBuffer; - } + }, + ); + if (result.done) { + this.validateBytesReceived(true); + return { done: true, value: undefined as any }; } - // Step 3: Decode raw result back into text - // console.log('CNT', chunkContent); - const decodedResult = decodeChunkContents(chunkContent, chunk.chunkIndex); - // console.log('OUTPUT', decodedResult); + this.handleChunkReceived(result.value); + + let decodedResult: string; + try { + decodedResult = decoder.decode(result.value.content); + } catch (err) { + throw new DataStreamError( + `Cannot decode datastream chunk ${result.value.chunkIndex} as text: ${err}`, + DataStreamErrorReason.DecodeFailed, + ); + } - return { - done: false, - value: decodedResult, - }; + return { done: false, value: decodedResult }; } catch (err) { cleanup(); throw err; diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 89f70bb9dc..860265ca8b 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -19,9 +19,10 @@ import type { TextStreamInfo, } from '../../types'; import { encodeBase64, isCompressionStreamSupported, numberToBigInt, splitUtf8 } from '../../utils'; -import { gzipCompress, gzipCompressStream } from '../compression'; +import { StreamingDeflate, gzipCompress, gzipCompressStream } from '../compression'; import { COMPRESSION_ATTRIBUTE, + COMPRESSION_DEFLATE_RAW, COMPRESSION_GZIP, INLINE_PAYLOAD_ATTRIBUTE, STREAM_CHUNK_SIZE_BYTES, @@ -33,6 +34,8 @@ import { createStreamHeaderPacket, } from './header-utils'; +const textEncoder = new TextEncoder(); + /** * Manages sending custom user data via data channels. * @internal @@ -69,7 +72,7 @@ export default class OutgoingDataStreamManager { /** {@inheritDoc LocalParticipant.sendText} */ async sendText(text: string, options?: SendTextOptions): Promise { const streamId = crypto.randomUUID(); - const textInBytes = new TextEncoder().encode(text); + const textInBytes = textEncoder.encode(text); const totalTextLength = textInBytes.byteLength; // Fast path: when the full payload is known up front, there are no attachments, and the @@ -181,7 +184,7 @@ export default class OutgoingDataStreamManager { [INLINE_PAYLOAD_ATTRIBUTE]: text, }; if (isCompressionStreamSupported()) { - const raw = new TextEncoder().encode(text); + const raw = textEncoder.encode(text); const compressed = await gzipCompress(raw); if (compressed.byteLength < raw.byteLength) { inlineAttributes[INLINE_PAYLOAD_ATTRIBUTE] = encodeBase64(compressed); @@ -208,20 +211,19 @@ export default class OutgoingDataStreamManager { const streamId = options?.streamId ?? crypto.randomUUID(); const destinationIdentities = options?.destinationIdentities; const compressOption = options?.compress ?? true; - const compress = isCompressionStreamSupported() && compressOption && this.shouldCompress(destinationIdentities); + // The sender compresses with fflate (pure JS, always available), so eligibility is purely about + // the recipients: advertising data streams v2 means "can decompress a deflate-raw stream". + const compress = compressOption && this.allRecipientsSupportV2(destinationIdentities); const info: TextStreamInfo = { id: streamId, mimeType: 'text/plain', timestamp: Date.now(), topic: options?.topic ?? '', - // Compressed streams have an unknown total length up front, so it is left undefined and the - // trailer signals completion (see receiver validation). - // - // FIXME: make this instead the size before compression maybe? - size: compress ? undefined : options?.totalSize, + // Size is the pre-compression byte length; the receiver counts decompressed bytes against it. + size: options?.totalSize, attributes: compress - ? { ...options?.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_GZIP } + ? { ...options?.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW } : options?.attributes, encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled ? Encryption_Type.GCM @@ -233,65 +235,55 @@ export default class OutgoingDataStreamManager { await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); let chunkId = 0; - let compressionIndex = 1; const engine = this.engine; + // One deflate context for the whole stream: the dictionary persists across writes, and each + // write's output is sync-flushed so the receiver can decode it on arrival (see StreamingDeflate). + const compressor = compress ? new StreamingDeflate() : undefined; + + // Sends `bytes` as one or more streamChunk packets, splitting at the MTU budget. Writes are + // already serialized by the WritableStream, so no extra locking is needed. + const sendChunks = async (bytes: Uint8Array) => { + let byteOffset = 0; + while (byteOffset < bytes.byteLength) { + const subChunk = bytes.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); + const chunkPacket = new DataPacket({ + destinationIdentities, + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + content: subChunk, + streamId, + chunkIndex: numberToBigInt(chunkId), + }), + }, + }); + await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); + chunkId += 1; + byteOffset += subChunk.byteLength; + } + }; + const writableStream = new WritableStream({ - // Uncompressed path: split each write on UTF-8 boundaries so every chunk decodes - // independently on the receiver (required for pre-v2 receivers). async write(text) { - // console.log('WRITE', text); - const writeMutex = new Mutex(); - const sendChunkPacket = async (chunk: Uint8Array, uncompressedByteLength: number, compressionIndex: number = 0) => { - const unlock = await writeMutex.lock(); - - let byteOffset = 0; - try { - while (byteOffset < chunk.byteLength) { - const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); - const chunkPacket = new DataPacket({ - destinationIdentities, - value: { - case: 'streamChunk', - value: new DataStream_Chunk({ - content: subChunk, - streamId, - chunkIndex: numberToBigInt(chunkId), - iv: new Uint8Array([ // FIXME: swap with dedicated fields! - compressionIndex, - (uncompressedByteLength >> 8) & 0xff, - uncompressedByteLength & 0xff, - ]), - }), - }, - }); - // console.log('PACKET', chunkPacket); - await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); - chunkId += 1; - byteOffset += subChunk.byteLength; - } - } finally { - unlock(); - } - }; - - // Try to compress data if compression is supported - if (compress) { - const raw = new TextEncoder().encode(text); - const compressed = await gzipCompress(raw); - if (compressed.byteLength < raw.byteLength) { - await sendChunkPacket(compressed, raw.length, compressionIndex); - compressionIndex += 1; - return; + if (compressor) { + await sendChunks(compressor.compressWrite(textEncoder.encode(text))); + } else { + // Uncompressed path: split each write on UTF-8 boundaries so every chunk decodes + // independently on the receiver (required for pre-v2 receivers). + for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { + await sendChunks(textByteChunk); } } - - // Fallback to old uncompressed path if compression isn't possible / doesn't make it smaller - for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { - await sendChunkPacket(textByteChunk, textByteChunk.length, 0); - } }, async close() { + if (compressor) { + // Terminate the deflate stream so the receiver's decompressor can close cleanly. + const tail = compressor.end(); + if (tail.byteLength > 0) { + await sendChunks(tail); + } + } await sendStreamTrailer(streamId, destinationIdentities, engine); }, abort(err) { From 09e2fa1e87e52abe34ef95a5884ab55186dbb008 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 10 Jun 2026 15:18:14 -0400 Subject: [PATCH 15/68] refactor: move to deflate-raw for all compression, move to CompressionStream for single packet sends So now fflate is only used in the multi packet data stream case, and could be more easily broken out into its own concrete dependency block. --- .../data-stream/compression-roundtrip.test.ts | 73 ++++++++++++++++ src/room/data-stream/compression.test.ts | 22 ++--- src/room/data-stream/compression.ts | 86 ++++++++----------- src/room/data-stream/constants.ts | 25 ++++-- .../incoming/IncomingDataStreamManager.ts | 29 ++++--- .../outgoing/OutgoingDataStreamManager.ts | 66 +++++++++++--- src/room/types.ts | 9 ++ 7 files changed, 219 insertions(+), 91 deletions(-) diff --git a/src/room/data-stream/compression-roundtrip.test.ts b/src/room/data-stream/compression-roundtrip.test.ts index 8685b3816c..1e81530106 100644 --- a/src/room/data-stream/compression-roundtrip.test.ts +++ b/src/room/data-stream/compression-roundtrip.test.ts @@ -3,6 +3,7 @@ import { describe, expect, it, vi } from 'vitest'; import log from '../../logger'; import { CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DATA_STREAM_V2 } from '../../version'; import type RTCEngine from '../RTCEngine'; +import { StreamingDeflate } from './compression'; import IncomingDataStreamManager from './incoming/IncomingDataStreamManager'; import type { ByteStreamReader, TextStreamReader } from './incoming/StreamReader'; import OutgoingDataStreamManager from './outgoing/OutgoingDataStreamManager'; @@ -284,6 +285,78 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { expect(await reader.readAll()).toBe(expected); }); + it('compresses the sendText chunked fallback with the platform compressor, not fflate', async () => { + const { manager, sentPackets } = createSender(); + const fflateSpy = vi.spyOn(StreamingDeflate.prototype, 'compressWrite'); + + try { + // High entropy → too big to inline even compressed → falls back to the chunked stream. + let payload = ''; + while (payload.length < 60_000) { + payload += Math.random().toString(36).slice(2); + } + await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); + + expect(fflateSpy).not.toHaveBeenCalled(); + const header = sentPackets.find((p) => p.value.case === 'streamHeader'); + const headerValue = header!.value.value as Extract< + DataPacket['value'], + { case: 'streamHeader' } + >['value']; + expect(headerValue.attributes['lk.compression']).toBe('deflate-raw'); + + // The wire format is identical, so the same receiver path decodes it. + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe(payload); + } finally { + fflateSpy.mockRestore(); + } + }); + + it('still uses fflate for multi-write streamText streams', async () => { + const { manager, sentPackets } = createSender(); + const fflateSpy = vi.spyOn(StreamingDeflate.prototype, 'compressWrite'); + + try { + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); + await writer.write('one '); + await writer.write('two'); + await writer.close(); + + expect(fflateSpy).toHaveBeenCalledTimes(2); + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe('one two'); + } finally { + fflateSpy.mockRestore(); + } + }); + + it('rejects a second write on a singleWrite stream', async () => { + const { manager } = createSender(); + + const writer = await manager.streamText({ + topic: 't', + destinationIdentities: [RECEIVER], + singleWrite: true, + }); + await writer.write('the only write'); + await expect(writer.write('one too many')).rejects.toThrow(/more than once/); + }); + + it('round-trips a singleWrite stream closed without any writes', async () => { + const { manager, sentPackets } = createSender(); + + const writer = await manager.streamText({ + topic: 't', + destinationIdentities: [RECEIVER], + singleWrite: true, + }); + await writer.close(); + + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe(''); + }); + it('does not compress for a pre-v2 recipient (uncompressed round-trip)', async () => { const { manager, sentPackets } = createSender(CLIENT_PROTOCOL_DATA_STREAM_RPC); const payload = 'plain text '.repeat(2_000); diff --git a/src/room/data-stream/compression.test.ts b/src/room/data-stream/compression.test.ts index e477b46efa..87da69ab94 100644 --- a/src/room/data-stream/compression.test.ts +++ b/src/room/data-stream/compression.test.ts @@ -1,9 +1,8 @@ import { describe, expect, it } from 'vitest'; import { StreamingDeflate, - gzipCompress, - gzipDecompress, - gzipDecompressStream, + deflateRawCompress, + deflateRawDecompress, inflateRawStream, } from './compression'; @@ -47,34 +46,35 @@ async function collect(stream: ReadableStream): Promise return out; } -describe('data-stream gzip helpers', () => { +describe('data-stream buffered deflate-raw helpers (inline payloads)', () => { it('round-trips a buffered payload', async () => { const original = bytes('the quick brown fox '.repeat(500)); - const restored = await gzipDecompress(await gzipCompress(original)); + const restored = await deflateRawDecompress(await deflateRawCompress(original)); expect(text(restored)).toBe(text(original)); }); it('actually compresses repetitive data', async () => { const original = bytes('A'.repeat(50_000)); - const compressed = await gzipCompress(original); + const compressed = await deflateRawCompress(original); expect(compressed.byteLength).toBeLessThan(original.byteLength); }); it('round-trips an empty payload', async () => { - const restored = await gzipDecompress(await gzipCompress(new Uint8Array(0))); + const restored = await deflateRawDecompress(await deflateRawCompress(new Uint8Array(0))); expect(restored.byteLength).toBe(0); }); - it('streams decompression of a payload split across many compressed input chunks', async () => { + it('streams decompression of a one-shot payload split across many input chunks', async () => { const original = bytes('hello compressed world '.repeat(2_000)); - const compressed = await gzipCompress(original); + const compressed = await deflateRawCompress(original); - // Feed the compressed bytes in small slices to exercise incremental decompression. + // Feed the compressed bytes in small slices to exercise incremental decompression - a one-shot + // buffer is also a valid input for the streaming decompressor. const slices: Uint8Array[] = []; for (let i = 0; i < compressed.byteLength; i += 100) { slices.push(compressed.slice(i, i + 100)); } - const restored = await collect(gzipDecompressStream(streamOf(...slices))); + const restored = await collect(inflateRawStream(streamOf(...slices))); expect(text(restored)).toBe(text(original)); }); }); diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index 89bca71d58..dab93bccba 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -1,15 +1,15 @@ /** - * gzip compression helpers for data streams, built on the platform `CompressionStream` / - * `DecompressionStream`. The buffered variants are for the inline (single-packet) case where the - * payload is small and bounded; {@link gzipDecompressStream} streams for the chunked (multi-packet) - * case, consuming compressed input and producing decompressed output incrementally rather than - * buffering it all. + * Compression helpers for data streams. The buffered deflate-raw variants are for the inline + * (single-packet) case where the payload is small and bounded; {@link StreamingDeflate} / + * {@link inflateRawStream} serve the chunked (multi-packet) case, producing/consuming compressed + * data incrementally rather than buffering it all. {@link gzipCompressStream} remains for the + * legacy chunked byte-stream scheme (one gzip member per write). * * These operate on bytes (not strings) so a single set of helpers serves both text and byte streams; * the `TextEncoder`/`TextDecoder` boundary lives at the manager/reader edges. * - * Like the rest of the SDK, these drive `getWriter()`/`getReader()` directly instead of - * `pipeThrough`, which sidesteps the `CompressionStream` lib-type mismatches. + * Like the rest of the SDK, the platform-stream helpers drive `getWriter()`/`getReader()` directly + * instead of `pipeThrough`, which sidesteps the `CompressionStream` lib-type mismatches. * * @internal */ @@ -67,6 +67,21 @@ export class StreamingDeflate { } } +/** + * Compresses a fully-known payload into a raw-deflate stream, exposing the compressed output as a + * readable so callers can forward it incrementally. Produces the same wire format as + * {@link StreamingDeflate} (a raw-deflate stream terminated by a final block) but via the platform + * `CompressionStream` — usable only when the entire payload is written at once, since the platform + * compressor cannot flush mid-stream (its only "flush" is the close at the end). + */ +export function deflateRawCompressStream(data: Uint8Array): ReadableStream { + const cs = new CompressionStream('deflate-raw'); + const writer = cs.writable.getWriter(); + writer.write(data as NonSharedUint8Array); + writer.close(); + return cs.readable; +} + /** * Streams raw-deflate input through a single `DecompressionStream('deflate-raw')` for the lifetime * of the stream (inverse of {@link StreamingDeflate}). Inflate emits output greedily, so as long as @@ -97,16 +112,25 @@ export function inflateRawStream(input: ReadableStream): ReadableStr return ds.readable; } -/** gzip-compresses a byte array in full. Use for inline payloads; prefer the streaming path for the - * chunked case. */ -export async function gzipCompress(data: Uint8Array): Promise { - const cs = new CompressionStream('gzip'); +/** deflate-raw compresses a byte array in full. Use for inline payloads; prefer the streaming + * path for the chunked case. */ +export async function deflateRawCompress(data: Uint8Array): Promise { + const cs = new CompressionStream('deflate-raw'); const writer = cs.writable.getWriter(); writer.write(data as NonSharedUint8Array); writer.close(); return collect(cs.readable); } +/** Decompresses a raw-deflate byte array in full (inverse of {@link deflateRawCompress}). */ +export async function deflateRawDecompress(data: Uint8Array): Promise { + const ds = new DecompressionStream('deflate-raw'); + const writer = ds.writable.getWriter(); + writer.write(data as NonSharedUint8Array); + writer.close(); + return collect(ds.readable); +} + /** * gzip-compresses a byte array, exposing the compressed output as a readable stream so callers can * forward it incrementally instead of buffering the whole result. The input is written in full (the @@ -120,46 +144,6 @@ export function gzipCompressStream(data: Uint8Array): ReadableStream return cs.readable; } -/** gunzips a byte array in full (inverse of {@link gzipCompress}). */ -export async function gzipDecompress(data: Uint8Array): Promise { - const ds = new DecompressionStream('gzip'); - const writer = ds.writable.getWriter(); - writer.write(data as NonSharedUint8Array); - writer.close(); - return collect(ds.readable); -} - -/** - * Streams a gzip-compressed byte stream through decompression, feeding compressed chunks into the - * `DecompressionStream` as they arrive and exposing the decompressed output as a readable. Source - * errors are forwarded by aborting the decompression input. - */ -export function gzipDecompressStream( - input: ReadableStream, -): ReadableStream { - const ds = new DecompressionStream('gzip'); - const writer = ds.writable.getWriter(); - // Drive compressed input into the decompressor in the background; the IIFE handles its own errors - // by aborting the writable (which surfaces on the readable side), so this never rejects. - const pipe = (async () => { - const reader = input.getReader(); - try { - for (;;) { - const { done, value } = await reader.read(); - if (done) { - break; - } - await writer.write(value as NonSharedUint8Array); - } - await writer.close(); - } catch (err) { - await writer.abort(err).catch(() => {}); - } - })(); - pipe.catch(() => {}); - return ds.readable; -} - /** Concatenates all chunks of a byte stream into one array. */ async function collect(stream: ReadableStream): Promise { const reader = stream.getReader(); diff --git a/src/room/data-stream/constants.ts b/src/room/data-stream/constants.ts index f20480c156..bb18ee842f 100644 --- a/src/room/data-stream/constants.ts +++ b/src/room/data-stream/constants.ts @@ -19,21 +19,32 @@ export const STREAM_CHUNK_SIZE_BYTES = 15_000; /** * Reserved data-stream header attribute signaling that the payload (inline or chunked) is * compressed. Self-describing: the sender sets it when it compresses, and the receiver decompresses - * iff it is present. The only supported value today is {@link COMPRESSION_GZIP}. + * iff it is present. Inline payloads and chunked text streams use + * {@link COMPRESSION_DEFLATE_RAW}; chunked byte streams still use the legacy + * {@link COMPRESSION_GZIP} member scheme. * * @internal */ export const COMPRESSION_ATTRIBUTE = 'lk.compression'; -/** Value of {@link COMPRESSION_ATTRIBUTE} for gzip-compressed payloads. @internal */ +/** + * Value of {@link COMPRESSION_ATTRIBUTE} for the legacy chunked byte-stream scheme: each `write()` + * is its own complete gzip member, tagged with a member index in the chunk `version` field. + * Slated to migrate to {@link COMPRESSION_DEFLATE_RAW}. + * + * @internal + */ export const COMPRESSION_GZIP = 'gzip'; /** - * Value of {@link COMPRESSION_ATTRIBUTE} for chunked streams compressed with a single raw-deflate - * context shared across the whole stream. The sender sync-flushes at every write boundary so the - * receiver can decompress each chunk as it arrives, and terminates the deflate stream with a final - * block before the trailer. Receivers concatenate chunk contents in `chunkIndex` order through one - * raw-deflate (windowBits -15) decompressor. + * Value of {@link COMPRESSION_ATTRIBUTE} for raw-deflate-compressed payloads. + * + * For inline (single-packet) payloads this is a one-shot raw-deflate buffer, base64'd into the + * payload attribute. For chunked streams it is a single raw-deflate context shared across the whole + * stream: the sender sync-flushes at every write boundary so the receiver can decompress each chunk + * as it arrives, and terminates the deflate stream with a final block before the trailer. Receivers + * concatenate chunk contents in `chunkIndex` order through one raw-deflate (windowBits -15) + * decompressor. * * @internal */ diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 2c163ebb44..89da9f5a40 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -9,7 +9,7 @@ import log from '../../../logger'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { type ByteStreamInfo, type StreamController, type TextStreamInfo } from '../../types'; import { bigIntToNumber, decodeBase64, numberToBigInt } from '../../utils'; -import { gzipDecompress, inflateRawStream } from '../compression'; +import { deflateRawDecompress, inflateRawStream } from '../compression'; import { COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RAW, @@ -165,6 +165,10 @@ export default class IncomingDataStreamManager { encryptionType, }; + // Inline byte payloads are one-shot deflate-raw; chunked byte streams still use the legacy + // per-write gzip member scheme (see decompressedChunkStream). + const inlineCompressed = + info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_DEFLATE_RAW; const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; // Single-packet stream: the entire payload was smuggled into a reserved header attribute. @@ -177,7 +181,10 @@ export default class IncomingDataStreamManager { streamHandlerCallback( new ByteStreamReader( info, - createInlineStream(streamHeader.streamId, compressed ? gzipDecompress(bytes) : bytes), + createInlineStream( + streamHeader.streamId, + inlineCompressed ? deflateRawDecompress(bytes) : bytes, + ), bigIntToNumber(streamHeader.totalLength), ), { identity: participantIdentity }, @@ -244,11 +251,9 @@ export default class IncomingDataStreamManager { attachedStreamIds: streamHeader.contentHeader.value.attachedStreamIds, }; - // Inline payloads are compressed as one-shot gzip; chunked streams as a single raw-deflate - // stream spanning all chunks (see COMPRESSION_DEFLATE_RAW). - const inlineCompressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; - const streamCompressed = - info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_DEFLATE_RAW; + // Both inline and chunked text payloads are deflate-raw compressed; inline as a one-shot + // buffer, chunked as a single stream spanning all chunks (see COMPRESSION_DEFLATE_RAW). + const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_DEFLATE_RAW; // Single-packet stream: the entire payload was smuggled into a reserved header attribute. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. @@ -256,9 +261,9 @@ export default class IncomingDataStreamManager { if (typeof inlinePayload !== 'undefined') { delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; delete info.attributes![COMPRESSION_ATTRIBUTE]; - // Compressed text is base64(gzip(utf-8)); uncompressed text is the raw string. - const content = inlineCompressed - ? gzipDecompress(decodeBase64(inlinePayload)) + // Compressed text is base64(deflate-raw(utf-8)); uncompressed text is the raw string. + const content = compressed + ? deflateRawDecompress(decodeBase64(inlinePayload)) : new TextEncoder().encode(inlinePayload); streamHandlerCallback( new TextStreamReader( @@ -271,7 +276,7 @@ export default class IncomingDataStreamManager { return; } - if (streamCompressed) { + if (compressed) { delete info.attributes![COMPRESSION_ATTRIBUTE]; } @@ -297,7 +302,7 @@ export default class IncomingDataStreamManager { streamHandlerCallback( new TextStreamReader( info, - streamCompressed ? inflateRawChunkStream(stream, streamHeader.streamId) : stream, + compressed ? inflateRawChunkStream(stream, streamHeader.streamId) : stream, // `totalLength` is the pre-compression size, and the reader sees decompressed bytes, so // it applies to both paths. bigIntToNumber(streamHeader.totalLength), diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 860265ca8b..6066f66d8e 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -19,7 +19,12 @@ import type { TextStreamInfo, } from '../../types'; import { encodeBase64, isCompressionStreamSupported, numberToBigInt, splitUtf8 } from '../../utils'; -import { StreamingDeflate, gzipCompress, gzipCompressStream } from '../compression'; +import { + StreamingDeflate, + deflateRawCompress, + deflateRawCompressStream, + gzipCompressStream, +} from '../compression'; import { COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RAW, @@ -102,6 +107,8 @@ export default class OutgoingDataStreamManager { topic: options?.topic, attachedStreamIds: fileIds, attributes: options?.attributes, + // The full payload goes out in the single write below, so the platform compressor suffices. + singleWrite: true, }); await writer.write(text); @@ -143,7 +150,8 @@ export default class OutgoingDataStreamManager { ); } - /** Whether to gzip-compress a stream: all recipients support v2 and the runtime can compress. */ + /** Whether to compress a chunked byte stream (legacy gzip-member scheme): all recipients support + * v2 and the runtime can compress. */ private shouldCompress(destinationIdentities?: Array): boolean { return this.allRecipientsSupportV2(destinationIdentities) && isCompressionStreamSupported(); } @@ -177,18 +185,19 @@ export default class OutgoingDataStreamManager { }; // Compress when the runtime supports it, but only keep the result if it actually shrinks the - // payload (gzip adds ~18 bytes of framing, so tiny strings get larger). Uncompressed inline - // payloads stay as the raw string; compressed ones are base64'd and flagged via an attribute. + // payload (deflate framing plus the base64 expansion makes tiny strings larger). Uncompressed + // inline payloads stay as the raw string; compressed ones are base64'd and flagged via an + // attribute. const inlineAttributes: Record = { ...info.attributes, [INLINE_PAYLOAD_ATTRIBUTE]: text, }; if (isCompressionStreamSupported()) { const raw = textEncoder.encode(text); - const compressed = await gzipCompress(raw); + const compressed = await deflateRawCompress(raw); if (compressed.byteLength < raw.byteLength) { inlineAttributes[INLINE_PAYLOAD_ATTRIBUTE] = encodeBase64(compressed); - inlineAttributes[COMPRESSION_ATTRIBUTE] = COMPRESSION_GZIP; + inlineAttributes[COMPRESSION_ATTRIBUTE] = COMPRESSION_DEFLATE_RAW; } } @@ -237,9 +246,17 @@ export default class OutgoingDataStreamManager { let chunkId = 0; const engine = this.engine; - // One deflate context for the whole stream: the dictionary persists across writes, and each - // write's output is sync-flushed so the receiver can decode it on arrival (see StreamingDeflate). - const compressor = compress ? new StreamingDeflate() : undefined; + // When the whole payload arrives in a single write (the sendText fallback), the platform + // compressor produces the identical wire format without fflate — its inability to flush + // mid-stream doesn't matter because the only flush is the close at the end. + const oneShotCompress = + compress && options?.singleWrite === true && isCompressionStreamSupported(); + + // Otherwise, one deflate context for the whole stream: the dictionary persists across writes, + // and each write's output is sync-flushed so the receiver can decode it on arrival (see + // StreamingDeflate). + const compressor = compress && !oneShotCompress ? new StreamingDeflate() : undefined; + let wroteOnce = false; // Sends `bytes` as one or more streamChunk packets, splitting at the MTU budget. Writes are // already serialized by the WritableStream, so no extra locking is needed. @@ -266,7 +283,25 @@ export default class OutgoingDataStreamManager { const writableStream = new WritableStream({ async write(text) { - if (compressor) { + if (oneShotCompress) { + if (wroteOnce) { + // @throws-transformer ignore + throw new Error( + 'streamText was opened with singleWrite, but write() was called more than once', + ); + } + wroteOnce = true; + // Forward compressed output as it is produced; closing the compressor's input emits the + // terminating final block as part of the output, so close() below only sends the trailer. + const reader = deflateRawCompressStream(textEncoder.encode(text)).getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + await sendChunks(value); + } + } else if (compressor) { await sendChunks(compressor.compressWrite(textEncoder.encode(text))); } else { // Uncompressed path: split each write on UTF-8 boundaries so every chunk decodes @@ -283,6 +318,17 @@ export default class OutgoingDataStreamManager { if (tail.byteLength > 0) { await sendChunks(tail); } + } else if (oneShotCompress && !wroteOnce) { + // Closed without ever writing: emit a valid empty deflate stream (just a final block) so + // the receiver's decompressor still terminates cleanly. + const reader = deflateRawCompressStream(new Uint8Array(0)).getReader(); + for (;;) { + const { done, value } = await reader.read(); + if (done) { + break; + } + await sendChunks(value); + } } await sendStreamTrailer(streamId, destinationIdentities, engine); }, diff --git a/src/room/types.ts b/src/room/types.ts index fc91dde06f..a21f3b68ba 100644 --- a/src/room/types.ts +++ b/src/room/types.ts @@ -34,6 +34,15 @@ export interface StreamTextOptions { replyToStreamId?: string; totalSize?: number; attributes?: Record; + /** + * Promise that the stream's entire payload will arrive in exactly ONE `write()` call before + * `close()`. Lets the sender compress with a one-shot platform `CompressionStream` (whose only + * "flush" is the close at the end) instead of the per-write-flushable deflate. Writing more than + * once with this set is an error. + * + * @internal + */ + singleWrite?: boolean; } export type StreamBytesOptions = { From b8aaf48fa233351c0ac0ec7af0f570679b977462 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 10 Jun 2026 15:44:50 -0400 Subject: [PATCH 16/68] fix: get rid of singleWrite mode on streamText --- .../outgoing/OutgoingDataStreamManager.ts | 43 +------------------ src/room/types.ts | 9 ---- 2 files changed, 2 insertions(+), 50 deletions(-) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 6066f66d8e..158525aa1d 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -22,7 +22,6 @@ import { encodeBase64, isCompressionStreamSupported, numberToBigInt, splitUtf8 } import { StreamingDeflate, deflateRawCompress, - deflateRawCompressStream, gzipCompressStream, } from '../compression'; import { @@ -107,8 +106,6 @@ export default class OutgoingDataStreamManager { topic: options?.topic, attachedStreamIds: fileIds, attributes: options?.attributes, - // The full payload goes out in the single write below, so the platform compressor suffices. - singleWrite: true, }); await writer.write(text); @@ -246,17 +243,10 @@ export default class OutgoingDataStreamManager { let chunkId = 0; const engine = this.engine; - // When the whole payload arrives in a single write (the sendText fallback), the platform - // compressor produces the identical wire format without fflate — its inability to flush - // mid-stream doesn't matter because the only flush is the close at the end. - const oneShotCompress = - compress && options?.singleWrite === true && isCompressionStreamSupported(); - // Otherwise, one deflate context for the whole stream: the dictionary persists across writes, // and each write's output is sync-flushed so the receiver can decode it on arrival (see // StreamingDeflate). - const compressor = compress && !oneShotCompress ? new StreamingDeflate() : undefined; - let wroteOnce = false; + const compressor = compress ? new StreamingDeflate() : undefined; // Sends `bytes` as one or more streamChunk packets, splitting at the MTU budget. Writes are // already serialized by the WritableStream, so no extra locking is needed. @@ -283,25 +273,7 @@ export default class OutgoingDataStreamManager { const writableStream = new WritableStream({ async write(text) { - if (oneShotCompress) { - if (wroteOnce) { - // @throws-transformer ignore - throw new Error( - 'streamText was opened with singleWrite, but write() was called more than once', - ); - } - wroteOnce = true; - // Forward compressed output as it is produced; closing the compressor's input emits the - // terminating final block as part of the output, so close() below only sends the trailer. - const reader = deflateRawCompressStream(textEncoder.encode(text)).getReader(); - for (;;) { - const { done, value } = await reader.read(); - if (done) { - break; - } - await sendChunks(value); - } - } else if (compressor) { + if (compressor) { await sendChunks(compressor.compressWrite(textEncoder.encode(text))); } else { // Uncompressed path: split each write on UTF-8 boundaries so every chunk decodes @@ -318,17 +290,6 @@ export default class OutgoingDataStreamManager { if (tail.byteLength > 0) { await sendChunks(tail); } - } else if (oneShotCompress && !wroteOnce) { - // Closed without ever writing: emit a valid empty deflate stream (just a final block) so - // the receiver's decompressor still terminates cleanly. - const reader = deflateRawCompressStream(new Uint8Array(0)).getReader(); - for (;;) { - const { done, value } = await reader.read(); - if (done) { - break; - } - await sendChunks(value); - } } await sendStreamTrailer(streamId, destinationIdentities, engine); }, diff --git a/src/room/types.ts b/src/room/types.ts index a21f3b68ba..fc91dde06f 100644 --- a/src/room/types.ts +++ b/src/room/types.ts @@ -34,15 +34,6 @@ export interface StreamTextOptions { replyToStreamId?: string; totalSize?: number; attributes?: Record; - /** - * Promise that the stream's entire payload will arrive in exactly ONE `write()` call before - * `close()`. Lets the sender compress with a one-shot platform `CompressionStream` (whose only - * "flush" is the close at the end) instead of the per-write-flushable deflate. Writing more than - * once with this set is an error. - * - * @internal - */ - singleWrite?: boolean; } export type StreamBytesOptions = { From fe33ba5f870031bdbd05f31e75780ad826f5c4c8 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 11 Jun 2026 10:48:03 -0400 Subject: [PATCH 17/68] fix: clean up code --- src/room/data-stream/compression.ts | 2 +- src/room/data-stream/incoming/IncomingDataStreamManager.ts | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index dab93bccba..9f85e5047e 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -96,7 +96,7 @@ export function inflateRawStream(input: ReadableStream): ReadableStr const pipe = (async () => { const reader = input.getReader(); try { - for (;;) { + while (true) { const { done, value } = await reader.read(); if (done) { break; diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 89da9f5a40..2d0cf6ecd2 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -429,7 +429,7 @@ function inflateRawChunkStream( let lastChunkIndex = -1; const compressedBytes = new ReadableStream({ pull: async (controller) => { - for (;;) { + while (true) { const { done, value } = await srcReader.read(); if (done) { controller.close(); @@ -476,12 +476,13 @@ function inflateRawChunkStream( return new ReadableStream({ pull: async (controller) => { - for (;;) { + while (true) { const { done, value } = await decompressedReader.read(); if (done) { const tail = decodeOrThrow(); if (tail.length > 0) { - controller.enqueue(makeChunk(streamId, outIndex++, encoder.encode(tail))); + controller.enqueue(makeChunk(streamId, outIndex, encoder.encode(tail))); + outIndex += 1; } controller.close(); return; From 3ca6f5f17a32ba9f8553292ff42037a6b84d66c0 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 11 Jun 2026 16:49:45 -0400 Subject: [PATCH 18/68] feat: remove compression from streamText, in practice this doesn't really make much of a difference for agent transcriptions And I don't think it would be too hard to re-introduce in the future as an opt in setting, either with fflate, or if compressionstream gets an explicit "flush" option --- examples/data-stream-benchmark/benchmark.ts | 3 +- package.json | 1 - pnpm-lock.yaml | 8 - .../data-stream/compression-roundtrip.test.ts | 166 +++++++++--------- src/room/data-stream/compression.test.ts | 78 ++------ src/room/data-stream/compression.ts | 69 +------- .../outgoing/OutgoingDataStreamManager.ts | 160 +++++++++++------ src/room/types.ts | 1 - 8 files changed, 216 insertions(+), 270 deletions(-) diff --git a/examples/data-stream-benchmark/benchmark.ts b/examples/data-stream-benchmark/benchmark.ts index 9a3cfe9a22..86276122ac 100644 --- a/examples/data-stream-benchmark/benchmark.ts +++ b/examples/data-stream-benchmark/benchmark.ts @@ -17,7 +17,7 @@ const MAX_FILL_COUNT = BOX_DURATION_MS; const FILL_RGB = '52,152,219'; /** Chunk size to split up the data stream payload into. If `0`, send all at once with `sendText`. */ -const STREAM_CHUNK_SIZE_BYTES = 1000; +const STREAM_CHUNK_SIZE_BYTES = 0; const TOPIC = 'benchmark'; const SENDER_IDENTITY = 'bench-sender'; @@ -26,6 +26,7 @@ const RECEIVER_IDENTITY = 'bench-receiver'; const SIZES: Array<{ label: string; bytes: number }> = [ { label: '10 B', bytes: 10 }, { label: '100 B', bytes: 100 }, + { label: '512 B', bytes: 512 }, { label: '1 KB', bytes: 1_000 }, { label: '15 KB', bytes: 15_000 }, { label: '100 KB', bytes: 100_000 }, diff --git a/package.json b/package.json index f6146429db..2561dbc657 100644 --- a/package.json +++ b/package.json @@ -74,7 +74,6 @@ "@livekit/mutex": "1.1.1", "@livekit/protocol": "1.46.6", "events": "^3.3.0", - "fflate": "^0.8.3", "jose": "^6.1.0", "loglevel": "^1.9.2", "sdp-transform": "^2.15.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f45da23418..247d0397cf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,9 +29,6 @@ importers: events: specifier: ^3.3.0 version: 3.3.0 - fflate: - specifier: ^0.8.3 - version: 0.8.3 jose: specifier: ^6.1.0 version: 6.2.3 @@ -2519,9 +2516,6 @@ packages: picomatch: optional: true - fflate@0.8.3: - resolution: {integrity: sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==} - file-entry-cache@8.0.0: resolution: {integrity: sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ==} engines: {node: '>=16.0.0'} @@ -6562,8 +6556,6 @@ snapshots: optionalDependencies: picomatch: 4.0.4 - fflate@0.8.3: {} - file-entry-cache@8.0.0: dependencies: flat-cache: 4.0.1 diff --git a/src/room/data-stream/compression-roundtrip.test.ts b/src/room/data-stream/compression-roundtrip.test.ts index 1e81530106..65615e20d2 100644 --- a/src/room/data-stream/compression-roundtrip.test.ts +++ b/src/room/data-stream/compression-roundtrip.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it, vi } from 'vitest'; import log from '../../logger'; import { CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DATA_STREAM_V2 } from '../../version'; import type RTCEngine from '../RTCEngine'; -import { StreamingDeflate } from './compression'; import IncomingDataStreamManager from './incoming/IncomingDataStreamManager'; import type { ByteStreamReader, TextStreamReader } from './incoming/StreamReader'; import OutgoingDataStreamManager from './outgoing/OutgoingDataStreamManager'; @@ -11,6 +10,32 @@ import OutgoingDataStreamManager from './outgoing/OutgoingDataStreamManager'; const RECEIVER = 'bob'; const hasCompression = typeof CompressionStream !== 'undefined'; +/** High-entropy text: too big to inline even after compression, forcing the chunked fallback. */ +function randomText(length: number): string { + let s = ''; + while (s.length < length) { + s += Math.random().toString(36).slice(2); + } + return s.slice(0, length); +} + +/** High-entropy multibyte text (random CJK), so compressed output chunk boundaries fall + * mid-character and the payload stays over the inline budget even compressed. */ +function randomMultibyteText(chars: number): string { + let s = ''; + for (let i = 0; i < chars; i += 1) { + s += String.fromCharCode(0x4e00 + Math.floor(Math.random() * 0x51a5)); + } + return s; +} + +/** Total streamChunk content bytes across the captured packets. */ +function chunkContentBytes(packets: DataPacket[]): number { + return packets + .filter((p) => p.value.case === 'streamChunk') + .reduce((sum, p) => sum + (p.value.value as DataStream_Chunk).content.byteLength, 0); +} + /** An OutgoingDataStreamManager whose engine captures every sent packet. */ function createSender(recipientProtocol = CLIENT_PROTOCOL_DATA_STREAM_V2) { const sentPackets: DataPacket[] = []; @@ -82,10 +107,7 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { it('round-trips a large chunked compressed text payload (multi-packet)', async () => { const { manager, sentPackets } = createSender(); // High entropy → too big to inline even compressed → chunked + compressed. - let payload = ''; - while (payload.length < 60_000) { - payload += Math.random().toString(36).slice(2); - } + const payload = randomText(60_000); await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); @@ -96,12 +118,10 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { it('round-trips chunked compressed text with multibyte UTF-8 (reframing on char boundaries)', async () => { const { manager, sentPackets } = createSender(); - // Emoji + CJK so gzip/output chunk boundaries fall mid-character. - const payload = '日本語🚀テスト '.repeat(4_000); + // High-entropy CJK so compressed output chunk boundaries fall mid-character. + const payload = randomMultibyteText(30_000); - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - await writer.write(payload); - await writer.close(); + await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); const reader = await receiveText(sentPackets, 't'); @@ -124,7 +144,7 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { expect(Array.from(flatten(await reader.readAll()))).toEqual(Array.from(payload)); }); - it('round-trips text written across multiple writes (shared deflate context)', async () => { + it('round-trips text written across multiple writes (uncompressed streamText)', async () => { const { manager, sentPackets } = createSender(); const parts = ['first part ', '日本語🚀 second ', 'x'.repeat(20_000), ' tail']; @@ -158,12 +178,10 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { expect(Array.from(flatten(await reader.readAll()))).toEqual(expected); }); - it('marks chunked compressed text streams with the deflate-raw attribute', async () => { + it('marks the chunked compressed sendText fallback with the deflate-raw attribute', async () => { const { manager, sentPackets } = createSender(); - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - await writer.write('compressed contents'); - await writer.close(); + await manager.sendText(randomText(60_000), { topic: 't', destinationIdentities: [RECEIVER] }); const header = sentPackets.find((p) => p.value.case === 'streamHeader'); const headerValue = header!.value.value as Extract< @@ -227,10 +245,9 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => {}); try { - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - await writer.write('part one '); - await writer.write('part two'); - await writer.close(); + // Compressed chunked fallback: the stateful inflater is what needs dup protection. + const payload = randomText(60_000); + await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); // Replay the first chunk packet again right after the original. const firstChunkIdx = sentPackets.findIndex((p) => p.value.case === 'streamChunk'); @@ -238,7 +255,7 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { packets.splice(firstChunkIdx + 1, 0, packets[firstChunkIdx]); const reader = await receiveText(packets, 't'); - expect(await reader.readAll()).toBe('part one part two'); + expect(await reader.readAll()).toBe(payload); expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('duplicate chunk')); } finally { warnSpy.mockRestore(); @@ -248,10 +265,8 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { it('errors the stream when a chunk goes missing (gap in chunk indices)', async () => { const { manager, sentPackets } = createSender(); - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - await writer.write('part one '); - await writer.write('part two'); - await writer.close(); + // Compressed chunked fallback: the stateful inflater cannot tolerate gaps. + await manager.sendText(randomText(60_000), { topic: 't', destinationIdentities: [RECEIVER] }); // Drop the first chunk packet entirely. const firstChunkIdx = sentPackets.findIndex((p) => p.value.case === 'streamChunk'); @@ -261,7 +276,7 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { await expect(reader.readAll()).rejects.toThrow(/[Mm]issing chunk/); }); - it('round-trips a long stream of many small writes (transcription pattern)', async () => { + it('sends a long stream of many small writes uncompressed (transcription pattern)', async () => { const { manager, sentPackets } = createSender(); const writes = Array.from( { length: 500 }, @@ -275,103 +290,80 @@ describe.skipIf(!hasCompression)('data stream compression round-trip', () => { await writer.close(); const expected = writes.join(''); - // Context takeover should compress the repetitive writes well below the raw size. - const compressedTotal = sentPackets - .filter((p) => p.value.case === 'streamChunk') - .reduce((sum, p) => sum + (p.value.value as DataStream_Chunk).content.byteLength, 0); - expect(compressedTotal).toBeLessThan(expected.length / 3); + // streamText never compresses: the chunk contents are exactly the payload bytes. + expect(chunkContentBytes(sentPackets)).toBe(new TextEncoder().encode(expected).byteLength); const reader = await receiveText(sentPackets, 't'); expect(await reader.readAll()).toBe(expected); }); - it('compresses the sendText chunked fallback with the platform compressor, not fflate', async () => { + it('streamText never compresses, even for v2 recipients', async () => { const { manager, sentPackets } = createSender(); - const fflateSpy = vi.spyOn(StreamingDeflate.prototype, 'compressWrite'); - try { - // High entropy → too big to inline even compressed → falls back to the chunked stream. - let payload = ''; - while (payload.length < 60_000) { - payload += Math.random().toString(36).slice(2); - } - await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); + await writer.write('one '); + await writer.write('two'); + await writer.close(); - expect(fflateSpy).not.toHaveBeenCalled(); - const header = sentPackets.find((p) => p.value.case === 'streamHeader'); - const headerValue = header!.value.value as Extract< - DataPacket['value'], - { case: 'streamHeader' } - >['value']; - expect(headerValue.attributes['lk.compression']).toBe('deflate-raw'); + // No compression attribute on the header, and chunk contents are plain UTF-8. + const header = sentPackets.find((p) => p.value.case === 'streamHeader'); + const headerValue = header!.value.value as Extract< + DataPacket['value'], + { case: 'streamHeader' } + >['value']; + expect(headerValue.attributes['lk.compression']).toBeUndefined(); + const contents = sentPackets + .filter((p) => p.value.case === 'streamChunk') + .map((p) => new TextDecoder().decode((p.value.value as DataStream_Chunk).content)); + expect(contents.join('')).toBe('one two'); - // The wire format is identical, so the same receiver path decodes it. - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe(payload); - } finally { - fflateSpy.mockRestore(); - } + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe('one two'); }); - it('still uses fflate for multi-write streamText streams', async () => { + it('actually shrinks a compressible chunked sendText payload on the wire', async () => { const { manager, sentPackets } = createSender(); - const fflateSpy = vi.spyOn(StreamingDeflate.prototype, 'compressWrite'); - - try { - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - await writer.write('one '); - await writer.write('two'); - await writer.close(); - - expect(fflateSpy).toHaveBeenCalledTimes(2); - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe('one two'); - } finally { - fflateSpy.mockRestore(); + // Moderately compressible (small vocabulary) but high enough entropy that the compressed + // output still exceeds the inline budget → chunked fallback with real compression win. + const vocabulary = randomText(2_000).match(/.{1,8}/g)!; + let payload = ''; + while (payload.length < 200_000) { + payload += vocabulary[Math.floor(Math.random() * vocabulary.length)] + ' '; } - }); - it('rejects a second write on a singleWrite stream', async () => { - const { manager } = createSender(); + await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); - const writer = await manager.streamText({ - topic: 't', - destinationIdentities: [RECEIVER], - singleWrite: true, - }); - await writer.write('the only write'); - await expect(writer.write('one too many')).rejects.toThrow(/more than once/); + expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); + expect(chunkContentBytes(sentPackets)).toBeLessThan(payload.length / 2); + + const reader = await receiveText(sentPackets, 't'); + expect(await reader.readAll()).toBe(payload); }); - it('round-trips a singleWrite stream closed without any writes', async () => { + it('round-trips a streamText stream closed without any writes', async () => { const { manager, sentPackets } = createSender(); - const writer = await manager.streamText({ - topic: 't', - destinationIdentities: [RECEIVER], - singleWrite: true, - }); + const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); await writer.close(); const reader = await receiveText(sentPackets, 't'); expect(await reader.readAll()).toBe(''); }); - it('does not compress for a pre-v2 recipient (uncompressed round-trip)', async () => { + it('does not compress sendText for a pre-v2 recipient (uncompressed round-trip)', async () => { const { manager, sentPackets } = createSender(CLIENT_PROTOCOL_DATA_STREAM_RPC); const payload = 'plain text '.repeat(2_000); - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - await writer.write(payload); - await writer.close(); + await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); - // No compression attribute on the header. + // No compression attribute on the header, and the payload went out as raw bytes. const header = sentPackets.find((p) => p.value.case === 'streamHeader'); const headerValue = header!.value.value as Extract< DataPacket['value'], { case: 'streamHeader' } >['value']; expect(headerValue.attributes['lk.compression']).toBeUndefined(); + expect(chunkContentBytes(sentPackets)).toBe(new TextEncoder().encode(payload).byteLength); const reader = await receiveText(sentPackets, 't'); expect(await reader.readAll()).toBe(payload); diff --git a/src/room/data-stream/compression.test.ts b/src/room/data-stream/compression.test.ts index 87da69ab94..148706fd80 100644 --- a/src/room/data-stream/compression.test.ts +++ b/src/room/data-stream/compression.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { - StreamingDeflate, deflateRawCompress, + deflateRawCompressStream, deflateRawDecompress, inflateRawStream, } from './compression'; @@ -79,77 +79,33 @@ describe('data-stream buffered deflate-raw helpers (inline payloads)', () => { }); }); -describe('StreamingDeflate + inflateRawStream', () => { - it('round-trips multiple writes through a single decompressor', async () => { - const deflate = new StreamingDeflate(); - const writes = ['first write ', 'second write with 日本語🚀 ', 'third '.repeat(100)]; - const parts = writes.map((w) => deflate.compressWrite(bytes(w))); - parts.push(deflate.end()); - - const restored = await collect(inflateRawStream(streamOf(...parts))); - expect(text(restored)).toBe(writes.join('')); +describe('deflateRawCompressStream + inflateRawStream', () => { + it('round-trips a one-shot payload through the streaming decompressor', async () => { + const original = 'first part 日本語🚀 ' + 'repeated filler '.repeat(2_000); + const compressed = await collect(deflateRawCompressStream(bytes(original))); + const restored = await collect(inflateRawStream(streamOf(compressed))); + expect(text(restored)).toBe(original); }); - it("emits each write's content before any further input arrives (timeliness)", async () => { - const deflate = new StreamingDeflate(); - - let inputController!: ReadableStreamDefaultController; - const input = new ReadableStream({ - start(controller) { - inputController = controller; - }, - }); - const reader = inflateRawStream(input).getReader(); - const decoder = new TextDecoder(); - - const writes = ['hello there, ', 'hello again - repeating myself, hello hello ', 'bye']; - for (const write of writes) { - inputController.enqueue(deflate.compressWrite(bytes(write))); - // The write's full content must come out without sending anything further. - let got = ''; - while (got.length < write.length) { - const { done, value } = await reader.read(); - expect(done).toBe(false); - got += decoder.decode(value!, { stream: true }); - } - expect(got).toBe(write); - } - - inputController.enqueue(deflate.end()); - inputController.close(); - const { done } = await reader.read(); - expect(done).toBe(true); - }); - - it('reuses the compression context across writes (later similar writes shrink)', () => { - const deflate = new StreamingDeflate(); - const sentence = 'the quick brown fox jumps over the lazy dog and keeps on jumping. '; - const first = deflate.compressWrite(bytes(sentence.repeat(10))); - const second = deflate.compressWrite(bytes(sentence.repeat(10))); - // The second write is pure back-references into the persisted window. - expect(second.byteLength).toBeLessThan(first.byteLength / 4); + it('actually compresses repetitive data', async () => { + const original = bytes('the quick brown fox '.repeat(2_000)); + const compressed = await collect(deflateRawCompressStream(original)); + expect(compressed.byteLength).toBeLessThan(original.byteLength / 3); }); it('round-trips incompressible input', async () => { - const deflate = new StreamingDeflate(); const original = new Uint8Array(10_000); for (let i = 0; i < original.length; i += 1) { original[i] = Math.floor(Math.random() * 256); } - const parts = [deflate.compressWrite(original), deflate.end()]; - const restored = await collect(inflateRawStream(streamOf(...parts))); + const compressed = await collect(deflateRawCompressStream(original)); + const restored = await collect(inflateRawStream(streamOf(compressed))); expect(Array.from(restored)).toEqual(Array.from(original)); }); - it('handles an empty write', async () => { - const deflate = new StreamingDeflate(); - const parts = [ - deflate.compressWrite(bytes('before ')), - deflate.compressWrite(new Uint8Array(0)), - deflate.compressWrite(bytes('after')), - deflate.end(), - ]; - const restored = await collect(inflateRawStream(streamOf(...parts))); - expect(text(restored)).toBe('before after'); + it('round-trips an empty payload', async () => { + const compressed = await collect(deflateRawCompressStream(new Uint8Array(0))); + const restored = await collect(inflateRawStream(streamOf(compressed))); + expect(restored.byteLength).toBe(0); }); }); diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index 9f85e5047e..06b32deb94 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -1,9 +1,10 @@ /** * Compression helpers for data streams. The buffered deflate-raw variants are for the inline - * (single-packet) case where the payload is small and bounded; {@link StreamingDeflate} / - * {@link inflateRawStream} serve the chunked (multi-packet) case, producing/consuming compressed - * data incrementally rather than buffering it all. {@link gzipCompressStream} remains for the - * legacy chunked byte-stream scheme (one gzip member per write). + * (single-packet) case where the payload is small and bounded; {@link deflateRawCompressStream} / + * {@link inflateRawStream} serve the chunked (multi-packet) `sendText` fallback, where the whole + * payload is known up front but the compressed output is produced/consumed incrementally rather + * than buffered. {@link gzipCompressStream} remains for the legacy chunked byte-stream scheme + * (one gzip member per write). * * These operate on bytes (not strings) so a single set of helpers serves both text and byte streams; * the `TextEncoder`/`TextDecoder` boundary lives at the manager/reader edges. @@ -13,66 +14,12 @@ * * @internal */ -import { Deflate } from 'fflate'; - -/** - * Per-stream raw-deflate compressor for chunked data streams. One instance lives for the whole - * stream, so the LZ77 window persists across writes — repeated content in later writes compresses - * against earlier ones (the permessage-deflate "context takeover" model). Each - * {@link StreamingDeflate.compressWrite} output is flushed to a byte boundary, so a receiver - * feeding the bytes through a single raw-deflate decompressor can decode every write's content as - * soon as it arrives, without waiting for the stream to end. - * - * Built on fflate rather than `CompressionStream` because the platform compressor has no flush — - * it only emits buffered output on close, which would force a fresh (dictionary-reset) compressor - * per write. - */ -export class StreamingDeflate { - private pending: Array = []; - - private deflate = new Deflate((chunk) => { - this.pending.push(chunk); - }); - - /** Compresses one write. The returned bytes are byte-aligned (sync flush), so the receiver can - * decompress the write's full content immediately upon receiving them. */ - compressWrite(data: Uint8Array): Uint8Array { - this.deflate.push(data); - this.deflate.flush(); - return this.takePending(); - } - - /** Terminates the deflate stream with an empty final block. Call exactly once, after the last - * write; the returned bytes must be delivered so the receiver's decompressor can close cleanly. */ - end(): Uint8Array { - this.deflate.push(new Uint8Array(0), true); - return this.takePending(); - } - - private takePending(): Uint8Array { - if (this.pending.length === 1) { - const only = this.pending[0]; - this.pending = []; - return only; - } - const total = this.pending.reduce((sum, chunk) => sum + chunk.byteLength, 0); - const result = new Uint8Array(total); - let offset = 0; - for (const chunk of this.pending) { - result.set(chunk, offset); - offset += chunk.byteLength; - } - this.pending = []; - return result; - } -} /** * Compresses a fully-known payload into a raw-deflate stream, exposing the compressed output as a - * readable so callers can forward it incrementally. Produces the same wire format as - * {@link StreamingDeflate} (a raw-deflate stream terminated by a final block) but via the platform - * `CompressionStream` — usable only when the entire payload is written at once, since the platform - * compressor cannot flush mid-stream (its only "flush" is the close at the end). + * readable so callers can forward it incrementally. Usable only when the entire payload is written + * at once, since the platform compressor cannot flush mid-stream (its only "flush" is the close at + * the end) — incremental multi-write senders (`streamText`) therefore send uncompressed. */ export function deflateRawCompressStream(data: Uint8Array): ReadableStream { const cs = new CompressionStream('deflate-raw'); diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 158525aa1d..ba9d9c7493 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -19,11 +19,7 @@ import type { TextStreamInfo, } from '../../types'; import { encodeBase64, isCompressionStreamSupported, numberToBigInt, splitUtf8 } from '../../utils'; -import { - StreamingDeflate, - deflateRawCompress, - gzipCompressStream, -} from '../compression'; +import { deflateRawCompress, deflateRawCompressStream, gzipCompressStream } from '../compression'; import { COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RAW, @@ -78,6 +74,7 @@ export default class OutgoingDataStreamManager { const streamId = crypto.randomUUID(); const textInBytes = textEncoder.encode(text); const totalTextLength = textInBytes.byteLength; + const compress = options?.compress ?? true; // Fast path: when the full payload is known up front, there are no attachments, and the // payload fits (with header overhead) under the MTU, smuggle it into a reserved header @@ -99,20 +96,36 @@ export default class OutgoingDataStreamManager { options?.onProgress?.(totalProgress); }; - const writer = await this.streamText({ - streamId, - totalSize: totalTextLength, - destinationIdentities: options?.destinationIdentities, - topic: options?.topic, - attachedStreamIds: fileIds, - attributes: options?.attributes, - }); - - await writer.write(text); - // set text part of progress to 1 - handleProgress(1, 0); + let info: TextStreamInfo; + if (compress && this.shouldCompress(options?.destinationIdentities)) { + // The full payload is known up front, so the chunked fallback can compress it in one shot + // with the platform compressor (incremental writers can't compress; see streamText). + info = await this.sendChunkedCompressedText( + streamId, + text, + totalTextLength, + fileIds, + options, + ); + // set text part of progress to 1 + handleProgress(1, 0); + } else { + const writer = await this.streamText({ + streamId, + totalSize: totalTextLength, + destinationIdentities: options?.destinationIdentities, + topic: options?.topic, + attachedStreamIds: fileIds, + attributes: options?.attributes, + }); + + await writer.write(text); + // set text part of progress to 1 + handleProgress(1, 0); - await writer.close(); + await writer.close(); + info = writer.info; + } if (options?.attachments && fileIds) { await Promise.all( @@ -127,7 +140,7 @@ export default class OutgoingDataStreamManager { ), ); } - return writer.info; + return info; } /** @@ -147,8 +160,8 @@ export default class OutgoingDataStreamManager { ); } - /** Whether to compress a chunked byte stream (legacy gzip-member scheme): all recipients support - * v2 and the runtime can compress. */ + /** Whether to compress a chunked stream: all recipients support v2 and the runtime can + * compress. */ private shouldCompress(destinationIdentities?: Array): boolean { return this.allRecipientsSupportV2(destinationIdentities) && isCompressionStreamSupported(); } @@ -210,27 +223,86 @@ export default class OutgoingDataStreamManager { return info; } + /** + * Sends `text` as a compressed chunked stream: one raw-deflate stream spanning every chunk + * packet's content, terminated by the trailer. Used by the sendText fallback when the payload is + * too large to send inline — the full payload is known up front, so the platform compressor + * works even though it cannot flush mid-stream (the close is the only flush needed). Incremental + * writers (streamText) cannot use this and send uncompressed instead. + */ + private async sendChunkedCompressedText( + streamId: string, + text: string, + totalTextLength: number, + attachedStreamIds: Array | undefined, + options?: SendTextOptions, + ): Promise { + const destinationIdentities = options?.destinationIdentities; + const engine = this.engine; + + const info: TextStreamInfo = { + id: streamId, + mimeType: 'text/plain', + timestamp: Date.now(), + topic: options?.topic ?? '', + // Size is the pre-compression byte length; the receiver counts decompressed bytes against it. + size: totalTextLength, + attributes: { ...options?.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, + encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled + ? Encryption_Type.GCM + : Encryption_Type.NONE, + attachedStreamIds, + }; + const header = buildTextStreamHeader(info); + const packet = createStreamHeaderPacket(header, destinationIdentities); + await engine.sendDataPacket(packet, DataChannelKind.RELIABLE); + + // Forward compressed output as it is produced, splitting at the MTU budget. + let chunkId = 0; + const reader = deflateRawCompressStream(textEncoder.encode(text)).getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + let byteOffset = 0; + while (byteOffset < value.byteLength) { + const subChunk = value.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); + const chunkPacket = new DataPacket({ + destinationIdentities, + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + content: subChunk, + streamId, + chunkIndex: numberToBigInt(chunkId), + }), + }, + }); + await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); + chunkId += 1; + byteOffset += subChunk.byteLength; + } + } + + await sendStreamTrailer(streamId, destinationIdentities, engine); + return info; + } + /** * @internal */ async streamText(options?: StreamTextOptions): Promise { const streamId = options?.streamId ?? crypto.randomUUID(); const destinationIdentities = options?.destinationIdentities; - const compressOption = options?.compress ?? true; - // The sender compresses with fflate (pure JS, always available), so eligibility is purely about - // the recipients: advertising data streams v2 means "can decompress a deflate-raw stream". - const compress = compressOption && this.allRecipientsSupportV2(destinationIdentities); const info: TextStreamInfo = { id: streamId, mimeType: 'text/plain', timestamp: Date.now(), topic: options?.topic ?? '', - // Size is the pre-compression byte length; the receiver counts decompressed bytes against it. size: options?.totalSize, - attributes: compress - ? { ...options?.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW } - : options?.attributes, + attributes: options?.attributes, encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled ? Encryption_Type.GCM : Encryption_Type.NONE, @@ -243,11 +315,6 @@ export default class OutgoingDataStreamManager { let chunkId = 0; const engine = this.engine; - // Otherwise, one deflate context for the whole stream: the dictionary persists across writes, - // and each write's output is sync-flushed so the receiver can decode it on arrival (see - // StreamingDeflate). - const compressor = compress ? new StreamingDeflate() : undefined; - // Sends `bytes` as one or more streamChunk packets, splitting at the MTU budget. Writes are // already serialized by the WritableStream, so no extra locking is needed. const sendChunks = async (bytes: Uint8Array) => { @@ -273,24 +340,15 @@ export default class OutgoingDataStreamManager { const writableStream = new WritableStream({ async write(text) { - if (compressor) { - await sendChunks(compressor.compressWrite(textEncoder.encode(text))); - } else { - // Uncompressed path: split each write on UTF-8 boundaries so every chunk decodes - // independently on the receiver (required for pre-v2 receivers). - for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { - await sendChunks(textByteChunk); - } + // Incremental writers are never compressed (the platform compressor cannot flush + // mid-stream, and per-write flushing costs more than it saves at typical write sizes — + // see sendChunkedCompressedText for the one-shot compressed path). Split each write on + // UTF-8 boundaries so every chunk decodes independently on the receiver. + for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { + await sendChunks(textByteChunk); } }, async close() { - if (compressor) { - // Terminate the deflate stream so the receiver's decompressor can close cleanly. - const tail = compressor.end(); - if (tail.byteLength > 0) { - await sendChunks(tail); - } - } await sendStreamTrailer(streamId, destinationIdentities, engine); }, abort(err) { @@ -344,7 +402,9 @@ export default class OutgoingDataStreamManager { async streamBytes(options?: StreamBytesOptions) { const streamId = options?.streamId ?? crypto.randomUUID(); const destinationIdentities = options?.destinationIdentities; - const compressOption = options?.compress ?? true; + const compressOption = + options?.compress ?? + (typeof options?.totalSize === 'number' && options.totalSize > 2_000) /* bytes */; const compress = compressOption && this.shouldCompress(destinationIdentities); const info: ByteStreamInfo = { diff --git a/src/room/types.ts b/src/room/types.ts index fc91dde06f..83d01de666 100644 --- a/src/room/types.ts +++ b/src/room/types.ts @@ -26,7 +26,6 @@ export interface SendTextOptions { export interface StreamTextOptions { topic?: string; destinationIdentities?: Array; - compress?: boolean; type?: 'create' | 'update'; streamId?: string; version?: number; From e918ca207ef73bad8dc943d1f30f9807e9ed3666 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 12 Jun 2026 10:19:00 -0400 Subject: [PATCH 19/68] feat: add compression into bytestreams too --- src/room/data-stream/compression.ts | 20 +- src/room/data-stream/constants.ts | 24 +- .../incoming/IncomingDataStreamManager.ts | 208 ++++------- .../outgoing/OutgoingDataStreamManager.ts | 330 ++++++++++-------- src/room/types.ts | 7 +- 5 files changed, 268 insertions(+), 321 deletions(-) diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index 06b32deb94..cacde6a3f2 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -1,10 +1,9 @@ /** * Compression helpers for data streams. The buffered deflate-raw variants are for the inline * (single-packet) case where the payload is small and bounded; {@link deflateRawCompressStream} / - * {@link inflateRawStream} serve the chunked (multi-packet) `sendText` fallback, where the whole - * payload is known up front but the compressed output is produced/consumed incrementally rather - * than buffered. {@link gzipCompressStream} remains for the legacy chunked byte-stream scheme - * (one gzip member per write). + * {@link inflateRawStream} serve the chunked (multi-packet) `sendText`/`sendFile` fallback, where + * the whole payload is known up front but the compressed output is produced/consumed incrementally + * rather than buffered. * * These operate on bytes (not strings) so a single set of helpers serves both text and byte streams; * the `TextEncoder`/`TextDecoder` boundary lives at the manager/reader edges. @@ -78,19 +77,6 @@ export async function deflateRawDecompress(data: Uint8Array): Promise { - const cs = new CompressionStream('gzip'); - const writer = cs.writable.getWriter(); - writer.write(data as NonSharedUint8Array); - writer.close(); - return cs.readable; -} - /** Concatenates all chunks of a byte stream into one array. */ async function collect(stream: ReadableStream): Promise { const reader = stream.getReader(); diff --git a/src/room/data-stream/constants.ts b/src/room/data-stream/constants.ts index bb18ee842f..c4f7dcfdf0 100644 --- a/src/room/data-stream/constants.ts +++ b/src/room/data-stream/constants.ts @@ -19,32 +19,22 @@ export const STREAM_CHUNK_SIZE_BYTES = 15_000; /** * Reserved data-stream header attribute signaling that the payload (inline or chunked) is * compressed. Self-describing: the sender sets it when it compresses, and the receiver decompresses - * iff it is present. Inline payloads and chunked text streams use - * {@link COMPRESSION_DEFLATE_RAW}; chunked byte streams still use the legacy - * {@link COMPRESSION_GZIP} member scheme. + * iff it is present. Both inline and chunked text and byte streams use {@link COMPRESSION_DEFLATE_RAW}. * * @internal */ export const COMPRESSION_ATTRIBUTE = 'lk.compression'; -/** - * Value of {@link COMPRESSION_ATTRIBUTE} for the legacy chunked byte-stream scheme: each `write()` - * is its own complete gzip member, tagged with a member index in the chunk `version` field. - * Slated to migrate to {@link COMPRESSION_DEFLATE_RAW}. - * - * @internal - */ -export const COMPRESSION_GZIP = 'gzip'; - /** * Value of {@link COMPRESSION_ATTRIBUTE} for raw-deflate-compressed payloads. * * For inline (single-packet) payloads this is a one-shot raw-deflate buffer, base64'd into the - * payload attribute. For chunked streams it is a single raw-deflate context shared across the whole - * stream: the sender sync-flushes at every write boundary so the receiver can decompress each chunk - * as it arrives, and terminates the deflate stream with a final block before the trailer. Receivers - * concatenate chunk contents in `chunkIndex` order through one raw-deflate (windowBits -15) - * decompressor. + * payload attribute. For chunked streams it is a single raw-deflate context spanning the whole + * stream, terminated by a final block before the trailer; receivers concatenate chunk contents in + * `chunkIndex` order through one raw-deflate (windowBits -15) decompressor. The format also + * supports sync-flushing at write boundaries (context takeover) so a future incremental sender + * could compress without a protocol change, though current senders (`sendText`/`sendFile`) + * compress the full payload in one shot. * * @internal */ diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 2d0cf6ecd2..f0a6be42a6 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -13,7 +13,6 @@ import { deflateRawDecompress, inflateRawStream } from '../compression'; import { COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RAW, - COMPRESSION_GZIP, INLINE_PAYLOAD_ATTRIBUTE, } from '../constants'; import { @@ -165,11 +164,9 @@ export default class IncomingDataStreamManager { encryptionType, }; - // Inline byte payloads are one-shot deflate-raw; chunked byte streams still use the legacy - // per-write gzip member scheme (see decompressedChunkStream). - const inlineCompressed = - info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_DEFLATE_RAW; - const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_GZIP; + // Both inline and chunked byte payloads are deflate-raw compressed; inline as a one-shot + // base64 buffer, chunked as a single stream spanning all chunks (mirrors text). + const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_DEFLATE_RAW; // Single-packet stream: the entire payload was smuggled into a reserved header attribute. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. @@ -177,13 +174,15 @@ export default class IncomingDataStreamManager { if (typeof inlinePayload !== 'undefined') { delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; delete info.attributes![COMPRESSION_ATTRIBUTE]; + // Inline bytes are always base64 (binary isn't a valid attribute string), optionally + // deflate-raw compressed. const bytes = decodeBase64(inlinePayload); streamHandlerCallback( new ByteStreamReader( info, createInlineStream( streamHeader.streamId, - inlineCompressed ? deflateRawDecompress(bytes) : bytes, + compressed ? deflateRawDecompress(bytes) : bytes, ), bigIntToNumber(streamHeader.totalLength), ), @@ -218,9 +217,10 @@ export default class IncomingDataStreamManager { streamHandlerCallback( new ByteStreamReader( info, - compressed ? decompressedChunkStream(stream, streamHeader.streamId, 'byte') : stream, - // Compressed streams report no total length; completion is driven by the trailer. - compressed ? undefined : bigIntToNumber(streamHeader.totalLength), + compressed ? inflateRawByteChunkStream(stream, streamHeader.streamId) : stream, + // `totalLength` is the pre-compression size, and the reader counts decompressed bytes, + // so it applies to both paths (mirrors text). + bigIntToNumber(streamHeader.totalLength), ), { identity: participantIdentity, @@ -405,29 +405,18 @@ function createInlineStream( } /** - * Transforms a stream of deflate-raw-compressed text `DataStream_Chunk`s into a stream of - * decompressed chunks, so `TextStreamReader` consumes it unchanged. - * - * The sender runs a single raw-deflate context across the whole stream, sync-flushing at every - * write boundary, so the receiver feeds all chunk contents (in `chunkIndex` order) through ONE - * decompressor and gets each write's content emitted as soon as its chunks arrive. A streaming - * `TextDecoder` reframes the decompressed bytes on UTF-8 character boundaries (a write larger than - * the MTU spans several packets, which may split a codepoint) so each synthesized chunk decodes - * independently. Errors on the source stream (e.g. encryption mismatch, abnormal end) propagate to - * the reader. + * Unwraps a stream of compressed `DataStream_Chunk`s to their compressed bytes (in `chunkIndex` + * order), guarding ordering for the stateful decompressor that consumes them. A stateful + * decompressor silently corrupts on duplicated or out-of-order input, so duplicates are dropped + * (with a warning - in-order delivery is expected on the reliable channel, but reconnect handling + * may replay) and a gap is a hard error. Shared by the text and byte deflate-raw decoders. */ -function inflateRawChunkStream( - raw: ReadableStream, +function orderedCompressedBytes( + srcReader: ReadableStreamDefaultReader, streamId: string, -): ReadableStream { - const srcReader = raw.getReader(); - - // Stage 1: unwrap chunk packets to compressed bytes, guarding ordering. A stateful decompressor - // silently corrupts on duplicated or out-of-order input, so duplicates are dropped (with a - // warning - in-order delivery is expected on the reliable channel, but reconnect handling may - // replay) and a gap is a hard error. +): ReadableStream { let lastChunkIndex = -1; - const compressedBytes = new ReadableStream({ + return new ReadableStream({ pull: async (controller) => { while (true) { const { done, value } = await srcReader.read(); @@ -455,44 +444,40 @@ function inflateRawChunkStream( }, cancel: (reason) => srcReader.cancel(reason), }); +} - // Stage 2: one decompressor for the stream's lifetime. - const decompressedReader = inflateRawStream(compressedBytes).getReader(); +/** + * Transforms a stream of deflate-raw-compressed byte `DataStream_Chunk`s into a stream of + * decompressed chunks, so `ByteStreamReader` consumes it unchanged. All chunk contents are fed (in + * `chunkIndex` order) through ONE raw-deflate decompressor for the stream's lifetime; decompressed + * output is re-wrapped as chunks as soon as it is produced. The sender (sendFile) compresses the + * whole payload in one shot, but the format also supports a single context-takeover stream + * sync-flushed at write boundaries, so a future incremental streamBytes could compress with no + * protocol change. Errors on the source stream propagate to the reader. + */ +function inflateRawByteChunkStream( + raw: ReadableStream, + streamId: string, +): ReadableStream { + const srcReader = raw.getReader(); + const decompressedReader = inflateRawStream( + orderedCompressedBytes(srcReader, streamId), + ).getReader(); - // Stage 3: reframe decompressed bytes on UTF-8 boundaries and re-wrap as chunks. - const decoder = new TextDecoder('utf-8', { fatal: true }); - const encoder = new TextEncoder(); let outIndex = 0; - const decodeOrThrow = (bytes?: Uint8Array): string => { - try { - return bytes ? decoder.decode(bytes, { stream: true }) : decoder.decode(); - } catch (err) { - throw new DataStreamError( - `Cannot decode compressed data stream ${streamId} as text: ${err}`, - DataStreamErrorReason.DecodeFailed, - ); - } - }; - return new ReadableStream({ pull: async (controller) => { while (true) { const { done, value } = await decompressedReader.read(); if (done) { - const tail = decodeOrThrow(); - if (tail.length > 0) { - controller.enqueue(makeChunk(streamId, outIndex, encoder.encode(tail))); - outIndex += 1; - } controller.close(); return; } - const text = decodeOrThrow(value); - if (text.length > 0) { - controller.enqueue(makeChunk(streamId, outIndex++, encoder.encode(text))); + if (value.byteLength > 0) { + controller.enqueue(makeChunk(streamId, outIndex++, value)); return; } - // Everything so far was a partial codepoint; keep pulling. + // Inflate can emit empty reads; keep pulling until there is content or the stream ends. } }, cancel: (reason) => { @@ -503,97 +488,56 @@ function inflateRawChunkStream( } /** - * Transforms a raw stream of (compressed) `DataStream_Chunk`s into a stream of decompressed - * `DataStream_Chunk`s, so the existing `ByteStreamReader`/`TextStreamReader` consume it unchanged. - * - * The sender compresses each `write()` into its own gzip member and tags every chunk with that - * member's index in `chunk.version`. Browsers' `DecompressionStream` only accepts a single member - * per gzip stream, so we feed each member's chunks into its own `DecompressionStream` as they arrive - * (never buffering the whole member) and start a fresh one when the member index changes, draining - * decompressed output incrementally. For text, a streaming `TextDecoder` reframes the decompressed - * bytes on UTF-8 character boundaries across members so each synthesized chunk decodes independently. - * Errors on the source stream (e.g. encryption mismatch, abnormal end) propagate to the reader. + * Transforms a stream of deflate-raw-compressed text `DataStream_Chunk`s into a stream of + * decompressed chunks, so `TextStreamReader` consumes it unchanged. Builds on + * {@link inflateRawByteChunkStream} (single decompressor + ordering guard) and adds a streaming + * `TextDecoder` that reframes the decompressed bytes on UTF-8 character boundaries (a write larger + * than the MTU spans several packets, which may split a codepoint) so each synthesized chunk + * decodes independently. Errors on the source stream propagate to the reader. */ -function decompressedChunkStream( +function inflateRawChunkStream( raw: ReadableStream, streamId: string, - kind: 'byte' | 'text', ): ReadableStream { - const srcReader = raw.getReader(); - const decoder = kind === 'text' ? new TextDecoder() : undefined; - const encoder = kind === 'text' ? new TextEncoder() : undefined; - let outIndex = 0; + const byteReader = inflateRawByteChunkStream(raw, streamId).getReader(); - const enqueueDecompressed = ( - controller: ReadableStreamDefaultController, - bytes: Uint8Array, - ) => { - const content = decoder ? encoder!.encode(decoder.decode(bytes, { stream: true })) : bytes; - if (content.byteLength > 0) { - controller.enqueue(makeChunk(streamId, outIndex++, content)); + const decoder = new TextDecoder('utf-8', { fatal: true }); + let outIndex = 0; + const decodeOrThrow = (bytes?: Uint8Array): string => { + try { + return bytes ? decoder.decode(bytes, { stream: true }) : decoder.decode(); + } catch (err) { + throw new DataStreamError( + `Cannot decode compressed data stream ${streamId} as text: ${err}`, + DataStreamErrorReason.DecodeFailed, + ); } }; - const pump = async (controller: ReadableStreamDefaultController) => { - let currentMember: number | undefined; - let dsWriter: WritableStreamDefaultWriter | null = null; - let drain: Promise | null = null; - - const openMember = () => { - const ds = new DecompressionStream('gzip'); - dsWriter = ds.writable.getWriter(); - const dsReader = ds.readable.getReader(); - // Drain this member's decompressed output concurrently with feeding its input. - drain = (async () => { - for (;;) { - const { done, value } = await dsReader.read(); - if (done) { - break; + const encoder = new TextEncoder(); + return new ReadableStream({ + pull: async (controller) => { + while (true) { + const { done, value } = await byteReader.read(); + if (done) { + const tail = decodeOrThrow(); + if (tail.length > 0) { + controller.enqueue(makeChunk(streamId, outIndex, encoder.encode(tail))); + outIndex += 1; } - enqueueDecompressed(controller, value); + controller.close(); + return; } - })(); - }; - - // Close the current member's compressor input and wait for its remaining output to drain. - const closeMember = async () => { - if (dsWriter) { - await dsWriter.close(); - await drain; - dsWriter = null; - drain = null; - } - }; - - for (;;) { - const { done, value } = await srcReader.read(); - if (done) { - await closeMember(); - const tail = decoder?.decode(); - if (tail) { - controller.enqueue(makeChunk(streamId, outIndex++, encoder!.encode(tail))); + const text = decodeOrThrow(value.content); + if (text.length > 0) { + controller.enqueue(makeChunk(streamId, outIndex++, encoder.encode(text))); + return; } - controller.close(); - return; - } - // A change in member index means the previous gzip member is complete. - if (currentMember !== undefined && value.version !== currentMember) { - await closeMember(); - } - if (!dsWriter) { - openMember(); + // Everything so far was a partial codepoint; keep pulling. } - currentMember = value.version; - await dsWriter!.write(value.content as NonSharedUint8Array); - } - }; - - return new ReadableStream({ - start: (controller) => { - pump(controller).catch((err) => controller.error(err)); }, cancel: (reason) => { - srcReader.cancel(reason); + byteReader.cancel(reason).catch(() => {}); }, }); } diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index ba9d9c7493..1317c74c79 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -19,11 +19,10 @@ import type { TextStreamInfo, } from '../../types'; import { encodeBase64, isCompressionStreamSupported, numberToBigInt, splitUtf8 } from '../../utils'; -import { deflateRawCompress, deflateRawCompressStream, gzipCompressStream } from '../compression'; +import { deflateRawCompress, deflateRawCompressStream } from '../compression'; import { COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RAW, - COMPRESSION_GZIP, INLINE_PAYLOAD_ATTRIBUTE, STREAM_CHUNK_SIZE_BYTES, } from '../constants'; @@ -238,8 +237,6 @@ export default class OutgoingDataStreamManager { options?: SendTextOptions, ): Promise { const destinationIdentities = options?.destinationIdentities; - const engine = this.engine; - const info: TextStreamInfo = { id: streamId, mimeType: 'text/plain', @@ -255,37 +252,83 @@ export default class OutgoingDataStreamManager { }; const header = buildTextStreamHeader(info); const packet = createStreamHeaderPacket(header, destinationIdentities); - await engine.sendDataPacket(packet, DataChannelKind.RELIABLE); + await this.sendChunkedCompressed( + packet, + streamId, + destinationIdentities, + textEncoder.encode(text), + ); + return info; + } + + /** + * Shared one-shot compressed-chunk send for `sendText`/`sendFile`: sends the prebuilt header + * packet, deflate-raw compresses the full `bytes` and forwards the compressed output as + * `streamChunk` packets (split at the MTU budget, contiguous chunk indices for the receiver's + * ordering guard), then sends the trailer. The platform compressor can't flush mid-stream, so + * this only works when the whole payload is known up front. + */ + private async sendChunkedCompressed( + headerPacket: DataPacket, + streamId: string, + destinationIdentities: Array | undefined, + bytes: Uint8Array, + ): Promise { + const engine = this.engine; + await engine.sendDataPacket(headerPacket, DataChannelKind.RELIABLE); - // Forward compressed output as it is produced, splitting at the MTU budget. let chunkId = 0; - const reader = deflateRawCompressStream(textEncoder.encode(text)).getReader(); + const reader = deflateRawCompressStream(bytes).getReader(); while (true) { const { done, value } = await reader.read(); if (done) { break; } - let byteOffset = 0; - while (byteOffset < value.byteLength) { - const subChunk = value.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); - const chunkPacket = new DataPacket({ - destinationIdentities, - value: { - case: 'streamChunk', - value: new DataStream_Chunk({ - content: subChunk, - streamId, - chunkIndex: numberToBigInt(chunkId), - }), - }, - }); - await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); - chunkId += 1; - byteOffset += subChunk.byteLength; - } + chunkId = await sendBytesAsChunks(engine, streamId, destinationIdentities, value, chunkId); } await sendStreamTrailer(streamId, destinationIdentities, engine); + } + + /** + * Attempts to send `bytes` as a single header packet with the payload smuggled into a reserved + * attribute (the byte mirror of {@link trySendInlineText}). Binary can't live in a string + * attribute, so the inline payload is always base64; it is deflate-raw compressed first when the + * runtime supports it and that shrinks the payload. Returns the {@link ByteStreamInfo} if sent + * inline, or `null` if the caller should fall back (recipient is pre-v2, or the packet exceeds + * the MTU). + */ + private async trySendInlineBytes( + info: ByteStreamInfo, + bytes: Uint8Array, + destinationIdentities: Array | undefined, + onProgress?: (progress: number) => void, + ): Promise { + if (!this.allRecipientsSupportV2(destinationIdentities)) { + return null; + } + + const inlineAttributes: Record = { + ...info.attributes, + [INLINE_PAYLOAD_ATTRIBUTE]: encodeBase64(bytes), + }; + if (isCompressionStreamSupported()) { + const compressed = await deflateRawCompress(bytes); + if (compressed.byteLength < bytes.byteLength) { + inlineAttributes[INLINE_PAYLOAD_ATTRIBUTE] = encodeBase64(compressed); + inlineAttributes[COMPRESSION_ATTRIBUTE] = COMPRESSION_DEFLATE_RAW; + } + } + + const header = buildByteStreamHeader({ ...info, attributes: inlineAttributes }); + const packet = createStreamHeaderPacket(header, destinationIdentities); + + if (packet.toBinary().byteLength > STREAM_CHUNK_SIZE_BYTES) { + return null; + } + + await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); + onProgress?.(1); return info; } @@ -378,47 +421,76 @@ export default class OutgoingDataStreamManager { return { id: streamId }; } - private async _sendFile(streamId: string, file: File, options?: SendFileOptions) { - const writer = await this.streamBytes({ - streamId, - totalSize: file.size, + /** + * Sends a file as a byte stream, mirroring {@link sendText}: the full payload is known up front, + * so it tries the single-packet inline path first, then a one-shot deflate-raw compressed chunked + * stream, then an uncompressed chunked stream. Reading the whole file into memory is the + * trade-off for inline + one-shot compression; {@link streamBytes} remains for incremental sends. + */ + private async _sendFile( + streamId: string, + file: File, + options?: SendFileOptions, + ): Promise { + const destinationIdentities = options?.destinationIdentities; + const bytes = new Uint8Array(await file.arrayBuffer()); + + const info: ByteStreamInfo = { + id: streamId, name: file.name, mimeType: options?.mimeType ?? file.type, - topic: options?.topic, - destinationIdentities: options?.destinationIdentities, - }); - const reader = file.stream().getReader(); - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - await writer.write(value); + topic: options?.topic ?? '', + timestamp: Date.now(), + // Pre-compression byte length; the receiver counts decompressed bytes against it. + size: bytes.byteLength, + encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled + ? Encryption_Type.GCM + : Encryption_Type.NONE, + }; + + const inlineInfo = await this.trySendInlineBytes( + info, + bytes, + destinationIdentities, + options?.onProgress, + ); + if (inlineInfo) { + return inlineInfo; } - await writer.close(); - return writer.info; + + const compress = options?.compress ?? true; + if (compress && this.shouldCompress(destinationIdentities)) { + const header = buildByteStreamHeader({ + ...info, + attributes: { ...info.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, + }); + const packet = createStreamHeaderPacket(header, destinationIdentities); + await this.sendChunkedCompressed(packet, streamId, destinationIdentities, bytes); + options?.onProgress?.(1); + return info; + } + + // Uncompressed: header + plain chunk packets + trailer. + const header = buildByteStreamHeader(info); + const packet = createStreamHeaderPacket(header, destinationIdentities); + await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); + await sendBytesAsChunks(this.engine, streamId, destinationIdentities, bytes, 0); + await sendStreamTrailer(streamId, destinationIdentities, this.engine); + options?.onProgress?.(1); + return info; } async streamBytes(options?: StreamBytesOptions) { const streamId = options?.streamId ?? crypto.randomUUID(); const destinationIdentities = options?.destinationIdentities; - const compressOption = - options?.compress ?? - (typeof options?.totalSize === 'number' && options.totalSize > 2_000) /* bytes */; - const compress = compressOption && this.shouldCompress(destinationIdentities); const info: ByteStreamInfo = { id: streamId, mimeType: options?.mimeType ?? 'application/octet-stream', topic: options?.topic ?? '', timestamp: Date.now(), - attributes: compress - ? { ...options?.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_GZIP } - : options?.attributes, - // Compressed streams have an unknown total length up front; left undefined (see receiver). - // - // FIXME: make this instead the size before compression maybe? - size: compress ? undefined : options?.totalSize, + attributes: options?.attributes, + size: options?.totalSize, name: options?.name ?? 'unknown', encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled ? Encryption_Type.GCM @@ -435,47 +507,31 @@ export default class OutgoingDataStreamManager { const engine = this.engine; const logLocal = this.log; - const writableStream = compress - ? createCompressedChunkWritable( - streamId, - destinationIdentities, - engine, - (chunk) => chunk, - ) - : new WritableStream({ - async write(chunk) { - const unlock = await writeMutex.lock(); - - let byteOffset = 0; - try { - while (byteOffset < chunk.byteLength) { - const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); - const chunkPacket = new DataPacket({ - destinationIdentities, - value: { - case: 'streamChunk', - value: new DataStream_Chunk({ - content: subChunk, - streamId, - chunkIndex: numberToBigInt(chunkId), - }), - }, - }); - await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); - chunkId += 1; - byteOffset += subChunk.byteLength; - } - } finally { - unlock(); - } - }, - async close() { - await sendStreamTrailer(streamId, destinationIdentities, engine); - }, - abort(err) { - logLocal.error('Sink error:', err); - }, - }); + // Incremental byte streams are never compressed (the platform compressor can't flush + // mid-stream); one-shot compression lives in sendText/sendFile. A future streamBytes could + // send a context-takeover deflate-raw stream — receivers already decode that wire format. + const writableStream = new WritableStream({ + async write(chunk) { + const unlock = await writeMutex.lock(); + try { + chunkId = await sendBytesAsChunks( + engine, + streamId, + destinationIdentities, + chunk, + chunkId, + ); + } finally { + unlock(); + } + }, + async close() { + await sendStreamTrailer(streamId, destinationIdentities, engine); + }, + abort(err) { + logLocal.error('Sink error:', err); + }, + }); const byteWriter = new ByteStreamWriter(writableStream, info); @@ -483,82 +539,48 @@ export default class OutgoingDataStreamManager { } } -/** Sends a `streamTrailer` packet, marking the end of a stream. */ -async function sendStreamTrailer( - streamId: string, - destinationIdentities: Array | undefined, - engine: RTCEngine, -): Promise { - const trailerPacket = new DataPacket({ - destinationIdentities, - value: { case: 'streamTrailer', value: new DataStream_Trailer({ streamId }) }, - }); - await engine.sendDataPacket(trailerPacket, DataChannelKind.RELIABLE); -} - /** - * Builds a `WritableStream` whose writes are gzip-compressed and emitted as `streamChunk` packets, - * followed by a `streamTrailer` on close. Each `write()` is compressed into its own gzip member and - * its compressed bytes are split into `STREAM_CHUNK_SIZE_BYTES` pieces sent immediately — including a - * final partial piece — so a write's data is delivered without waiting to fill a chunk or for the - * stream to close (low latency for incremental senders). The receiver decompresses the concatenation - * of members (a valid multi-member gzip stream), so this is only used for recipients that understand - * compressed streams. + * Splits `bytes` into `streamChunk` packets at the MTU budget and sends each over the reliable + * channel, returning the next chunk index so callers can keep indices contiguous across calls. */ -function createCompressedChunkWritable( +async function sendBytesAsChunks( + engine: RTCEngine, streamId: string, destinationIdentities: Array | undefined, - engine: RTCEngine, - encode: (chunk: T) => Uint8Array, -): WritableStream { - let chunkId = 0; - // Each write() is compressed into its own gzip member. Browsers' DecompressionStream only accepts a - // single member per gzip stream, so we tag every chunk with its member index (in the chunk's spare - // `version` field); the receiver segments on it and decompresses each member independently. - let memberId = 0; - - const sendChunk = async (content: Uint8Array, version: number) => { + bytes: Uint8Array, + startChunkId: number, +): Promise { + let chunkId = startChunkId; + let byteOffset = 0; + while (byteOffset < bytes.byteLength) { + const subChunk = bytes.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); const chunkPacket = new DataPacket({ destinationIdentities, value: { case: 'streamChunk', value: new DataStream_Chunk({ - content: content as NonSharedUint8Array, + content: subChunk, streamId, chunkIndex: numberToBigInt(chunkId), - version, }), }, }); await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); chunkId += 1; - }; - - return new WritableStream({ - async write(chunk) { - const member = memberId; - memberId += 1; - const reader = gzipCompressStream(encode(chunk)).getReader(); - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - await Promise.all( - new Array(Math.ceil(value.length / STREAM_CHUNK_SIZE_BYTES)).fill(null).map((_, i) => { - return sendChunk( - value.slice(i * STREAM_CHUNK_SIZE_BYTES, (i + 1) * STREAM_CHUNK_SIZE_BYTES), - member, - ); - }), - ); - } - }, - async close() { - await sendStreamTrailer(streamId, destinationIdentities, engine); - }, - abort() { - // Each write compresses independently, so there is no persistent compressor to tear down. - }, + byteOffset += subChunk.byteLength; + } + return chunkId; +} + +/** Sends a `streamTrailer` packet, marking the end of a stream. */ +async function sendStreamTrailer( + streamId: string, + destinationIdentities: Array | undefined, + engine: RTCEngine, +): Promise { + const trailerPacket = new DataPacket({ + destinationIdentities, + value: { case: 'streamTrailer', value: new DataStream_Trailer({ streamId }) }, }); + await engine.sendDataPacket(trailerPacket, DataChannelKind.RELIABLE); } diff --git a/src/room/types.ts b/src/room/types.ts index 83d01de666..f3f1f1c21b 100644 --- a/src/room/types.ts +++ b/src/room/types.ts @@ -21,6 +21,9 @@ export interface SendTextOptions { attachments?: Array; onProgress?: (progress: number) => void; attributes?: Record; + /** Whether to compress the payload (deflate-raw). Defaults to true. Compression is only applied + * when every recipient supports data streams v2 and the runtime can compress. */ + compress?: boolean; } export interface StreamTextOptions { @@ -40,7 +43,6 @@ export type StreamBytesOptions = { topic?: string; attributes?: Record; destinationIdentities?: Array; - compress?: boolean; streamId?: string; mimeType?: string; totalSize?: number; @@ -52,6 +54,9 @@ export type SendFileOptions = Pick< > & { onProgress?: (progress: number) => void; encryptionType?: Encryption_Type.NONE; + /** Whether to compress the payload (deflate-raw). Defaults to true. Compression is only applied + * when every recipient supports data streams v2 and the runtime can compress. */ + compress?: boolean; }; export type DataPublishOptions = { From 6d092602fae444e159053ef892707bd7ff6a5102 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 12 Jun 2026 10:54:58 -0400 Subject: [PATCH 20/68] refactor: move around outgoing data stream single packet / compression code --- src/room/data-stream/compression.ts | 42 +- .../outgoing/OutgoingDataStreamManager.ts | 401 +++++++----------- src/room/utils.ts | 48 +++ 3 files changed, 240 insertions(+), 251 deletions(-) diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index cacde6a3f2..ead2eefd8f 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -1,9 +1,9 @@ /** - * Compression helpers for data streams. The buffered deflate-raw variants are for the inline - * (single-packet) case where the payload is small and bounded; {@link deflateRawCompressStream} / - * {@link inflateRawStream} serve the chunked (multi-packet) `sendText`/`sendFile` fallback, where - * the whole payload is known up front but the compressed output is produced/consumed incrementally - * rather than buffered. + * Compression helpers for data streams. The buffered deflate-raw variant ({@link deflateRawCompress}) + * is for the inline (single-packet) case where the payload is small and bounded; + * {@link deflateRawCompressReadable} / {@link inflateRawStream} serve the chunked (multi-packet) + * `sendText`/`sendFile` paths, streaming the compressed bytes through without buffering the whole + * payload. * * These operate on bytes (not strings) so a single set of helpers serves both text and byte streams; * the `TextEncoder`/`TextDecoder` boundary lives at the manager/reader edges. @@ -15,16 +15,34 @@ */ /** - * Compresses a fully-known payload into a raw-deflate stream, exposing the compressed output as a - * readable so callers can forward it incrementally. Usable only when the entire payload is written - * at once, since the platform compressor cannot flush mid-stream (its only "flush" is the close at - * the end) — incremental multi-write senders (`streamText`) therefore send uncompressed. + * Pipes a byte stream through `CompressionStream('deflate-raw')`, exposing the compressed output as + * a readable — the compression counterpart of {@link inflateRawStream}. Drives the source into the + * compressor in the background (forwarding source errors via `abort`), so callers can forward the + * compressed output incrementally without buffering the whole payload. Used for the chunked + * `sendText`/`sendFile` paths, where the full payload is known up front but is streamed (e.g. from + * `file.stream()`) rather than held in memory. */ -export function deflateRawCompressStream(data: Uint8Array): ReadableStream { +export function deflateRawCompressReadable( + input: ReadableStream, +): ReadableStream { const cs = new CompressionStream('deflate-raw'); const writer = cs.writable.getWriter(); - writer.write(data as NonSharedUint8Array); - writer.close(); + const pipe = (async () => { + const reader = input.getReader(); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + await writer.write(value as NonSharedUint8Array); + } + await writer.close(); + } catch (err) { + await writer.abort(err).catch(() => {}); + } + })(); + pipe.catch(() => {}); return cs.readable; } diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 1317c74c79..6a98c971c4 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -18,8 +18,15 @@ import type { StreamTextOptions, TextStreamInfo, } from '../../types'; -import { encodeBase64, isCompressionStreamSupported, numberToBigInt, splitUtf8 } from '../../utils'; -import { deflateRawCompress, deflateRawCompressStream } from '../compression'; +import { + encodeBase64, + isCompressionStreamSupported, + numberToBigInt, + readBytesInChunks, + readableFromBytes, + splitUtf8, +} from '../../utils'; +import { deflateRawCompress, deflateRawCompressReadable } from '../compression'; import { COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RAW, @@ -75,13 +82,45 @@ export default class OutgoingDataStreamManager { const totalTextLength = textInBytes.byteLength; const compress = options?.compress ?? true; - // Fast path: when the full payload is known up front, there are no attachments, and the - // payload fits (with header overhead) under the MTU, smuggle it into a reserved header - // attribute and send a single `streamHeader` packet - no chunk/trailer packets. - if (!options?.attachments || options.attachments.length === 0) { - const inlineInfo = await this.trySendInlineText(streamId, text, totalTextLength, options); - if (inlineInfo) { - return inlineInfo; + let info: TextStreamInfo = { + id: streamId, + mimeType: 'text/plain', + timestamp: Date.now(), + topic: options?.topic ?? '', + size: totalTextLength, // NOTE: size is always the pre-compression byte length + attributes: options?.attributes, + encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled + ? Encryption_Type.GCM + : Encryption_Type.NONE, + }; + + // Phase 1: Try to send as a single packet data stream + const noAttachments = !options?.attachments || options.attachments.length === 0; + if (noAttachments && this.allRecipientsSupportV2(options?.destinationIdentities)) { + // Compress when the runtime supports it, but only keep the result if it actually shrinks the + // payload (deflate framing plus the base64 expansion makes tiny strings larger). Uncompressed + // inline payloads stay as the raw string; compressed ones are base64'd and flagged via an + // attribute. + const inlineAttributes: Record = { + ...info.attributes, + [INLINE_PAYLOAD_ATTRIBUTE]: text, + }; + if (compress && isCompressionStreamSupported()) { + const raw = textEncoder.encode(text); + const compressed = await deflateRawCompress(raw); + if (compressed.byteLength < raw.byteLength) { + inlineAttributes[INLINE_PAYLOAD_ATTRIBUTE] = encodeBase64(compressed); + inlineAttributes[COMPRESSION_ATTRIBUTE] = COMPRESSION_DEFLATE_RAW; + } + } + + const header = buildTextStreamHeader({ ...info, attributes: inlineAttributes }); + const packet = createStreamHeaderPacket(header, options?.destinationIdentities); + + if (packet.toBinary().byteLength <= STREAM_CHUNK_SIZE_BYTES) { + await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); + options?.onProgress?.(1); + return info; } } @@ -95,20 +134,28 @@ export default class OutgoingDataStreamManager { options?.onProgress?.(totalProgress); }; - let info: TextStreamInfo; - if (compress && this.shouldCompress(options?.destinationIdentities)) { - // The full payload is known up front, so the chunked fallback can compress it in one shot - // with the platform compressor (incremental writers can't compress; see streamText). - info = await this.sendChunkedCompressedText( + // Phase 2: Try to send a multi packet data stream with compressed bytes + if ( + compress && + isCompressionStreamSupported() && + this.allRecipientsSupportV2(options?.destinationIdentities) + ) { + info.attributes = { ...info.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }; + info.attachedStreamIds = fileIds; + + const header = buildTextStreamHeader(info); + const packet = createStreamHeaderPacket(header, options?.destinationIdentities); + await this.sendChunkedByteStream( + packet, streamId, - text, - totalTextLength, - fileIds, - options, + options?.destinationIdentities, + deflateRawCompressReadable(readableFromBytes(textEncoder.encode(text))), ); + // set text part of progress to 1 handleProgress(1, 0); } else { + // Phase 3 / fallback: header + plain uncompressed chunk packets + trailer. const writer = await this.streamText({ streamId, totalSize: totalTextLength, @@ -159,179 +206,32 @@ export default class OutgoingDataStreamManager { ); } - /** Whether to compress a chunked stream: all recipients support v2 and the runtime can - * compress. */ - private shouldCompress(destinationIdentities?: Array): boolean { - return this.allRecipientsSupportV2(destinationIdentities) && isCompressionStreamSupported(); - } - - /** - * Attempts to send `text` as a single header packet with the payload smuggled into a reserved - * attribute. Returns the resulting {@link TextStreamInfo} if it was sent inline, or `null` if the - * caller should fall back to the regular chunked stream (recipient doesn't support data streams - * v2, or the payload is too large to fit under the MTU). - */ - private async trySendInlineText( - streamId: string, - text: string, - totalTextLength: number, - options?: SendTextOptions, - ): Promise { - if (!this.allRecipientsSupportV2(options?.destinationIdentities)) { - return null; - } - - const info: TextStreamInfo = { - id: streamId, - mimeType: 'text/plain', - timestamp: Date.now(), - topic: options?.topic ?? '', - size: totalTextLength, - attributes: options?.attributes, - encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled - ? Encryption_Type.GCM - : Encryption_Type.NONE, - }; - - // Compress when the runtime supports it, but only keep the result if it actually shrinks the - // payload (deflate framing plus the base64 expansion makes tiny strings larger). Uncompressed - // inline payloads stay as the raw string; compressed ones are base64'd and flagged via an - // attribute. - const inlineAttributes: Record = { - ...info.attributes, - [INLINE_PAYLOAD_ATTRIBUTE]: text, - }; - if (isCompressionStreamSupported()) { - const raw = textEncoder.encode(text); - const compressed = await deflateRawCompress(raw); - if (compressed.byteLength < raw.byteLength) { - inlineAttributes[INLINE_PAYLOAD_ATTRIBUTE] = encodeBase64(compressed); - inlineAttributes[COMPRESSION_ATTRIBUTE] = COMPRESSION_DEFLATE_RAW; - } - } - - const header = buildTextStreamHeader({ ...info, attributes: inlineAttributes }); - const packet = createStreamHeaderPacket(header, options?.destinationIdentities); - - if (packet.toBinary().byteLength > STREAM_CHUNK_SIZE_BYTES) { - return null; - } - - await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); - options?.onProgress?.(1); - return info; - } - - /** - * Sends `text` as a compressed chunked stream: one raw-deflate stream spanning every chunk - * packet's content, terminated by the trailer. Used by the sendText fallback when the payload is - * too large to send inline — the full payload is known up front, so the platform compressor - * works even though it cannot flush mid-stream (the close is the only flush needed). Incremental - * writers (streamText) cannot use this and send uncompressed instead. - */ - private async sendChunkedCompressedText( - streamId: string, - text: string, - totalTextLength: number, - attachedStreamIds: Array | undefined, - options?: SendTextOptions, - ): Promise { - const destinationIdentities = options?.destinationIdentities; - const info: TextStreamInfo = { - id: streamId, - mimeType: 'text/plain', - timestamp: Date.now(), - topic: options?.topic ?? '', - // Size is the pre-compression byte length; the receiver counts decompressed bytes against it. - size: totalTextLength, - attributes: { ...options?.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, - encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled - ? Encryption_Type.GCM - : Encryption_Type.NONE, - attachedStreamIds, - }; - const header = buildTextStreamHeader(info); - const packet = createStreamHeaderPacket(header, destinationIdentities); - await this.sendChunkedCompressed( - packet, - streamId, - destinationIdentities, - textEncoder.encode(text), - ); - return info; - } - /** - * Shared one-shot compressed-chunk send for `sendText`/`sendFile`: sends the prebuilt header - * packet, deflate-raw compresses the full `bytes` and forwards the compressed output as - * `streamChunk` packets (split at the MTU budget, contiguous chunk indices for the receiver's - * ordering guard), then sends the trailer. The platform compressor can't flush mid-stream, so - * this only works when the whole payload is known up front. + * Shared chunked-stream send for `sendText`/`sendFile`: sends the prebuilt header packet, then + * forwards `source` (optionally deflate-raw compressed) as `streamChunk` packets re-chunked to + * the MTU budget with contiguous indices, then sends the trailer. The source is consumed + * incrementally, so a `file.stream()` is never buffered in full. The platform compressor can't + * flush mid-stream, so compression is only used when the whole payload is available as a stream + * up front (not for incremental writers like `streamText`/`streamBytes`). */ - private async sendChunkedCompressed( + private async sendChunkedByteStream( headerPacket: DataPacket, streamId: string, destinationIdentities: Array | undefined, - bytes: Uint8Array, + source: ReadableStream, ): Promise { const engine = this.engine; await engine.sendDataPacket(headerPacket, DataChannelKind.RELIABLE); let chunkId = 0; - const reader = deflateRawCompressStream(bytes).getReader(); - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - chunkId = await sendBytesAsChunks(engine, streamId, destinationIdentities, value, chunkId); + for await (const chunk of readBytesInChunks(source, STREAM_CHUNK_SIZE_BYTES)) { + await sendChunkPacket(engine, streamId, destinationIdentities, chunk, chunkId); + chunkId += 1; } await sendStreamTrailer(streamId, destinationIdentities, engine); } - /** - * Attempts to send `bytes` as a single header packet with the payload smuggled into a reserved - * attribute (the byte mirror of {@link trySendInlineText}). Binary can't live in a string - * attribute, so the inline payload is always base64; it is deflate-raw compressed first when the - * runtime supports it and that shrinks the payload. Returns the {@link ByteStreamInfo} if sent - * inline, or `null` if the caller should fall back (recipient is pre-v2, or the packet exceeds - * the MTU). - */ - private async trySendInlineBytes( - info: ByteStreamInfo, - bytes: Uint8Array, - destinationIdentities: Array | undefined, - onProgress?: (progress: number) => void, - ): Promise { - if (!this.allRecipientsSupportV2(destinationIdentities)) { - return null; - } - - const inlineAttributes: Record = { - ...info.attributes, - [INLINE_PAYLOAD_ATTRIBUTE]: encodeBase64(bytes), - }; - if (isCompressionStreamSupported()) { - const compressed = await deflateRawCompress(bytes); - if (compressed.byteLength < bytes.byteLength) { - inlineAttributes[INLINE_PAYLOAD_ATTRIBUTE] = encodeBase64(compressed); - inlineAttributes[COMPRESSION_ATTRIBUTE] = COMPRESSION_DEFLATE_RAW; - } - } - - const header = buildByteStreamHeader({ ...info, attributes: inlineAttributes }); - const packet = createStreamHeaderPacket(header, destinationIdentities); - - if (packet.toBinary().byteLength > STREAM_CHUNK_SIZE_BYTES) { - return null; - } - - await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); - onProgress?.(1); - return info; - } - /** * @internal */ @@ -381,12 +281,13 @@ export default class OutgoingDataStreamManager { } }; + // Incremental text streams are never compressed (CompressionStream does not support flushing + // mid-stream); one-shot compression lives in sendText. + // + // Note that a future streamText could send a context-takeover style deflate-raw stream with + // intermedia explicit `Z_SYNC_FLUSH`s - receivers already will handle this properly today. const writableStream = new WritableStream({ async write(text) { - // Incremental writers are never compressed (the platform compressor cannot flush - // mid-stream, and per-write flushing costs more than it saves at typical write sizes — - // see sendChunkedCompressedText for the one-shot compressed path). Split each write on - // UTF-8 boundaries so every chunk decodes independently on the receiver. for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { await sendChunks(textByteChunk); } @@ -422,10 +323,10 @@ export default class OutgoingDataStreamManager { } /** - * Sends a file as a byte stream, mirroring {@link sendText}: the full payload is known up front, - * so it tries the single-packet inline path first, then a one-shot deflate-raw compressed chunked - * stream, then an uncompressed chunked stream. Reading the whole file into memory is the - * trade-off for inline + one-shot compression; {@link streamBytes} remains for incremental sends. + * Streams a file as a chunked byte stream, compressed (deflate-raw) when the runtime supports it + * and every recipient is on data streams v2. The file is piped `file.stream()` → + * (`CompressionStream`) → chunk packets, so it is never fully buffered in memory — unlike + * `sendText`, there is no inline single-packet fast path for files. */ private async _sendFile( streamId: string, @@ -433,51 +334,60 @@ export default class OutgoingDataStreamManager { options?: SendFileOptions, ): Promise { const destinationIdentities = options?.destinationIdentities; - const bytes = new Uint8Array(await file.arrayBuffer()); - - const info: ByteStreamInfo = { - id: streamId, - name: file.name, - mimeType: options?.mimeType ?? file.type, - topic: options?.topic ?? '', - timestamp: Date.now(), - // Pre-compression byte length; the receiver counts decompressed bytes against it. - size: bytes.byteLength, - encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled - ? Encryption_Type.GCM - : Encryption_Type.NONE, - }; - - const inlineInfo = await this.trySendInlineBytes( - info, - bytes, - destinationIdentities, - options?.onProgress, - ); - if (inlineInfo) { - return inlineInfo; - } - const compress = options?.compress ?? true; - if (compress && this.shouldCompress(destinationIdentities)) { - const header = buildByteStreamHeader({ - ...info, - attributes: { ...info.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, - }); + + // Phase 1: Try to send as a single packet data stream + // + // This is not being done explictly for files, because it's challenging to determine ahead of + // time how well the file contents will compress (and whether the total output will be under the + // MTU). Revisit this in the future though. + + // Phase 2: Try to send a multi packet data stream with compressed bytes + if (compress && isCompressionStreamSupported() && this.allRecipientsSupportV2(destinationIdentities)) { + const info: ByteStreamInfo = { + id: streamId, + name: file.name, + mimeType: options?.mimeType ?? file.type, + topic: options?.topic ?? '', + timestamp: Date.now(), + size: file.size, + attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, + encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled + ? Encryption_Type.GCM + : Encryption_Type.NONE, + }; + + const header = buildByteStreamHeader(info); const packet = createStreamHeaderPacket(header, destinationIdentities); - await this.sendChunkedCompressed(packet, streamId, destinationIdentities, bytes); - options?.onProgress?.(1); + await this.sendChunkedByteStream( + packet, + streamId, + destinationIdentities, + deflateRawCompressReadable(file.stream()), + ); + return info; } - // Uncompressed: header + plain chunk packets + trailer. - const header = buildByteStreamHeader(info); - const packet = createStreamHeaderPacket(header, destinationIdentities); - await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); - await sendBytesAsChunks(this.engine, streamId, destinationIdentities, bytes, 0); - await sendStreamTrailer(streamId, destinationIdentities, this.engine); - options?.onProgress?.(1); - return info; + // Phase 3 / fallback: header + plain uncompressed chunk packets + trailer. + const writer = await this.streamBytes({ + streamId, + totalSize: file.size, + name: file.name, + mimeType: options?.mimeType ?? file.type, + topic: options?.topic, + destinationIdentities: options?.destinationIdentities, + }); + const reader = file.stream().getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + await writer.write(value); + } + await writer.close(); + return writer.info; } async streamBytes(options?: StreamBytesOptions) { @@ -507,9 +417,11 @@ export default class OutgoingDataStreamManager { const engine = this.engine; const logLocal = this.log; - // Incremental byte streams are never compressed (the platform compressor can't flush - // mid-stream); one-shot compression lives in sendText/sendFile. A future streamBytes could - // send a context-takeover deflate-raw stream — receivers already decode that wire format. + // Incremental byte streams are never compressed (CompressionStream does not support flushing + // mid-stream); one-shot compression lives in sendFile. + // + // Note that a future streamBytes could send a context-takeover style deflate-raw stream with + // intermedia explicit `Z_SYNC_FLUSH`s - receivers already will handle this properly today. const writableStream = new WritableStream({ async write(chunk) { const unlock = await writeMutex.lock(); @@ -539,6 +451,28 @@ export default class OutgoingDataStreamManager { } } +/** Sends a single `streamChunk` packet (content must already be within the MTU budget). */ +async function sendChunkPacket( + engine: RTCEngine, + streamId: string, + destinationIdentities: Array | undefined, + content: Uint8Array, + chunkId: number, +): Promise { + const chunkPacket = new DataPacket({ + destinationIdentities, + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + content, + streamId, + chunkIndex: numberToBigInt(chunkId), + }), + }, + }); + await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); +} + /** * Splits `bytes` into `streamChunk` packets at the MTU budget and sends each over the reliable * channel, returning the next chunk index so callers can keep indices contiguous across calls. @@ -554,18 +488,7 @@ async function sendBytesAsChunks( let byteOffset = 0; while (byteOffset < bytes.byteLength) { const subChunk = bytes.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); - const chunkPacket = new DataPacket({ - destinationIdentities, - value: { - case: 'streamChunk', - value: new DataStream_Chunk({ - content: subChunk, - streamId, - chunkIndex: numberToBigInt(chunkId), - }), - }, - }); - await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); + await sendChunkPacket(engine, streamId, destinationIdentities, subChunk, chunkId); chunkId += 1; byteOffset += subChunk.byteLength; } diff --git a/src/room/utils.ts b/src/room/utils.ts index 5e3945124f..ceb91a320d 100644 --- a/src/room/utils.ts +++ b/src/room/utils.ts @@ -781,6 +781,54 @@ export function splitUtf8(s: string, n: number): NonSharedUint8Array[] { return result; } +/** Wraps a byte array in a `ReadableStream` that yields it as a single chunk and then closes. */ +export function readableFromBytes(bytes: Uint8Array): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(bytes as NonSharedUint8Array); + controller.close(); + }, + }); +} + +/** + * Re-chunks a byte stream into pieces of exactly `chunkSize` bytes (the final piece may be + * smaller), coalescing or splitting the source's pieces as needed. Memory use is bounded to roughly + * `chunkSize` plus one source read, so it never buffers the whole stream — used to pack + * `CompressionStream`/`file.stream()` output into MTU-sized data-stream chunks. + */ +export async function* readBytesInChunks( + source: ReadableStream, + chunkSize: number, +): AsyncGenerator { + const reader = source.getReader(); + let buffer = new Uint8Array(0); + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + if (value.byteLength === 0) { + continue; + } + const merged = new Uint8Array(buffer.byteLength + value.byteLength); + merged.set(buffer); + merged.set(value, buffer.byteLength); + buffer = merged; + while (buffer.byteLength >= chunkSize) { + yield buffer.slice(0, chunkSize); + buffer = buffer.slice(chunkSize); + } + } + if (buffer.byteLength > 0) { + yield buffer; + } + } finally { + reader.releaseLock(); + } +} + /** Encodes a byte array as a base64 string (suitable for embedding binary data in a string field). */ export function encodeBase64(bytes: Uint8Array): string { let binary = ''; From 087fe22a84211bfe596fb5f9b4845ec431e8d89d Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 12 Jun 2026 13:29:05 -0400 Subject: [PATCH 21/68] fix: reduce needless diff churn --- src/room/data-stream/incoming/StreamReader.ts | 33 ++--- .../outgoing/OutgoingDataStreamManager.ts | 127 +++++++----------- 2 files changed, 65 insertions(+), 95 deletions(-) diff --git a/src/room/data-stream/incoming/StreamReader.ts b/src/room/data-stream/incoming/StreamReader.ts index e95ea55fc6..9057c22772 100644 --- a/src/room/data-stream/incoming/StreamReader.ts +++ b/src/room/data-stream/incoming/StreamReader.ts @@ -201,8 +201,6 @@ export class TextStreamReader extends BaseStreamReader { // Suppress unhandled rejection on reader.closed — errors are // already propagated through reader.read() to the consumer. reader.closed.catch(() => {}); - // Each chunk decodes independently: the sender splits uncompressed writes on UTF-8 boundaries, - // and compressed streams are reframed on UTF-8 boundaries by the decompression transform. const decoder = new TextDecoder('utf-8', { fatal: true }); const signal = this.signal; @@ -235,22 +233,25 @@ export class TextStreamReader extends BaseStreamReader { ); if (result.done) { this.validateBytesReceived(true); - return { done: true, value: undefined as any }; - } - - this.handleChunkReceived(result.value); + return { done: true, value: undefined }; + } else { + this.handleChunkReceived(result.value); - let decodedResult: string; - try { - decodedResult = decoder.decode(result.value.content); - } catch (err) { - throw new DataStreamError( - `Cannot decode datastream chunk ${result.value.chunkIndex} as text: ${err}`, - DataStreamErrorReason.DecodeFailed, - ); + let decodedResult: string; + try { + decodedResult = decoder.decode(result.value.content); + } catch (err) { + throw new DataStreamError( + `Cannot decode datastream chunk ${result.value.chunkIndex} as text: ${err}`, + DataStreamErrorReason.DecodeFailed, + ); + } + + return { + done: false, + value: decodedResult, + }; } - - return { done: false, value: decodedResult }; } catch (err) { cleanup(); throw err; diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 6a98c971c4..5a78674036 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -106,9 +106,8 @@ export default class OutgoingDataStreamManager { [INLINE_PAYLOAD_ATTRIBUTE]: text, }; if (compress && isCompressionStreamSupported()) { - const raw = textEncoder.encode(text); - const compressed = await deflateRawCompress(raw); - if (compressed.byteLength < raw.byteLength) { + const compressed = await deflateRawCompress(textInBytes); + if (compressed.byteLength < textInBytes.byteLength) { inlineAttributes[INLINE_PAYLOAD_ATTRIBUTE] = encodeBase64(compressed); inlineAttributes[COMPRESSION_ATTRIBUTE] = COMPRESSION_DEFLATE_RAW; } @@ -225,7 +224,18 @@ export default class OutgoingDataStreamManager { let chunkId = 0; for await (const chunk of readBytesInChunks(source, STREAM_CHUNK_SIZE_BYTES)) { - await sendChunkPacket(engine, streamId, destinationIdentities, chunk, chunkId); + const chunkPacket = new DataPacket({ + destinationIdentities, + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + content: chunk, + streamId, + chunkIndex: numberToBigInt(chunkId), + }), + }, + }); + await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); chunkId += 1; } @@ -258,29 +268,6 @@ export default class OutgoingDataStreamManager { let chunkId = 0; const engine = this.engine; - // Sends `bytes` as one or more streamChunk packets, splitting at the MTU budget. Writes are - // already serialized by the WritableStream, so no extra locking is needed. - const sendChunks = async (bytes: Uint8Array) => { - let byteOffset = 0; - while (byteOffset < bytes.byteLength) { - const subChunk = bytes.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); - const chunkPacket = new DataPacket({ - destinationIdentities, - value: { - case: 'streamChunk', - value: new DataStream_Chunk({ - content: subChunk, - streamId, - chunkIndex: numberToBigInt(chunkId), - }), - }, - }); - await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); - chunkId += 1; - byteOffset += subChunk.byteLength; - } - }; - // Incremental text streams are never compressed (CompressionStream does not support flushing // mid-stream); one-shot compression lives in sendText. // @@ -289,7 +276,21 @@ export default class OutgoingDataStreamManager { const writableStream = new WritableStream({ async write(text) { for (const textByteChunk of splitUtf8(text, STREAM_CHUNK_SIZE_BYTES)) { - await sendChunks(textByteChunk); + const chunk = new DataStream_Chunk({ + content: textByteChunk, + streamId, + chunkIndex: numberToBigInt(chunkId), + }); + const chunkPacket = new DataPacket({ + destinationIdentities, + value: { + case: 'streamChunk', + value: chunk, + }, + }); + await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); + + chunkId += 1; } }, async close() { @@ -425,14 +426,26 @@ export default class OutgoingDataStreamManager { const writableStream = new WritableStream({ async write(chunk) { const unlock = await writeMutex.lock(); + + let byteOffset = 0; try { - chunkId = await sendBytesAsChunks( - engine, - streamId, - destinationIdentities, - chunk, - chunkId, - ); + while (byteOffset < chunk.byteLength) { + const subChunk = chunk.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); + const chunkPacket = new DataPacket({ + destinationIdentities, + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + content: subChunk, + streamId, + chunkIndex: numberToBigInt(chunkId), + }), + }, + }); + await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); + chunkId += 1; + byteOffset += subChunk.byteLength; + } } finally { unlock(); } @@ -451,50 +464,6 @@ export default class OutgoingDataStreamManager { } } -/** Sends a single `streamChunk` packet (content must already be within the MTU budget). */ -async function sendChunkPacket( - engine: RTCEngine, - streamId: string, - destinationIdentities: Array | undefined, - content: Uint8Array, - chunkId: number, -): Promise { - const chunkPacket = new DataPacket({ - destinationIdentities, - value: { - case: 'streamChunk', - value: new DataStream_Chunk({ - content, - streamId, - chunkIndex: numberToBigInt(chunkId), - }), - }, - }); - await engine.sendDataPacket(chunkPacket, DataChannelKind.RELIABLE); -} - -/** - * Splits `bytes` into `streamChunk` packets at the MTU budget and sends each over the reliable - * channel, returning the next chunk index so callers can keep indices contiguous across calls. - */ -async function sendBytesAsChunks( - engine: RTCEngine, - streamId: string, - destinationIdentities: Array | undefined, - bytes: Uint8Array, - startChunkId: number, -): Promise { - let chunkId = startChunkId; - let byteOffset = 0; - while (byteOffset < bytes.byteLength) { - const subChunk = bytes.slice(byteOffset, byteOffset + STREAM_CHUNK_SIZE_BYTES); - await sendChunkPacket(engine, streamId, destinationIdentities, subChunk, chunkId); - chunkId += 1; - byteOffset += subChunk.byteLength; - } - return chunkId; -} - /** Sends a `streamTrailer` packet, marking the end of a stream. */ async function sendStreamTrailer( streamId: string, From 234d378df304cad91db6662497adfcd1404351c5 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 12 Jun 2026 15:32:51 -0400 Subject: [PATCH 22/68] feat: add max header size breaking change --- .../outgoing/OutgoingDataStreamManager.ts | 30 ++++++++++++++++--- src/room/errors.ts | 3 ++ 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 5a78674036..e68b13b4b8 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -9,6 +9,7 @@ import { type StructuredLogger } from '../../../logger'; import { CLIENT_PROTOCOL_DATA_STREAM_V2 } from '../../../version'; import type RTCEngine from '../../RTCEngine'; import { DataChannelKind } from '../../RTCEngine'; +import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { EngineEvent } from '../../events'; import type { ByteStreamInfo, @@ -220,7 +221,7 @@ export default class OutgoingDataStreamManager { source: ReadableStream, ): Promise { const engine = this.engine; - await engine.sendDataPacket(headerPacket, DataChannelKind.RELIABLE); + await sendHeaderPacket(engine, headerPacket); let chunkId = 0; for await (const chunk of readBytesInChunks(source, STREAM_CHUNK_SIZE_BYTES)) { @@ -263,7 +264,7 @@ export default class OutgoingDataStreamManager { }; const header = buildTextStreamHeader(info, options); const packet = createStreamHeaderPacket(header, destinationIdentities); - await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); + await sendHeaderPacket(this.engine, packet); let chunkId = 0; const engine = this.engine; @@ -344,7 +345,11 @@ export default class OutgoingDataStreamManager { // MTU). Revisit this in the future though. // Phase 2: Try to send a multi packet data stream with compressed bytes - if (compress && isCompressionStreamSupported() && this.allRecipientsSupportV2(destinationIdentities)) { + if ( + compress && + isCompressionStreamSupported() && + this.allRecipientsSupportV2(destinationIdentities) + ) { const info: ByteStreamInfo = { id: streamId, name: file.name, @@ -411,7 +416,7 @@ export default class OutgoingDataStreamManager { const header = buildByteStreamHeader(info); const packet = createStreamHeaderPacket(header, destinationIdentities); - await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); + await sendHeaderPacket(this.engine, packet); let chunkId = 0; const writeMutex = new Mutex(); @@ -464,6 +469,23 @@ export default class OutgoingDataStreamManager { } } +/** + * Sends a stream `streamHeader` packet, enforcing that it fits the MTU budget. The header carries + * the user attributes (plus topic/framing), and a single `DataPacket` larger than the MTU can't be + * reliably sent — so an oversized header (almost always due to large attributes) is a hard error + * rather than a malformed packet on the wire. The inline fast path does its own size check and + * falls back to the chunked path instead of calling this. + */ +async function sendHeaderPacket(engine: RTCEngine, packet: DataPacket): Promise { + if (packet.toBinary().byteLength > STREAM_CHUNK_SIZE_BYTES) { + throw new DataStreamError( + `data stream header exceeds the ${STREAM_CHUNK_SIZE_BYTES}-byte limit; reduce attribute size`, + DataStreamErrorReason.HeaderTooLarge, + ); + } + await engine.sendDataPacket(packet, DataChannelKind.RELIABLE); +} + /** Sends a `streamTrailer` packet, marking the end of a stream. */ async function sendStreamTrailer( streamId: string, diff --git a/src/room/errors.ts b/src/room/errors.ts index 0344f498bc..666afe9507 100644 --- a/src/room/errors.ts +++ b/src/room/errors.ts @@ -287,6 +287,9 @@ export enum DataStreamErrorReason { // Encryption type mismatch. EncryptionTypeMismatch = 8, + + // The serialized stream header packet (driven mainly by attributes) exceeds the MTU budget. + HeaderTooLarge = 9, } export class DataStreamError extends LivekitReasonedError { From f79abfcc470817cd3ba687400b0896eb5e849180 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 15 Jun 2026 11:33:58 -0400 Subject: [PATCH 23/68] feat: advertise new CAP_COMPRESSION_DEFLATE_RAW capability to SFU --- package.json | 2 +- pnpm-lock.yaml | 10 +++---- src/room/Room.ts | 26 ++++++++++++---- .../outgoing/OutgoingDataStreamManager.ts | 30 +++++++++++++++++-- src/room/participant/RemoteParticipant.ts | 27 +++++++++++++---- 5 files changed, 77 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index 2561dbc657..7292726bb6 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ }, "dependencies": { "@livekit/mutex": "1.1.1", - "@livekit/protocol": "1.46.6", + "@livekit/protocol": "1.46.8", "events": "^3.3.0", "jose": "^6.1.0", "loglevel": "^1.9.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 247d0397cf..7d9969f8a1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: 1.1.1 version: 1.1.1 '@livekit/protocol': - specifier: 1.46.6 - version: 1.46.6 + specifier: 1.46.8 + version: 1.46.8 '@types/dom-mediacapture-record': specifier: ^1 version: 1.0.22 @@ -1224,8 +1224,8 @@ packages: '@livekit/mutex@1.1.1': resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} - '@livekit/protocol@1.46.6': - resolution: {integrity: sha512-upzlHP1vi/kZ/QqALZTFskQ0ifqc2f15RKucHYOsIHJsaXvEYanG75mAb7o+Yomfs4XhQ4BaRsdY+TFHXpaqrg==} + '@livekit/protocol@1.46.8': + resolution: {integrity: sha512-mOjcCVLy4Q7qEaEE7gGLi5wXan0K3VTvSpto5Y0ftek2hauALxBW0+cyxNRoakT7dbWFfH+gqc2XQM0P4M1Q/g==} '@livekit/throws-transformer@0.1.3': resolution: {integrity: sha512-PBttE6W6g/2ALGu6kWOunZ5qdrXwP9Ge1An2/62OfE6Rhc0Abd4yp6ex2pWhwUfGxDsSZvFgoB1Ia/5mWAMuKQ==} @@ -5121,7 +5121,7 @@ snapshots: '@livekit/mutex@1.1.1': {} - '@livekit/protocol@1.46.6': + '@livekit/protocol@1.46.8': dependencies: '@bufbuild/protobuf': 1.10.1 diff --git a/src/room/Room.ts b/src/room/Room.ts index 2f71100f89..b12aa72273 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -118,6 +118,7 @@ import { getEmptyAudioStreamTrack, isBrowserSupported, isCloud, + isCompressionStreamSupported, isLocalAudioTrack, isLocalParticipant, isReactNative, @@ -272,6 +273,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.engine, this.log, this.getRemoteParticipantClientProtocol, + this.getRemoteParticipantCapabilities, this.getAllRemoteParticipantIdentities, ); @@ -976,11 +978,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) autoSubscribe: connectOptions.autoSubscribe, adaptiveStream: typeof roomOptions.adaptiveStream === 'object' ? true : roomOptions.adaptiveStream, - clientInfoCapabilities: - isFrameMetadataSupported(roomOptions.frameMetadata ?? roomOptions.packetTrailer) || - !!this.e2eeManager - ? [ClientInfo_Capability.CAP_PACKET_TRAILER] - : undefined, + clientInfoCapabilities: this.getClientInfoCapabilities(roomOptions), maxRetries: connectOptions.maxRetries, e2eeEnabled: !!this.e2eeManager, websocketTimeout: connectOptions.websocketTimeout, @@ -2510,10 +2508,28 @@ class Room extends (EventEmitter as new () => TypedEmitter) } } + /** The client capabilities this SDK advertises to other participants in its `ClientInfo`. */ + private getClientInfoCapabilities(roomOptions: InternalRoomOptions): ClientInfo_Capability[] { + const capabilities: ClientInfo_Capability[] = []; + if (isFrameMetadataSupported(roomOptions.frameMetadata ?? roomOptions.packetTrailer) || !!this.e2eeManager) { + capabilities.push(ClientInfo_Capability.CAP_PACKET_TRAILER); + } + if (isCompressionStreamSupported()) { + capabilities.push(ClientInfo_Capability.CAP_COMPRESSION_DEFLATE_RAW); + } + return capabilities; + } + private getRemoteParticipantClientProtocol = (identity: Participant['identity']) => { return this.remoteParticipants.get(identity)?.clientProtocol ?? CLIENT_PROTOCOL_DEFAULT; }; + private getRemoteParticipantCapabilities = ( + identity: Participant['identity'], + ): ClientInfo_Capability[] => { + return this.remoteParticipants.get(identity)?.capabilities ?? []; + }; + private getAllRemoteParticipantIdentities = () => { return Array.from(this.remoteParticipants.keys()); }; diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index e68b13b4b8..cfd74fae14 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -1,5 +1,6 @@ import { Mutex } from '@livekit/mutex'; import { + ClientInfo_Capability, DataPacket, DataStream_Chunk, DataStream_Trailer, @@ -56,6 +57,10 @@ export default class OutgoingDataStreamManager { * recipient can receive single-packet (inline) data streams. */ protected getRemoteParticipantClientProtocol: (identity: string) => number; + /** Returns the client capabilities a remote participant advertises, used to decide whether a + * recipient can decompress a deflate-raw compressed stream. */ + protected getRemoteParticipantCapabilities: (identity: string) => Array; + /** Returns the identities of every remote participant currently in the room, used to decide * whether a broadcast (no explicit destinations) can be sent inline. */ protected getAllRemoteParticipantIdentities: () => Array; @@ -64,11 +69,13 @@ export default class OutgoingDataStreamManager { engine: RTCEngine, log: StructuredLogger, getRemoteParticipantClientProtocol: (identity: string) => number, + getRemoteParticipantCapabilities: (identity: string) => Array, getAllRemoteParticipantIdentities: () => Array, ) { this.engine = engine; this.log = log; this.getRemoteParticipantClientProtocol = getRemoteParticipantClientProtocol; + this.getRemoteParticipantCapabilities = getRemoteParticipantCapabilities; this.getAllRemoteParticipantIdentities = getAllRemoteParticipantIdentities; } @@ -138,7 +145,8 @@ export default class OutgoingDataStreamManager { if ( compress && isCompressionStreamSupported() && - this.allRecipientsSupportV2(options?.destinationIdentities) + this.allRecipientsSupportV2(options?.destinationIdentities) && + this.allRecipientsSupportCompression(options?.destinationIdentities) ) { info.attributes = { ...info.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }; info.attachedStreamIds = fileIds; @@ -206,6 +214,23 @@ export default class OutgoingDataStreamManager { ); } + /** + * Returns true only if every recipient advertises the deflate-raw compression capability (so it + * can decompress a compressed stream). Resolved the same way as {@link allRecipientsSupportV2}: + * named destinations, or every remote participant for a broadcast; an empty room is eligible. + */ + private allRecipientsSupportCompression(destinationIdentities?: Array): boolean { + const identities = + destinationIdentities && destinationIdentities.length > 0 + ? destinationIdentities + : this.getAllRemoteParticipantIdentities(); + return identities.every((identity) => + this.getRemoteParticipantCapabilities(identity).includes( + ClientInfo_Capability.CAP_COMPRESSION_DEFLATE_RAW, + ), + ); + } + /** * Shared chunked-stream send for `sendText`/`sendFile`: sends the prebuilt header packet, then * forwards `source` (optionally deflate-raw compressed) as `streamChunk` packets re-chunked to @@ -348,7 +373,8 @@ export default class OutgoingDataStreamManager { if ( compress && isCompressionStreamSupported() && - this.allRecipientsSupportV2(destinationIdentities) + this.allRecipientsSupportV2(destinationIdentities) && + this.allRecipientsSupportCompression(destinationIdentities) ) { const info: ByteStreamInfo = { id: streamId, diff --git a/src/room/participant/RemoteParticipant.ts b/src/room/participant/RemoteParticipant.ts index 464d7c5d70..34d6b4b22a 100644 --- a/src/room/participant/RemoteParticipant.ts +++ b/src/room/participant/RemoteParticipant.ts @@ -1,8 +1,9 @@ -import type { - ParticipantInfo, - SubscriptionError, - UpdateSubscription, - UpdateTrackSettings, +import { + ClientInfo_Capability, + type ParticipantInfo, + type SubscriptionError, + type UpdateSubscription, + type UpdateTrackSettings, } from '@livekit/protocol'; import type { SignalClient } from '../../api/SignalClient'; import { DeferrableMap } from '../../utils/deferrable-map'; @@ -47,6 +48,16 @@ export default class RemoteParticipant extends Participant { **/ clientProtocol: number; + /** The client capabilities the remote participant advertises (e.g. deflate-raw compression + * support). Used to decide which peer-to-peer features can be used when sending to them. + * + * Differs from clientProtocol in that these are truely optional "additions" which can be used + * or not depending on client specific attributes rather than protocol level invariants. + * + * @internal + **/ + capabilities: Array; + private volumeMap: Map; private audioOutput?: AudioOutputOptions; @@ -72,6 +83,10 @@ export default class RemoteParticipant extends Participant { return new RemoteDataTrack(info, manager, { publisherIdentity: pi.identity }); }), pi.clientProtocol, + // FIXME: ParticipantInfo does not yet carry client capabilities. Until the + // protocol/server propagates `capabilities` onto ParticipantInfo, mock every remote as + // advertising deflate-raw compression support so compression stays enabled. + [ClientInfo_Capability.CAP_COMPRESSION_DEFLATE_RAW], ); } @@ -95,6 +110,7 @@ export default class RemoteParticipant extends Participant { kind: ParticipantKind = ParticipantKind.STANDARD, remoteDataTracks: Array = [], clientProtocol: number = CLIENT_PROTOCOL_DEFAULT, + capabilities: Array = [], ) { super(sid, identity || '', name, metadata, attributes, loggerOptions, kind); this.signalClient = signalClient; @@ -108,6 +124,7 @@ export default class RemoteParticipant extends Participant { ); this.volumeMap = new Map(); this.clientProtocol = clientProtocol; + this.capabilities = capabilities; } protected addTrackPublication(publication: RemoteTrackPublication) { From 7e342e4b9c455bd8397413f783fc552a4036be17 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 15 Jun 2026 15:23:35 -0400 Subject: [PATCH 24/68] feat: add proper OutgoingDataStreamManager tests --- .../OutgoingDataStreamManager.test.ts | 665 ++++++++++++++++++ 1 file changed, 665 insertions(+) create mode 100644 src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts new file mode 100644 index 0000000000..60af63e66e --- /dev/null +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts @@ -0,0 +1,665 @@ +import { ClientInfo_Capability, type DataPacket } from '@livekit/protocol'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import log from '../../../logger'; +import { + CLIENT_PROTOCOL_DATA_STREAM_RPC, + CLIENT_PROTOCOL_DATA_STREAM_V2, + CLIENT_PROTOCOL_DEFAULT, +} from '../../../version'; +import type RTCEngine from '../../RTCEngine'; +import { + COMPRESSION_ATTRIBUTE, + COMPRESSION_DEFLATE_RAW, + INLINE_PAYLOAD_ATTRIBUTE, +} from '../constants'; +import OutgoingDataStreamManager from './OutgoingDataStreamManager'; + +/** Builds a low quality random string of the given length. */ +function randomText(length: number): string { + let s = ''; + while (s.length < length) { + s += Math.random().toString(36).slice(2); + } + return s.slice(0, length); +} + +/** Fills a buffer with uniform random bytes — genuinely incompressible. */ +function randomBytes(length: number): Uint8Array { + const out = new Uint8Array(length); + // crypto.getRandomValues rejects requests over 65536 bytes, so chunk it. + for (let offset = 0; offset < length; offset += 65536) { + crypto.getRandomValues(out.subarray(offset, offset + 65536)); + } + return out; +} + +/** + * @param participants the remote participants in the room, mapped from identity to the client + * protocol each advertises. Defaults to a single v2 participant named "bob". + */ +function createManager( + participants: Record]> = { bob: CLIENT_PROTOCOL_DATA_STREAM_V2 }, +) { + const sentPackets: DataPacket[] = []; + const engine = { + sendDataPacket: vi.fn(async (packet: DataPacket) => { + sentPackets.push(packet); + }), + e2eeManager: undefined, + once: vi.fn(), + off: vi.fn(), + } as unknown as RTCEngine; + const manager = new OutgoingDataStreamManager( + engine, + log, + (identity) => (Array.isArray(participants[identity]) ? participants[identity][0] : participants[identity]) ?? CLIENT_PROTOCOL_DEFAULT, + (identity) => Array.isArray(participants[identity]) ? participants[identity][1] : [ClientInfo_Capability.CAP_COMPRESSION_DEFLATE_RAW], + () => Object.keys(participants), + ); + return { manager, sentPackets }; +} + +function headerOf(packet: DataPacket) { + return packet.value.value as Extract['value']; +} + +function chunkOf(packet: DataPacket) { + return packet.value.value as Extract['value']; +} + +function trailerOf(packet: DataPacket) { + return packet.value.value as Extract['value']; +} + +describe('OutgoingDataStreamManager', () => { + describe('v2 -> room of all v1', () => { + let manager: OutgoingDataStreamManager, sentPackets: Array; + beforeEach(() => { + const result = createManager({ + alice: CLIENT_PROTOCOL_DEFAULT, + bob: CLIENT_PROTOCOL_DEFAULT, + jim: CLIENT_PROTOCOL_DATA_STREAM_RPC, + }); + manager = result.manager; + sentPackets = result.sentPackets; + }); + + it('should send short TEXT data stream using non single packet "legacy" format and NO compression (happy path)', async () => { + const info = await manager.sendText('hello world', { + topic: 'my-topic', + }); + + // Make sure three packets were received, matching the legacy v1 data stream format + expect(sentPackets).toHaveLength(3); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + const chunk = chunkOf(sentPackets[1]); + expect(chunk.streamId).toStrictEqual(info.id); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toStrictEqual(new TextEncoder().encode('hello world')); + + expect(sentPackets[2].value.case).toStrictEqual('streamTrailer'); + const trailer = trailerOf(sentPackets[2]); + expect(trailer.streamId).toStrictEqual(info.id); + expect(trailer.reason).toStrictEqual(''); + }); + + it('should send short BYTE data stream using non single packet "legacy" format and NO compression (happy path)', async () => { + const writer = await manager.streamBytes({ + topic: 'my-topic', + }); + await writer.write(new Uint8Array([0x00, 0x01, 0x02, 0x03])); + await writer.close(); + + // Make sure three packets were received, matching the legacy v1 data stream format + expect(sentPackets).toHaveLength(3); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(writer.info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('byteHeader'); + + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + const chunk = chunkOf(sentPackets[1]); + expect(chunk.streamId).toStrictEqual(writer.info.id); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toStrictEqual(new Uint8Array([0x00, 0x01, 0x02, 0x03])); + + expect(sentPackets[2].value.case).toStrictEqual('streamTrailer'); + const trailer = trailerOf(sentPackets[2]); + expect(trailer.streamId).toStrictEqual(writer.info.id); + expect(trailer.reason).toStrictEqual(''); + }); + + it('should send long TEXT data stream without compression (happy path)', async () => { + const longPayload = new Array(40_000).fill('A').join(''); + const info = await manager.sendText(longPayload, { + topic: 'my-topic', + }); + + // Make sure five packets were received, matching the legacy v1 data stream format + expect(sentPackets).toHaveLength(5); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + for (let i = 0; i < 3; i += 1) { + expect(sentPackets[i+1].value.case).toStrictEqual('streamChunk'); + const chunk = chunkOf(sentPackets[i+1]); + expect(chunk.streamId).toStrictEqual(info.id); + expect(chunk.chunkIndex).toStrictEqual(BigInt(i)); + expect(chunk.content.every((char) => char === 'A'.charCodeAt(0))).toBeTruthy(); + } + + expect(sentPackets[4].value.case).toStrictEqual('streamTrailer'); + const trailer = trailerOf(sentPackets[4]); + expect(trailer.streamId).toStrictEqual(info.id); + expect(trailer.reason).toStrictEqual(''); + }); + + it('should send long BYTE data stream without compression (happy path)', async () => { + const writer = await manager.streamBytes({ + topic: 'my-topic', + }); + await writer.write(new Uint8Array(20_000).fill(0x01)); + await writer.write(new Uint8Array(20_000).fill(0x01)); + await writer.close(); + + // Make sure five packets were received, matching the legacy v1 data stream format + expect(sentPackets).toHaveLength(6); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(writer.info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('byteHeader'); + + // First write generates two packets, 15k long + 5k long + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + let chunk = chunkOf(sentPackets[1]); + expect(chunk.streamId).toStrictEqual(writer.info.id); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toHaveLength(15_000); // MTU + expect(chunk.content.every((byte) => byte === 0x01)).toBeTruthy(); + + expect(sentPackets[2].value.case).toStrictEqual('streamChunk'); + chunk = chunkOf(sentPackets[2]); + expect(chunk.streamId).toStrictEqual(writer.info.id); + expect(chunk.chunkIndex).toStrictEqual(1n); + expect(chunk.content).toHaveLength(5_000); // MTU + expect(chunk.content.every((byte) => byte === 0x01)).toBeTruthy(); + + // Second write generates two packets, 15k long + 5k long + expect(sentPackets[3].value.case).toStrictEqual('streamChunk'); + chunk = chunkOf(sentPackets[3]); + expect(chunk.streamId).toStrictEqual(writer.info.id); + expect(chunk.chunkIndex).toStrictEqual(2n); + expect(chunk.content).toHaveLength(15_000); + expect(chunk.content.every((byte) => byte === 0x01)).toBeTruthy(); + + expect(sentPackets[4].value.case).toStrictEqual('streamChunk'); + chunk = chunkOf(sentPackets[4]); + expect(chunk.streamId).toStrictEqual(writer.info.id); + expect(chunk.chunkIndex).toStrictEqual(3n); + expect(chunk.content).toHaveLength(5_000); + expect(chunk.content.every((byte) => byte === 0x01)).toBeTruthy(); + + expect(sentPackets[5].value.case).toStrictEqual('streamTrailer'); + const trailer = trailerOf(sentPackets[5]); + expect(trailer.streamId).toStrictEqual(writer.info.id); + expect(trailer.reason).toStrictEqual(''); + }); + }); + describe('v2 -> room of all v2', () => { + let manager: OutgoingDataStreamManager, sentPackets: Array; + beforeEach(() => { + const result = createManager({ + alice: CLIENT_PROTOCOL_DATA_STREAM_V2, + bob: CLIENT_PROTOCOL_DATA_STREAM_V2, + noCompression: [CLIENT_PROTOCOL_DATA_STREAM_V2, []], + }); + manager = result.manager; + sentPackets = result.sentPackets; + }); + + it('should send short TEXT data stream with single packet and compression (happy path)', async () => { + const info = await manager.sendText('hello hello compressible world', { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + }); + + // Make sure one single packet was used, since data streams v2 + compression is supported + // across all participants + expect(sentPackets).toHaveLength(1); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + // Make sure the contents of that packet was compressed + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeTypeOf('string'); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).not.toStrictEqual('hello hello compressible world'); + }); + it('should send short TEXT data stream with uncompressible payload in single packet', async () => { + const info = await manager.sendText('short', { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + }); + + // Make sure one single packet was used, since data streams v2 + compression is supported + // across all participants + expect(sentPackets).toHaveLength(1); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + // Make sure the contents of that packet was uncompressed - "short" isn't long enough to + // meaningfully compress with DEFLATE + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toStrictEqual('short'); + }); + it('should send short data stream with single packet and NO compression if remote participant does not support compression', async () => { + const info = await manager.sendText('hello hello compressible world', { + topic: 'my-topic', + destinationIdentities: ['noCompression'], + }); + + // Make sure one single packet was used, since data streams v2 is supported for that + // participant. + expect(sentPackets).toHaveLength(1); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + // Make sure the contents of that packet was NOT compressed + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toStrictEqual('hello hello compressible world'); + }); + it('should send long but highly compressible TEXT data stream as single packet', async () => { + // A phrase which repeats over and over should compress extremely well. + const text = new Array(20_000).fill('hello world').join(''); + + const info = await manager.sendText(text, { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + }); + + // Make sure one single packet was used, since data streams v2 is supported and the contents + // should be able to be highly compressed to be well under the 15k MTU + expect(sentPackets).toHaveLength(1); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + // Make sure the contents of that packet was compressed + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeTypeOf('string'); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]?.startsWith('hello world')).toBeFalsy(); + }); + it('should send long but somewhat compressible data stream as a compressed multi packet data stream', async () => { + // Mostly incompressible, but the hello world parts repeating should mean that the compressed + // contents is smaller than the full uncompressed data. + const text = new Array(50).fill(null).map(() => `hello world${randomText(1_000)}`).join(''); + + const info = await manager.sendText(text, { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + }); + + // 1 header + 3 data packets + 1 trailer = 5 total packets + // + // 3 data packets is less than the Math.ceil(~50k / 15k) = 4 packets that would be + // required if data was uncompressed + expect(sentPackets).toHaveLength(5); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + // Make sure the contents of that packet was compressed + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + + // Verify there are three data packets: + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + let chunk = chunkOf(sentPackets[1]); + expect(chunk.streamId).toStrictEqual(info.id); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toHaveLength(15_000); // MTU + + expect(sentPackets[2].value.case).toStrictEqual('streamChunk'); + expect(sentPackets[3].value.case).toStrictEqual('streamChunk'); + + // Final packet should be a trailer + expect(sentPackets[4].value.case).toStrictEqual('streamTrailer'); + const trailer = trailerOf(sentPackets[4]); + expect(trailer.streamId).toStrictEqual(info.id); + expect(trailer.reason).toStrictEqual(''); + }); + it('should send long, uncompressible data stream as a compressed multi packet data stream', async () => { + // This is random data which should be uncompressible + const bytes = randomBytes(50_000); + const info = await manager.sendFile(new File([bytes as NonSharedUint8Array], 'text.txt'), { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + }); + + // Math.ceil(~50k / 15k) = 4 data packets + // 1 header + 4 data packets + 1 trailer = 6 total packets + expect(sentPackets).toHaveLength(6); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('byteHeader'); + + // Make sure the contents of that packet was NOT compressed + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + + // Verify there are four data packets: + let totalLength = 0; + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + let chunk = chunkOf(sentPackets[1]); + expect(chunk.streamId).toStrictEqual(info.id); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toHaveLength(15_000); // MTU + totalLength += chunk.content.byteLength; + + expect(sentPackets[2].value.case).toStrictEqual('streamChunk'); + chunk = chunkOf(sentPackets[2]); + totalLength += chunk.content.byteLength; + + expect(sentPackets[3].value.case).toStrictEqual('streamChunk'); + chunk = chunkOf(sentPackets[3]); + totalLength += chunk.content.byteLength; + + expect(sentPackets[4].value.case).toStrictEqual('streamChunk'); + chunk = chunkOf(sentPackets[4]); + totalLength += chunk.content.byteLength; + + // Make sure total length is LARGER than the raw bytes length (only slightly, due to the extra + // DEFLATE metadata being added to an otherwise incompressible binary blob) + // + // This is sort of unfortunate that this happens, but the tradeoff to this slight size bump is + // that the whole binary doesn't have to be buffered into memory all at once. + expect(totalLength).toBeGreaterThan(bytes.byteLength); + + // Final packet should be a trailer + expect(sentPackets[5].value.case).toStrictEqual('streamTrailer'); + const trailer = trailerOf(sentPackets[5]); + expect(trailer.streamId).toStrictEqual(info.id); + expect(trailer.reason).toStrictEqual(''); + }); + it('should send short data stream with single packet but skip compression due to compress: false being passed', async () => { + const info = await manager.sendText('hello hello compressible world', { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + compress: false, + }); + + // Make sure one single packet was used, since data streams v2 is supported across all participants + expect(sentPackets).toHaveLength(1); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + // Make sure the contents of that packet was compressed + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeTypeOf('string'); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toStrictEqual('hello hello compressible world'); + }); + it('should send long but somewhat compressible data stream but skip compression due to compress: false being passed', async () => { + // Mostly incompressible, but the hello world parts repeating should mean that the compressed + // contents is smaller than the full uncompressed data. + const text = new Array(50).fill(null).map(() => `hello world${randomText(1_000)}`).join(''); + + const info = await manager.sendText(text, { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + compress: false, + }); + + // Math.ceil(~50k / 15k) = 4 data packets + // 1 header + 4 data packets + 1 trailer = 6 total packets + expect(sentPackets).toHaveLength(6); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + // Make sure the contents of that packet was uncompressed + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); + + // Verify there are four data packets: + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + let chunk = chunkOf(sentPackets[1]); + expect(chunk.streamId).toStrictEqual(info.id); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toHaveLength(15_000); // MTU + + expect(sentPackets[2].value.case).toStrictEqual('streamChunk'); + expect(sentPackets[3].value.case).toStrictEqual('streamChunk'); + expect(sentPackets[4].value.case).toStrictEqual('streamChunk'); + + // Final packet should be a trailer + expect(sentPackets[5].value.case).toStrictEqual('streamTrailer'); + const trailer = trailerOf(sentPackets[5]); + expect(trailer.streamId).toStrictEqual(info.id); + expect(trailer.reason).toStrictEqual(''); + }); + + it('should NEVER use compression or single packet data streams with streamText', async () => { + const writer = await manager.streamText({ + topic: 'my-topic', + destinationIdentities: ['noCompression'], + }); + + // Make sure the header packet was sent + expect(sentPackets).toHaveLength(1); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(writer.info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); // Make sure compression is disabled + + await writer.write('hello world'); + + // Make sure a single chunk packet was emitted + expect(sentPackets).toHaveLength(2 /* 1 header + 1 chunk */); + + expect(sentPackets[1].value.case).toBe('streamChunk'); + const chunk = chunkOf(sentPackets[1]); + expect(chunk.streamId).toStrictEqual(writer.info.id); + expect(chunk.content).toStrictEqual(new TextEncoder().encode('hello world')); + + await writer.close(); + + // Finally, verify the trailer + expect(sentPackets).toHaveLength(3 /* 1 header + 1 chunk + 1 trailer */); + expect(sentPackets[2].value.case).toBe('streamTrailer'); + }); + it('should NEVER use compression or single packet data streams with streamBytes', async () => { + const writer = await manager.streamBytes({ + topic: 'my-topic', + destinationIdentities: ['noCompression'], + }); + + // Make sure the header packet was sent + expect(sentPackets).toHaveLength(1); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(writer.info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('byteHeader'); + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); // Make sure compression is disabled + + await writer.write(new Uint8Array([0x00, 0x01, 0x02, 0x03])); + + // Make sure a single chunk packet was emitted + expect(sentPackets).toHaveLength(2 /* 1 header + 1 chunk */); + + expect(sentPackets[1].value.case).toBe('streamChunk'); + const chunk = chunkOf(sentPackets[1]); + expect(chunk.streamId).toStrictEqual(writer.info.id); + expect(chunk.content).toStrictEqual(new Uint8Array([0x00, 0x01, 0x02, 0x03])); + + await writer.close(); + + // Finally, verify the trailer + expect(sentPackets).toHaveLength(3 /* 1 header + 1 chunk + 1 trailer */); + expect(sentPackets[2].value.case).toBe('streamTrailer'); + }); + + it('should NOT send bytes single packet with sendFile', async () => { + // This is random data which should be uncompressible + const bytes = new Uint8Array(10_000).fill(0x01); + const info = await manager.sendFile(new File([bytes as NonSharedUint8Array], 'text.txt'), { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + }); + + // Should be a multi-packet result + // + // Sending single packet data streams for files is tricky because it's really difficult to + // determine ahead of time if a file can fit into a single packet without a ton of ahead of + // time in memory buffering. + expect(sentPackets).toHaveLength(3); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('byteHeader'); + + // Make sure the contents of that packet was NOT compressed + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + let chunk = chunkOf(sentPackets[1]); + expect(chunk.streamId).toStrictEqual(info.id); + expect(chunk.chunkIndex).toStrictEqual(0n); + + // Make sure contents were compressed + expect(chunk.content.byteLength).toBeLessThan(bytes.byteLength); + + // Final packet should be a trailer + expect(sentPackets[2].value.case).toStrictEqual('streamTrailer'); + const trailer = trailerOf(sentPackets[2]); + expect(trailer.streamId).toStrictEqual(info.id); + expect(trailer.reason).toStrictEqual(''); + }); + }); + describe('v2 -> room of mixed v1 / v2', () => { + let manager: OutgoingDataStreamManager, sentPackets: Array; + beforeEach(() => { + const result = createManager({ + alice: CLIENT_PROTOCOL_DEFAULT, + bob: CLIENT_PROTOCOL_DATA_STREAM_V2, + jim: CLIENT_PROTOCOL_DATA_STREAM_V2, + mallory: CLIENT_PROTOCOL_DATA_STREAM_RPC, + noCompression: [CLIENT_PROTOCOL_DATA_STREAM_V2, []], + }); + manager = result.manager; + sentPackets = result.sentPackets; + }); + + it('should send data stream using v1 legacy data stream format in room of mixed v1/v2', async () => { + const info = await manager.sendText('hello world', { + topic: 'my-topic', + }); + + // Make sure three packets were received, matching the legacy v1 data stream format + expect(sentPackets).toHaveLength(3); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + const chunk = chunkOf(sentPackets[1]); + expect(chunk.streamId).toStrictEqual(info.id); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toStrictEqual(new TextEncoder().encode('hello world')); + + expect(sentPackets[2].value.case).toStrictEqual('streamTrailer'); + const trailer = trailerOf(sentPackets[2]); + expect(trailer.streamId).toStrictEqual(info.id); + expect(trailer.reason).toStrictEqual(''); + }); + it('should send data stream using data stream v2 format + compression when only sending to a subset of participants that are all v2', async () => { + const info = await manager.sendText('hello hello compressible world', { + topic: 'my-topic', + destinationIdentities: ['bob', 'jim'], + }); + + // Make sure one single packet was used, since data streams v2 + compression is supported + // across bob + jim + expect(sentPackets).toHaveLength(1); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + // Make sure the contents of that packet was compressed + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeTypeOf('string'); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).not.toStrictEqual('hello hello compressible world'); + }); + it('should send data stream using data stream v2 format but NO compression when only sending to a subset of participants where one does NOT support compression', async () => { + const info = await manager.sendText('hello hello compressible world', { + topic: 'my-topic', + destinationIdentities: ['bob', 'jim', 'noCompression'], + }); + + // Make sure one single packet was used, since data streams v2 + compression is supported + // across bob + jim + expect(sentPackets).toHaveLength(1); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('textHeader'); + + // Make sure the contents of that packet was compressed + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toStrictEqual('hello hello compressible world'); + }); + }); +}); From 8d9ec441b80697bc125ae22ab26bb61fbddcba4a Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 16 Jun 2026 15:42:26 -0400 Subject: [PATCH 25/68] feat: add initial IncomingDataStreamManager tests --- .../IncomingDataStreamManager.test.ts | 675 ++++++++++++++++-- .../incoming/IncomingDataStreamManager.ts | 20 +- 2 files changed, 627 insertions(+), 68 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts index a6beb9ea2a..8d0b3db6b3 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -1,95 +1,636 @@ import { DataPacket, DataStream_ByteHeader, + DataStream_Chunk, DataStream_Header, DataStream_TextHeader, + DataStream_Trailer, Encryption_Type, } from '@livekit/protocol'; import { describe, expect, it } from 'vitest'; +import { createDeflateRaw, constants } from 'zlib'; import { encodeBase64 } from '../../utils'; -import { INLINE_PAYLOAD_ATTRIBUTE } from '../constants'; +import { INLINE_PAYLOAD_ATTRIBUTE, COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RAW, STREAM_CHUNK_SIZE_BYTES } from '../constants'; import IncomingDataStreamManager from './IncomingDataStreamManager'; import type { ByteStreamReader, TextStreamReader } from './StreamReader'; +import { deflateRawCompress } from '../compression'; -function inlineTextHeaderPacket(streamId: string, topic: string, text: string) { - const header = new DataStream_Header({ - streamId, - topic, - mimeType: 'text/plain', - timestamp: 0n, - totalLength: BigInt(new TextEncoder().encode(text).byteLength), - attributes: { [INLINE_PAYLOAD_ATTRIBUTE]: text, foo: 'bar' }, - contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, - }); - return new DataPacket({ - participantIdentity: 'alice', - value: { case: 'streamHeader', value: header }, - }); +/** Builds a low quality random string of the given length. */ +function randomText(length: number): string { + let s = ''; + while (s.length < length) { + s += Math.random().toString(36).slice(2); + } + return s.slice(0, length); } -function inlineByteHeaderPacket(streamId: string, topic: string, bytes: Uint8Array) { - const header = new DataStream_Header({ - streamId, - topic, - mimeType: 'application/octet-stream', - timestamp: 0n, - totalLength: BigInt(bytes.byteLength), - attributes: { [INLINE_PAYLOAD_ATTRIBUTE]: encodeBase64(bytes), foo: 'bar' }, - contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({ name: 'blob' }) }, - }); - return new DataPacket({ - participantIdentity: 'alice', - value: { case: 'streamHeader', value: header }, - }); -} +describe('IncomingDataStreamManager', () => { + describe('Receiving v1 data streams', () => { + it('should receive a v1 text data stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); -describe('IncomingDataStreamManager inline streams', () => { - it('synthesizes a complete text stream from an inline header', async () => { - const manager = new IncomingDataStreamManager(); - manager.setConnected(true); + const streamId = crypto.randomUUID(); + const text = 'hello world'; - const readerPromise = new Promise((resolve) => { - manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(text.length), + attributes: { foo: 'bar' }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: new TextEncoder().encode(text), + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId }), + }, + }), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual('hello world'); + expect(reader.info.attributes?.foo).toStrictEqual('bar'); }); - manager.handleDataStreamPacket( - inlineTextHeaderPacket('stream-1', 'my-topic', 'hello world'), - Encryption_Type.NONE, - ); + it('should receive a v1 bytes data stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerByteStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: 4n, + attributes: { foo: 'bar' }, + contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: new Uint8Array([0x01, 0x02, 0x03, 0x04]), + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId }), + }, + }), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual([new Uint8Array([0x01, 0x02, 0x03, 0x04])]); + expect(reader.info.attributes?.foo).toStrictEqual('bar'); + }); + + it('should receive a v1 text data stream with files', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const streamId = crypto.randomUUID(); + const streamReaderPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const attachmentStreamId = crypto.randomUUID(); + const attachmentStreamReaderPromise = new Promise((resolve) => { + manager.registerByteStreamHandler('my-topic', (reader) => resolve(reader)); + }); - const reader = await readerPromise; - expect(await reader.readAll()).toBe('hello world'); + // Send the main data stream body + const text = 'hello world'; + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(text.length), + attributes: { [INLINE_PAYLOAD_ATTRIBUTE]: text, foo: 'bar' }, + contentHeader: { + case: 'textHeader', + value: new DataStream_TextHeader({ + attachedStreamIds: [attachmentStreamId], + }), + }, + }), + }, + }), + Encryption_Type.NONE, + ); - // The reserved attribute is stripped, user attributes are preserved. - expect(reader.info.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeUndefined(); - expect(reader.info.attributes?.foo).toBe('bar'); + // Send an attachment + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId: attachmentStreamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: 3n, + attributes: {}, + contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId: attachmentStreamId, + chunkIndex: 0n, + content: new Uint8Array([0x01, 0x02, 0x03]), + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId: attachmentStreamId }), + }, + }), + Encryption_Type.NONE, + ); + + const streamReader = await streamReaderPromise; + expect(await streamReader.readAll()).toStrictEqual('hello world'); + expect(streamReader.info.attachedStreamIds).toHaveLength(1); + + const attachmentStreamReader = await attachmentStreamReaderPromise; + expect(await attachmentStreamReader.readAll()).toStrictEqual([new Uint8Array([0x01, 0x02, 0x03])]); + expect(streamReader.info.attachedStreamIds).toHaveLength(1); + }); }); - it('synthesizes a complete byte stream from an inline header', async () => { - const manager = new IncomingDataStreamManager(); - manager.setConnected(true); + describe('Receiving v2 data streams', () => { + it('should receive a v2 SINGLE PACKET + UNCOMPRESSED text data stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = 'hello world'; + + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(text.length), + attributes: { [INLINE_PAYLOAD_ATTRIBUTE]: text, foo: 'bar' }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual('hello world'); + expect(reader.info.attributes?.foo).toStrictEqual('bar'); + }); + + it('should receive a v2 SINGLE PACKET + COMPRESSED text data stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = 'hello world'; + const compressed = encodeBase64(await deflateRawCompress(new TextEncoder().encode(text))); - const payload = new Uint8Array([0, 1, 2, 255, 128, 64]); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(text.length), + attributes: { + [INLINE_PAYLOAD_ATTRIBUTE]: compressed, + [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW, + foo: 'bar' + }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); - const readerPromise = new Promise((resolve) => { - manager.registerByteStreamHandler('bytes-topic', (reader) => resolve(reader)); + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual('hello world'); + expect(reader.info.attributes?.foo).toStrictEqual('bar'); }); - manager.handleDataStreamPacket( - inlineByteHeaderPacket('stream-2', 'bytes-topic', payload), - Encryption_Type.NONE, - ); - - const reader = await readerPromise; - const chunks = await reader.readAll(); - const flattened = new Uint8Array(chunks.reduce((acc, c) => acc + c.byteLength, 0)); - let offset = 0; - for (const chunk of chunks) { - flattened.set(chunk, offset); - offset += chunk.byteLength; - } - expect(Array.from(flattened)).toEqual(Array.from(payload)); - expect(reader.info.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeUndefined(); - expect(reader.info.attributes?.foo).toBe('bar'); + it('should receive a v2 MULTI PACKET + COMPRESSED text data stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + + // NOTE: mostly incompressible, but the hello world parts repeating should mean that the compressed + // contents is smaller than the full uncompressed data. + const text = new Array(30).fill(null).map(() => `hello world${randomText(1_000)}`).join(''); + + const compressed = await deflateRawCompress(new TextEncoder().encode(text)); + + // Make sure the compressed text should be able to be split into two "packets" worth of data + expect(compressed.length).toBeLessThan(2 * STREAM_CHUNK_SIZE_BYTES); + + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(text.length), + attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: compressed.slice(0, STREAM_CHUNK_SIZE_BYTES), + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 1n, + content: compressed.slice(STREAM_CHUNK_SIZE_BYTES), + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId }), + }, + }), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual(text); + }); + + it.only('should receive a v2 multi packet + compressed data stream and emit chunks at Z_SYNC_FLUSH boundaries', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + + // Generate a mostly incompressible compressed stream, with Z_SYNC_FLUSH between each chunk + const input = new Array(30).fill(0).map(() => new TextEncoder().encode(`hello world${randomText(1_000)}`)); + const compressed: Array = []; + const stream = createDeflateRaw(); + stream.on('data', (chunk) => { + compressed.push(chunk); + }); + for (const item of input) { + stream.write(item); + await new Promise((resolve) => stream.flush(constants.Z_SYNC_FLUSH, resolve)); + } + const closePromise = new Promise((resolve, reject) => { + stream.once('end', resolve); + stream.once('error', reject); + }); + stream.end(); + await closePromise; + + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(input.reduce((acc, i) => acc + i.byteLength, 0)), + attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + for (let index = 0; index < input.length; index += 1) { + const item = input[index]; + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: BigInt(index), + content: item, + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + } + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId }), + }, + }), + Encryption_Type.NONE, + ); + + // Make sure that stream chunks each get emitted sequentially and are passed out of the reader + const reader = await readerPromise; + const iterator = reader[Symbol.asyncIterator](); + for (let index = 0; index < input.length; index += 1) { + const chunk = await iterator.next(); + expect(chunk.done).toStrictEqual(false); + expect(chunk.value).toStrictEqual(input[index]); + } + const final = await iterator.next(); + expect(final.done).toStrictEqual(true); + }); + + it(`should ignore a v2 TEXT data stream with compression if DecompressionStream doesn't exist`, async () => { + const text = 'hello world'; + const compressed = await deflateRawCompress(new TextEncoder().encode(text)); + + let originalCompressionStream: typeof CompressionStream, originalDecompressionStream: typeof DecompressionStream; + try { + originalCompressionStream = CompressionStream; + (CompressionStream as any) = undefined; + originalDecompressionStream = DecompressionStream; + (DecompressionStream as any) = undefined; + + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(text.length), + attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: compressed, + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId }), + }, + }), + Encryption_Type.NONE, + ); + + // Make sure promise is still pending; the data stream should have been dropped + await expect( + Promise.race([readerPromise, Promise.resolve('still pending')]), + ).resolves.toStrictEqual('still pending'); + } finally { + CompressionStream = originalCompressionStream!; + DecompressionStream = originalDecompressionStream!; + } + }); + + it(`should ignore a v2 BYTES data stream with compression if DecompressionStream doesn't exist`, async () => { + const bytes = new Uint8Array([0x01, 0x02, 0x03]); + const compressed = await deflateRawCompress(bytes); + + let originalCompressionStream: typeof CompressionStream, originalDecompressionStream: typeof DecompressionStream; + try { + originalCompressionStream = CompressionStream; + (CompressionStream as any) = undefined; + originalDecompressionStream = DecompressionStream; + (DecompressionStream as any) = undefined; + + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(bytes.length), + attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: compressed, + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId }), + }, + }), + Encryption_Type.NONE, + ); + + // Make sure promise is still pending; the data stream should have been dropped + await expect( + Promise.race([readerPromise, Promise.resolve('still pending')]), + ).resolves.toStrictEqual('still pending'); + } finally { + CompressionStream = originalCompressionStream!; + DecompressionStream = originalDecompressionStream!; + } + }); }); }); diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index f0a6be42a6..3f46efdea0 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -8,7 +8,7 @@ import { import log from '../../../logger'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { type ByteStreamInfo, type StreamController, type TextStreamInfo } from '../../types'; -import { bigIntToNumber, decodeBase64, numberToBigInt } from '../../utils'; +import { bigIntToNumber, decodeBase64, isCompressionStreamSupported, numberToBigInt } from '../../utils'; import { deflateRawDecompress, inflateRawStream } from '../compression'; import { COMPRESSION_ATTRIBUTE, @@ -168,6 +168,15 @@ export default class IncomingDataStreamManager { // base64 buffer, chunked as a single stream spanning all chunks (mirrors text). const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_DEFLATE_RAW; + if (compressed && !isCompressionStreamSupported()) { + // NOTE: this shouldn't really ever happen, if this warning is logged then the sender + // isn't properly abiding by the data streams v2 protocol. + log.warn( + `Data stream ${streamHeader.streamId} received with ${info.attributes![COMPRESSION_ATTRIBUTE]} compression, but this browser does not have support for DecompressionStream. Dropping...`, + ); + return; + } + // Single-packet stream: the entire payload was smuggled into a reserved header attribute. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; @@ -255,6 +264,15 @@ export default class IncomingDataStreamManager { // buffer, chunked as a single stream spanning all chunks (see COMPRESSION_DEFLATE_RAW). const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_DEFLATE_RAW; + if (compressed && !isCompressionStreamSupported()) { + // NOTE: this shouldn't really ever happen, if this warning is logged then the sender + // isn't properly abiding by the data streams v2 protocol. + log.warn( + `Data stream ${streamHeader.streamId} received with ${info.attributes![COMPRESSION_ATTRIBUTE]} compression, but this browser does not have support for DecompressionStream. Dropping...`, + ); + return; + } + // Single-packet stream: the entire payload was smuggled into a reserved header attribute. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; From 18fe70cadf3f0ed22045d35b3fa1306b21e3038d Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 16 Jun 2026 15:43:24 -0400 Subject: [PATCH 26/68] fix: ensure that compression only used when remote participants support it --- src/room/data-stream/outgoing/OutgoingDataStreamManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index cfd74fae14..3f51bc378a 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -113,7 +113,7 @@ export default class OutgoingDataStreamManager { ...info.attributes, [INLINE_PAYLOAD_ATTRIBUTE]: text, }; - if (compress && isCompressionStreamSupported()) { + if (compress && isCompressionStreamSupported() && this.allRecipientsSupportCompression(options?.destinationIdentities)) { const compressed = await deflateRawCompress(textInBytes); if (compressed.byteLength < textInBytes.byteLength) { inlineAttributes[INLINE_PAYLOAD_ATTRIBUTE] = encodeBase64(compressed); From 1932f7776d27696c4e608b6e9e764d6c5b6a4429 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 16 Jun 2026 16:13:44 -0400 Subject: [PATCH 27/68] fix: fiz zlib Z_SYNC_FLUSH test --- .../IncomingDataStreamManager.test.ts | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts index 8d0b3db6b3..55e0e14c9c 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -409,15 +409,18 @@ describe('IncomingDataStreamManager', () => { const streamId = crypto.randomUUID(); // Generate a mostly incompressible compressed stream, with Z_SYNC_FLUSH between each chunk - const input = new Array(30).fill(0).map(() => new TextEncoder().encode(`hello world${randomText(1_000)}`)); - const compressed: Array = []; + const text = new Array(30).fill(0).map(() => `hello world${randomText(1_000)}`); + const bytes = text.map((b) => new TextEncoder().encode(b)); const stream = createDeflateRaw(); - stream.on('data', (chunk) => { - compressed.push(chunk); - }); - for (const item of input) { + let pending: Array = []; + stream.on('data', (chunk: Buffer) => pending.push(chunk)); + + const compressed: Array = []; + for (const item of bytes) { stream.write(item); await new Promise((resolve) => stream.flush(constants.Z_SYNC_FLUSH, resolve)); + compressed.push(Buffer.concat(pending)); + pending = []; } const closePromise = new Promise((resolve, reject) => { stream.once('end', resolve); @@ -425,6 +428,9 @@ describe('IncomingDataStreamManager', () => { }); stream.end(); await closePromise; + // The final deflate block is emitted on end(); fold it into the last write's chunk so the + // receiver's single decompressor terminates cleanly right after emitting the last write. + compressed[compressed.length - 1] = Buffer.concat([compressed[compressed.length - 1], ...pending]); manager.handleDataStreamPacket( new DataPacket({ @@ -436,7 +442,7 @@ describe('IncomingDataStreamManager', () => { topic: 'my-topic', mimeType: 'text/plain', timestamp: 0n, - totalLength: BigInt(input.reduce((acc, i) => acc + i.byteLength, 0)), + totalLength: BigInt(bytes.reduce((acc, i) => acc + i.byteLength, 0)), attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, }), @@ -444,8 +450,8 @@ describe('IncomingDataStreamManager', () => { }), Encryption_Type.NONE, ); - for (let index = 0; index < input.length; index += 1) { - const item = input[index]; + for (let index = 0; index < compressed.length; index += 1) { + const item = compressed[index]; manager.handleDataStreamPacket( new DataPacket({ participantIdentity: 'alice', @@ -476,10 +482,12 @@ describe('IncomingDataStreamManager', () => { // Make sure that stream chunks each get emitted sequentially and are passed out of the reader const reader = await readerPromise; const iterator = reader[Symbol.asyncIterator](); - for (let index = 0; index < input.length; index += 1) { + + for (let index = 0; index < text.length; index += 1) { const chunk = await iterator.next(); expect(chunk.done).toStrictEqual(false); - expect(chunk.value).toStrictEqual(input[index]); + console.log('AA', chunk.value, text[index]); + expect(chunk.value).toStrictEqual(text[index]); } const final = await iterator.next(); expect(final.done).toStrictEqual(true); From 209e654b2544fde8889c9b4eb5f6e15f6e017822 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 16 Jun 2026 17:06:38 -0400 Subject: [PATCH 28/68] feat: add a few more IncomingDataStreamManager tests --- .../IncomingDataStreamManager.test.ts | 438 ++++++++++++++++-- 1 file changed, 412 insertions(+), 26 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts index 55e0e14c9c..f5933b25e1 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -14,6 +14,7 @@ import { INLINE_PAYLOAD_ATTRIBUTE, COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RA import IncomingDataStreamManager from './IncomingDataStreamManager'; import type { ByteStreamReader, TextStreamReader } from './StreamReader'; import { deflateRawCompress } from '../compression'; +import { attributes, DataStreamError } from '../../..'; /** Builds a low quality random string of the given length. */ function randomText(length: number): string { @@ -238,6 +239,299 @@ describe('IncomingDataStreamManager', () => { expect(await attachmentStreamReader.readAll()).toStrictEqual([new Uint8Array([0x01, 0x02, 0x03])]); expect(streamReader.info.attachedStreamIds).toHaveLength(1); }); + + it('should buffer packets when disconnected', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(false); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = 'hello world'; + + // Send three packets + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(text.length), + attributes: { foo: 'bar' }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: new TextEncoder().encode(text), + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId }), + }, + }), + Encryption_Type.NONE, + ); + + // Make sure promise still hasn't resolved + await expect( + Promise.race([readerPromise, Promise.resolve('still pending')]), + ).resolves.toStrictEqual('still pending'); + + // Simulate connecting + manager.setConnected(true); + + // Make sure it resolves after connected state set + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual('hello world'); + }); + + it('should merge in trailer attributes', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = 'hello world'; + + // Send three packets + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(text.length), + attributes: { foo: 'bar', baz: 'quux' }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: new TextEncoder().encode(text), + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId, attributes: { hello: 'world', foo: 'updated' } }), + }, + }), + Encryption_Type.NONE, + ); + + // Make sure it resolves after connected state set + const reader = await readerPromise; + expect(reader.info.attributes?.baz).toStrictEqual('quux'); + expect(reader.info.attributes?.hello).toStrictEqual('world'); + expect(reader.info.attributes?.foo).toStrictEqual('updated'); + }); + + it('should drop packets with incorrect EncryptionType', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = 'hello world'; + + // Send two packets, the second with an incorrect encryption value + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(text.length), + attributes: { foo: 'bar', baz: 'quux' }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: new TextEncoder().encode(text), + version: 0, + }), + }, + }), + Encryption_Type.GCM, // <-- NOTE: this has changed since the last packet + ); + + // Make sure an error is thrown from the reader + const reader = await readerPromise; + expect(() => reader.readAll()).rejects.toThrow('Encryption type mismatch'); + }); + + it('should throw an error if data stream does not have enough packets', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + + // Send a header, a 1 byte long chunk, and a trailer + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: 5n, + attributes: { foo: 'bar', baz: 'quux' }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: new Uint8Array([0x01]), + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId }), + }, + }), + Encryption_Type.NONE, + ); + + // Make sure an error is thrown from the reader + const reader = await readerPromise; + await expect(reader.readAll()).rejects.toThrow('Not enough chunk(s)'); + }); + + it('should throw an error if data stream has too many bytes', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + + // Send a header declaring 3 bytes, then a 5 byte long chunk, and a trailer + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: 3n, + attributes: { foo: 'bar', baz: 'quux' }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05]), + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + + // Make sure an error is thrown from the reader + const reader = await readerPromise; + await expect(reader.readAll()).rejects.toThrow('Extra chunk(s)'); + }); }); describe('Receiving v2 data streams', () => { @@ -276,6 +570,40 @@ describe('IncomingDataStreamManager', () => { expect(reader.info.attributes?.foo).toStrictEqual('bar'); }); + it('should receive a v2 SINGLE PACKET + UNCOMPRESSED byte data stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerByteStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const bytes = encodeBase64(new Uint8Array([0x01, 0x02, 0x03])); + + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: 3n, + attributes: { [INLINE_PAYLOAD_ATTRIBUTE]: bytes }, + contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual([new Uint8Array([0x01, 0x02, 0x03])]); + }); + it('should receive a v2 SINGLE PACKET + COMPRESSED text data stream', async () => { const manager = new IncomingDataStreamManager(); manager.setConnected(true); @@ -316,6 +644,44 @@ describe('IncomingDataStreamManager', () => { expect(reader.info.attributes?.foo).toStrictEqual('bar'); }); + it('should receive a v2 SINGLE PACKET + COMPRESSED byte data stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerByteStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const bytes = new Uint8Array([0x01, 0x02, 0x03]); + const compressed = encodeBase64(await deflateRawCompress(bytes)); + + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: BigInt(bytes.length), + attributes: { + [INLINE_PAYLOAD_ATTRIBUTE]: compressed, + [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW, + }, + contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual([new Uint8Array([0x01, 0x02, 0x03])]); + }); + it('should receive a v2 MULTI PACKET + COMPRESSED text data stream', async () => { const manager = new IncomingDataStreamManager(); manager.setConnected(true); @@ -398,7 +764,7 @@ describe('IncomingDataStreamManager', () => { expect(await reader.readAll()).toStrictEqual(text); }); - it.only('should receive a v2 multi packet + compressed data stream and emit chunks at Z_SYNC_FLUSH boundaries', async () => { + it('should receive a v2 multi packet + compressed data stream and emit chunks at Z_SYNC_FLUSH boundaries', async () => { const manager = new IncomingDataStreamManager(); manager.setConnected(true); @@ -412,25 +778,50 @@ describe('IncomingDataStreamManager', () => { const text = new Array(30).fill(0).map(() => `hello world${randomText(1_000)}`); const bytes = text.map((b) => new TextEncoder().encode(b)); const stream = createDeflateRaw(); - let pending: Array = []; - stream.on('data', (chunk: Buffer) => pending.push(chunk)); + + const merge = (parts: Array): Uint8Array => { + const out = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0)); + let offset = 0; + for (const part of parts) { + out.set(part, offset); + offset += part.byteLength; + } + return out; + }; + // Drain all currently-buffered compressor output as a single Uint8Array. Reading in paused + // mode (rather than via 'data' events) means each drain right after a flush callback + // deterministically captures exactly that write's flushed bytes, with no event-timing race + // that could split a write's output across two chunks. + const drain = (): Uint8Array => { + const parts: Array = []; + let part: Uint8Array | null; + while ((part = stream.read() as Uint8Array | null) !== null) { + parts.push(part); + } + return merge(parts); + }; const compressed: Array = []; for (const item of bytes) { stream.write(item); await new Promise((resolve) => stream.flush(constants.Z_SYNC_FLUSH, resolve)); - compressed.push(Buffer.concat(pending)); - pending = []; + compressed.push(drain()); } - const closePromise = new Promise((resolve, reject) => { - stream.once('end', resolve); - stream.once('error', reject); - }); - stream.end(); - await closePromise; // The final deflate block is emitted on end(); fold it into the last write's chunk so the // receiver's single decompressor terminates cleanly right after emitting the last write. - compressed[compressed.length - 1] = Buffer.concat([compressed[compressed.length - 1], ...pending]); + const tail: Array = []; + await new Promise((resolve, reject) => { + stream.once('error', reject); + stream.once('end', resolve); + stream.on('readable', () => { + let part: Uint8Array | null; + while ((part = stream.read() as Uint8Array | null) !== null) { + tail.push(part); + } + }); + stream.end(); + }); + compressed[compressed.length - 1] = merge([compressed[compressed.length - 1], ...tail]); manager.handleDataStreamPacket( new DataPacket({ @@ -450,8 +841,11 @@ describe('IncomingDataStreamManager', () => { }), Encryption_Type.NONE, ); + const reader = await readerPromise; + + // Send every write's compressed chunk (each ending on a Z_SYNC_FLUSH boundary) followed by + // the trailer... for (let index = 0; index < compressed.length; index += 1) { - const item = compressed[index]; manager.handleDataStreamPacket( new DataPacket({ participantIdentity: 'alice', @@ -460,7 +854,7 @@ describe('IncomingDataStreamManager', () => { value: new DataStream_Chunk({ streamId, chunkIndex: BigInt(index), - content: item, + content: compressed[index], version: 0, }), }, @@ -479,18 +873,10 @@ describe('IncomingDataStreamManager', () => { Encryption_Type.NONE, ); - // Make sure that stream chunks each get emitted sequentially and are passed out of the reader - const reader = await readerPromise; - const iterator = reader[Symbol.asyncIterator](); - - for (let index = 0; index < text.length; index += 1) { - const chunk = await iterator.next(); - expect(chunk.done).toStrictEqual(false); - console.log('AA', chunk.value, text[index]); - expect(chunk.value).toStrictEqual(text[index]); - } - const final = await iterator.next(); - expect(final.done).toStrictEqual(true); + // ...then read the whole stream back. The receiver re-chunks decompressed output on its own + // decompressor buffer boundaries (not the sender's per-write Z_SYNC_FLUSH boundaries), so the + // content is verified as a whole rather than per write. + expect(await reader.readAll()).toStrictEqual(text.join('')); }); it(`should ignore a v2 TEXT data stream with compression if DecompressionStream doesn't exist`, async () => { From 82b1f4c4016157a532831230f351a34e167578b3 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 17 Jun 2026 09:49:17 -0400 Subject: [PATCH 29/68] fix: reformat code --- src/room/Room.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/room/Room.ts b/src/room/Room.ts index b12aa72273..a8154f2e9c 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -2509,8 +2509,8 @@ class Room extends (EventEmitter as new () => TypedEmitter) } /** The client capabilities this SDK advertises to other participants in its `ClientInfo`. */ - private getClientInfoCapabilities(roomOptions: InternalRoomOptions): ClientInfo_Capability[] { - const capabilities: ClientInfo_Capability[] = []; + private getClientInfoCapabilities(roomOptions: InternalRoomOptions): Array { + const capabilities: Array = []; if (isFrameMetadataSupported(roomOptions.frameMetadata ?? roomOptions.packetTrailer) || !!this.e2eeManager) { capabilities.push(ClientInfo_Capability.CAP_PACKET_TRAILER); } @@ -2526,7 +2526,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) private getRemoteParticipantCapabilities = ( identity: Participant['identity'], - ): ClientInfo_Capability[] => { + ): Array => { return this.remoteParticipants.get(identity)?.capabilities ?? []; }; From c744642ee539de9be3c6aacf38b9272649951fb1 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 17 Jun 2026 09:49:28 -0400 Subject: [PATCH 30/68] fix: remove unused import --- src/room/data-stream/incoming/IncomingDataStreamManager.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts index f5933b25e1..572bfa2231 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -14,7 +14,6 @@ import { INLINE_PAYLOAD_ATTRIBUTE, COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RA import IncomingDataStreamManager from './IncomingDataStreamManager'; import type { ByteStreamReader, TextStreamReader } from './StreamReader'; import { deflateRawCompress } from '../compression'; -import { attributes, DataStreamError } from '../../..'; /** Builds a low quality random string of the given length. */ function randomText(length: number): string { From 01dfa70a635a5a0baefae9464b77e3cc6d99591e Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 17 Jun 2026 09:59:02 -0400 Subject: [PATCH 31/68] fix: remove Z_SYNC_FLUSH test, it was not really testing the right thing --- .../IncomingDataStreamManager.test.ts | 116 ------------------ 1 file changed, 116 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts index 572bfa2231..94fa6ef862 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -8,7 +8,6 @@ import { Encryption_Type, } from '@livekit/protocol'; import { describe, expect, it } from 'vitest'; -import { createDeflateRaw, constants } from 'zlib'; import { encodeBase64 } from '../../utils'; import { INLINE_PAYLOAD_ATTRIBUTE, COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RAW, STREAM_CHUNK_SIZE_BYTES } from '../constants'; import IncomingDataStreamManager from './IncomingDataStreamManager'; @@ -763,121 +762,6 @@ describe('IncomingDataStreamManager', () => { expect(await reader.readAll()).toStrictEqual(text); }); - it('should receive a v2 multi packet + compressed data stream and emit chunks at Z_SYNC_FLUSH boundaries', async () => { - const manager = new IncomingDataStreamManager(); - manager.setConnected(true); - - const readerPromise = new Promise((resolve) => { - manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); - }); - - const streamId = crypto.randomUUID(); - - // Generate a mostly incompressible compressed stream, with Z_SYNC_FLUSH between each chunk - const text = new Array(30).fill(0).map(() => `hello world${randomText(1_000)}`); - const bytes = text.map((b) => new TextEncoder().encode(b)); - const stream = createDeflateRaw(); - - const merge = (parts: Array): Uint8Array => { - const out = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0)); - let offset = 0; - for (const part of parts) { - out.set(part, offset); - offset += part.byteLength; - } - return out; - }; - // Drain all currently-buffered compressor output as a single Uint8Array. Reading in paused - // mode (rather than via 'data' events) means each drain right after a flush callback - // deterministically captures exactly that write's flushed bytes, with no event-timing race - // that could split a write's output across two chunks. - const drain = (): Uint8Array => { - const parts: Array = []; - let part: Uint8Array | null; - while ((part = stream.read() as Uint8Array | null) !== null) { - parts.push(part); - } - return merge(parts); - }; - - const compressed: Array = []; - for (const item of bytes) { - stream.write(item); - await new Promise((resolve) => stream.flush(constants.Z_SYNC_FLUSH, resolve)); - compressed.push(drain()); - } - // The final deflate block is emitted on end(); fold it into the last write's chunk so the - // receiver's single decompressor terminates cleanly right after emitting the last write. - const tail: Array = []; - await new Promise((resolve, reject) => { - stream.once('error', reject); - stream.once('end', resolve); - stream.on('readable', () => { - let part: Uint8Array | null; - while ((part = stream.read() as Uint8Array | null) !== null) { - tail.push(part); - } - }); - stream.end(); - }); - compressed[compressed.length - 1] = merge([compressed[compressed.length - 1], ...tail]); - - manager.handleDataStreamPacket( - new DataPacket({ - participantIdentity: 'alice', - value: { - case: 'streamHeader', - value: new DataStream_Header({ - streamId, - topic: 'my-topic', - mimeType: 'text/plain', - timestamp: 0n, - totalLength: BigInt(bytes.reduce((acc, i) => acc + i.byteLength, 0)), - attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, - contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, - }), - }, - }), - Encryption_Type.NONE, - ); - const reader = await readerPromise; - - // Send every write's compressed chunk (each ending on a Z_SYNC_FLUSH boundary) followed by - // the trailer... - for (let index = 0; index < compressed.length; index += 1) { - manager.handleDataStreamPacket( - new DataPacket({ - participantIdentity: 'alice', - value: { - case: 'streamChunk', - value: new DataStream_Chunk({ - streamId, - chunkIndex: BigInt(index), - content: compressed[index], - version: 0, - }), - }, - }), - Encryption_Type.NONE, - ); - } - manager.handleDataStreamPacket( - new DataPacket({ - participantIdentity: 'alice', - value: { - case: 'streamTrailer', - value: new DataStream_Trailer({ streamId }), - }, - }), - Encryption_Type.NONE, - ); - - // ...then read the whole stream back. The receiver re-chunks decompressed output on its own - // decompressor buffer boundaries (not the sender's per-write Z_SYNC_FLUSH boundaries), so the - // content is verified as a whole rather than per write. - expect(await reader.readAll()).toStrictEqual(text.join('')); - }); - it(`should ignore a v2 TEXT data stream with compression if DecompressionStream doesn't exist`, async () => { const text = 'hello world'; const compressed = await deflateRawCompress(new TextEncoder().encode(text)); From bead054334ad607cd6d91448c845c407bac2dc6e Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 17 Jun 2026 09:59:22 -0400 Subject: [PATCH 32/68] fix: add participant disconnect test --- .../IncomingDataStreamManager.test.ts | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts index 94fa6ef862..c2b527eb02 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -530,6 +530,59 @@ describe('IncomingDataStreamManager', () => { const reader = await readerPromise; await expect(reader.readAll()).rejects.toThrow('Extra chunk(s)'); }); + + it('should throw an error if participant disconnects while data stream is still not fully received', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + + // Send a header declaring 10 bytes, then a 5 byte long chunk + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: 10n, + attributes: { foo: 'bar', baz: 'quux' }, + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: 0n, + content: new Uint8Array([0x01, 0x02, 0x03, 0x04, 0x05]), + version: 0, + }), + }, + }), + Encryption_Type.NONE, + ); + + // Simulate a remote participant disconnect, which calls this method in the room handler + manager.validateParticipantHasNoActiveDataStreams('alice'); + + // Make sure an error is thrown from the reader + const reader = await readerPromise; + await expect(reader.readAll()).rejects.toThrow('Participant alice unexpectedly disconnected in the middle of sending data'); + }); }); describe('Receiving v2 data streams', () => { From 2455a65cc9f1913f7264ffc0a1a927699e74901f Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 17 Jun 2026 10:19:35 -0400 Subject: [PATCH 33/68] fix: add sendFile v1 outgoing data stream manager tests --- .../OutgoingDataStreamManager.test.ts | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts index 60af63e66e..49d1c01408 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts @@ -219,6 +219,39 @@ describe('OutgoingDataStreamManager', () => { expect(trailer.streamId).toStrictEqual(writer.info.id); expect(trailer.reason).toStrictEqual(''); }); + + it('should send a FILE via sendFile without compression (happy path)', async () => { + const bytes = new Uint8Array(20_000).fill(0x07); + const info = await manager.sendFile(new File([bytes as NonSharedUint8Array], 'text.txt'), { + topic: 'my-topic', + }); + + // Pre-v2 recipients: uncompressed, multi-packet legacy format. + // 20k of data -> 15k + 5k chunks. 1 header + 2 chunks + 1 trailer = 4 packets. + expect(sentPackets).toHaveLength(4); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('byteHeader'); + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); + + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + let chunk = chunkOf(sentPackets[1]); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toHaveLength(15_000); // MTU + expect(chunk.content.every((byte) => byte === 0x07)).toBeTruthy(); + + expect(sentPackets[2].value.case).toStrictEqual('streamChunk'); + chunk = chunkOf(sentPackets[2]); + expect(chunk.chunkIndex).toStrictEqual(1n); + expect(chunk.content).toHaveLength(5_000); + expect(chunk.content.every((byte) => byte === 0x07)).toBeTruthy(); + + expect(sentPackets[3].value.case).toStrictEqual('streamTrailer'); + expect(trailerOf(sentPackets[3]).streamId).toStrictEqual(info.id); + }); }); describe('v2 -> room of all v2', () => { let manager: OutgoingDataStreamManager, sentPackets: Array; @@ -580,6 +613,53 @@ describe('OutgoingDataStreamManager', () => { expect(trailer.streamId).toStrictEqual(info.id); expect(trailer.reason).toStrictEqual(''); }); + + it('should send a FILE via sendFile WITHOUT compression if remote does not support compression', async () => { + const bytes = new Uint8Array(10_000).fill(0x07); + const info = await manager.sendFile(new File([bytes as NonSharedUint8Array], 'text.txt'), { + topic: 'my-topic', + destinationIdentities: ['noCompression'], + }); + + // v2 recipient but no deflate-raw capability: uncompressed, multi-packet (never inline). + expect(sentPackets).toHaveLength(3); + + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.contentHeader.case).toBe('byteHeader'); + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); + expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeUndefined(); + + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + const chunk = chunkOf(sentPackets[1]); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toHaveLength(10_000); // uncompressed, single chunk under the MTU + expect(chunk.content.every((byte) => byte === 0x07)).toBeTruthy(); + + expect(sentPackets[2].value.case).toStrictEqual('streamTrailer'); + expect(trailerOf(sentPackets[2]).streamId).toStrictEqual(info.id); + }); + + it('should send an empty FILE via sendFile', async () => { + const info = await manager.sendFile(new File([], 'empty.bin'), { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + }); + + // An empty file still produces a well-formed compressed byte stream: a header declaring zero + // length, the deflate stream's final block, and a trailer. + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.contentHeader.case).toBe('byteHeader'); + expect(header.totalLength).toStrictEqual(0n); + expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + + const last = sentPackets[sentPackets.length - 1]; + expect(last.value.case).toStrictEqual('streamTrailer'); + expect(trailerOf(last).streamId).toStrictEqual(info.id); + }); }); describe('v2 -> room of mixed v1 / v2', () => { let manager: OutgoingDataStreamManager, sentPackets: Array; From 33a3ce4c0d231224382c026387747c731b7a4635 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 23 Jun 2026 15:44:12 -0400 Subject: [PATCH 34/68] feat: port over data streams implementation to use newly added proto fields --- package.json | 2 +- pnpm-lock.yaml | 10 +- src/room/Room.test.ts | 5 +- src/room/Room.ts | 6 +- src/room/data-stream/constants.ts | 33 ------ .../IncomingDataStreamManager.test.ts | 64 +++++----- .../incoming/IncomingDataStreamManager.ts | 58 ++++----- .../OutgoingDataStreamManager.test.ts | 112 +++++++++++------- .../outgoing/OutgoingDataStreamManager.ts | 45 ++++--- src/room/data-stream/outgoing/header-utils.ts | 21 +++- src/room/participant/RemoteParticipant.ts | 5 +- 11 files changed, 183 insertions(+), 178 deletions(-) diff --git a/package.json b/package.json index 7292726bb6..16172f300c 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ }, "dependencies": { "@livekit/mutex": "1.1.1", - "@livekit/protocol": "1.46.8", + "@livekit/protocol": "1.48.0", "events": "^3.3.0", "jose": "^6.1.0", "loglevel": "^1.9.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7d9969f8a1..b74bfe3cb3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: 1.1.1 version: 1.1.1 '@livekit/protocol': - specifier: 1.46.8 - version: 1.46.8 + specifier: 1.48.0 + version: 1.48.0 '@types/dom-mediacapture-record': specifier: ^1 version: 1.0.22 @@ -1224,8 +1224,8 @@ packages: '@livekit/mutex@1.1.1': resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} - '@livekit/protocol@1.46.8': - resolution: {integrity: sha512-mOjcCVLy4Q7qEaEE7gGLi5wXan0K3VTvSpto5Y0ftek2hauALxBW0+cyxNRoakT7dbWFfH+gqc2XQM0P4M1Q/g==} + '@livekit/protocol@1.48.0': + resolution: {integrity: sha512-fYHYgltH6YavAsokl3qsHLkBdQeKCl4UORVTub5crS3t8JtKFZ0uinHDFQ+XXdNKS6Ub9gcOjV+UHcDiqnWXoQ==} '@livekit/throws-transformer@0.1.3': resolution: {integrity: sha512-PBttE6W6g/2ALGu6kWOunZ5qdrXwP9Ge1An2/62OfE6Rhc0Abd4yp6ex2pWhwUfGxDsSZvFgoB1Ia/5mWAMuKQ==} @@ -5121,7 +5121,7 @@ snapshots: '@livekit/mutex@1.1.1': {} - '@livekit/protocol@1.46.8': + '@livekit/protocol@1.48.0': dependencies: '@bufbuild/protobuf': 1.10.1 diff --git a/src/room/Room.test.ts b/src/room/Room.test.ts index b6703d436b..ea837de892 100644 --- a/src/room/Room.test.ts +++ b/src/room/Room.test.ts @@ -81,7 +81,10 @@ describe('Room signaling options', () => { 'wss://test.livekit.io', 'test-token', expect.objectContaining({ - clientInfoCapabilities: [ClientInfo_Capability.CAP_PACKET_TRAILER], + clientInfoCapabilities: [ + ClientInfo_Capability.CAP_PACKET_TRAILER, + ClientInfo_Capability.CAP_COMPRESSION_DEFLATE_RAW, + ], e2eeEnabled: true, }), expect.any(AbortSignal), diff --git a/src/room/Room.ts b/src/room/Room.ts index a8154f2e9c..a243fa8d33 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -2509,11 +2509,15 @@ class Room extends (EventEmitter as new () => TypedEmitter) } /** The client capabilities this SDK advertises to other participants in its `ClientInfo`. */ - private getClientInfoCapabilities(roomOptions: InternalRoomOptions): Array { + private getClientInfoCapabilities( + roomOptions: InternalRoomOptions, + ): Array { const capabilities: Array = []; if (isFrameMetadataSupported(roomOptions.frameMetadata ?? roomOptions.packetTrailer) || !!this.e2eeManager) { capabilities.push(ClientInfo_Capability.CAP_PACKET_TRAILER); } + // Advertise deflate-raw decompression support so peers know they can send us compressed data + // streams (gated separately from clientProtocol — see the data streams v2 spec). if (isCompressionStreamSupported()) { capabilities.push(ClientInfo_Capability.CAP_COMPRESSION_DEFLATE_RAW); } diff --git a/src/room/data-stream/constants.ts b/src/room/data-stream/constants.ts index c4f7dcfdf0..4c072f3eb0 100644 --- a/src/room/data-stream/constants.ts +++ b/src/room/data-stream/constants.ts @@ -1,12 +1,3 @@ -/** - * Reserved data-stream header attribute used to "smuggle" a small payload directly inside a single - * `streamHeader` packet, avoiding the separate `streamChunk`/`streamTrailer` packets a normal data - * stream requires. For text streams the value is the raw string; for byte streams it is base64. - * - * @internal - */ -export const INLINE_PAYLOAD_ATTRIBUTE = 'lk.inline_payload'; - /** * Maximum size of a single data-stream chunk in bytes, and the budget used to decide whether a * payload can be sent inline as a single header packet. Kept below the ~16k data-channel MTU to @@ -15,27 +6,3 @@ export const INLINE_PAYLOAD_ATTRIBUTE = 'lk.inline_payload'; * @internal */ export const STREAM_CHUNK_SIZE_BYTES = 15_000; - -/** - * Reserved data-stream header attribute signaling that the payload (inline or chunked) is - * compressed. Self-describing: the sender sets it when it compresses, and the receiver decompresses - * iff it is present. Both inline and chunked text and byte streams use {@link COMPRESSION_DEFLATE_RAW}. - * - * @internal - */ -export const COMPRESSION_ATTRIBUTE = 'lk.compression'; - -/** - * Value of {@link COMPRESSION_ATTRIBUTE} for raw-deflate-compressed payloads. - * - * For inline (single-packet) payloads this is a one-shot raw-deflate buffer, base64'd into the - * payload attribute. For chunked streams it is a single raw-deflate context spanning the whole - * stream, terminated by a final block before the trailer; receivers concatenate chunk contents in - * `chunkIndex` order through one raw-deflate (windowBits -15) decompressor. The format also - * supports sync-flushing at write boundaries (context takeover) so a future incremental sender - * could compress without a protocol change, though current senders (`sendText`/`sendFile`) - * compress the full payload in one shot. - * - * @internal - */ -export const COMPRESSION_DEFLATE_RAW = 'deflate-raw'; diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts index c2b527eb02..67e9421c28 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -2,17 +2,17 @@ import { DataPacket, DataStream_ByteHeader, DataStream_Chunk, + DataStream_CompressionType, DataStream_Header, DataStream_TextHeader, DataStream_Trailer, Encryption_Type, } from '@livekit/protocol'; import { describe, expect, it } from 'vitest'; -import { encodeBase64 } from '../../utils'; -import { INLINE_PAYLOAD_ATTRIBUTE, COMPRESSION_ATTRIBUTE, COMPRESSION_DEFLATE_RAW, STREAM_CHUNK_SIZE_BYTES } from '../constants'; +import { deflateRawCompress } from '../compression'; +import { STREAM_CHUNK_SIZE_BYTES } from '../constants'; import IncomingDataStreamManager from './IncomingDataStreamManager'; import type { ByteStreamReader, TextStreamReader } from './StreamReader'; -import { deflateRawCompress } from '../compression'; /** Builds a low quality random string of the given length. */ function randomText(length: number): string { @@ -171,7 +171,8 @@ describe('IncomingDataStreamManager', () => { mimeType: 'text/plain', timestamp: 0n, totalLength: BigInt(text.length), - attributes: { [INLINE_PAYLOAD_ATTRIBUTE]: text, foo: 'bar' }, + attributes: { foo: 'bar' }, + inlineContent: new TextEncoder().encode(text), contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({ @@ -234,7 +235,9 @@ describe('IncomingDataStreamManager', () => { expect(streamReader.info.attachedStreamIds).toHaveLength(1); const attachmentStreamReader = await attachmentStreamReaderPromise; - expect(await attachmentStreamReader.readAll()).toStrictEqual([new Uint8Array([0x01, 0x02, 0x03])]); + expect(await attachmentStreamReader.readAll()).toStrictEqual([ + new Uint8Array([0x01, 0x02, 0x03]), + ]); expect(streamReader.info.attachedStreamIds).toHaveLength(1); }); @@ -357,7 +360,10 @@ describe('IncomingDataStreamManager', () => { participantIdentity: 'alice', value: { case: 'streamTrailer', - value: new DataStream_Trailer({ streamId, attributes: { hello: 'world', foo: 'updated' } }), + value: new DataStream_Trailer({ + streamId, + attributes: { hello: 'world', foo: 'updated' }, + }), }, }), Encryption_Type.NONE, @@ -581,7 +587,9 @@ describe('IncomingDataStreamManager', () => { // Make sure an error is thrown from the reader const reader = await readerPromise; - await expect(reader.readAll()).rejects.toThrow('Participant alice unexpectedly disconnected in the middle of sending data'); + await expect(reader.readAll()).rejects.toThrow( + 'Participant alice unexpectedly disconnected in the middle of sending data', + ); }); }); @@ -608,7 +616,8 @@ describe('IncomingDataStreamManager', () => { mimeType: 'text/plain', timestamp: 0n, totalLength: BigInt(text.length), - attributes: { [INLINE_PAYLOAD_ATTRIBUTE]: text, foo: 'bar' }, + attributes: { foo: 'bar' }, + inlineContent: new TextEncoder().encode(text), contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, }), }, @@ -630,7 +639,7 @@ describe('IncomingDataStreamManager', () => { }); const streamId = crypto.randomUUID(); - const bytes = encodeBase64(new Uint8Array([0x01, 0x02, 0x03])); + const bytes = new Uint8Array([0x01, 0x02, 0x03]); manager.handleDataStreamPacket( new DataPacket({ @@ -643,7 +652,7 @@ describe('IncomingDataStreamManager', () => { mimeType: 'text/plain', timestamp: 0n, totalLength: 3n, - attributes: { [INLINE_PAYLOAD_ATTRIBUTE]: bytes }, + inlineContent: bytes, contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({}) }, }), }, @@ -665,7 +674,7 @@ describe('IncomingDataStreamManager', () => { const streamId = crypto.randomUUID(); const text = 'hello world'; - const compressed = encodeBase64(await deflateRawCompress(new TextEncoder().encode(text))); + const compressed = await deflateRawCompress(new TextEncoder().encode(text)); manager.handleDataStreamPacket( new DataPacket({ @@ -678,11 +687,9 @@ describe('IncomingDataStreamManager', () => { mimeType: 'text/plain', timestamp: 0n, totalLength: BigInt(text.length), - attributes: { - [INLINE_PAYLOAD_ATTRIBUTE]: compressed, - [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW, - foo: 'bar' - }, + attributes: { foo: 'bar' }, + compression: DataStream_CompressionType.DEFLATE_RAW, + inlineContent: compressed, contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, }), }, @@ -705,7 +712,7 @@ describe('IncomingDataStreamManager', () => { const streamId = crypto.randomUUID(); const bytes = new Uint8Array([0x01, 0x02, 0x03]); - const compressed = encodeBase64(await deflateRawCompress(bytes)); + const compressed = await deflateRawCompress(bytes); manager.handleDataStreamPacket( new DataPacket({ @@ -718,10 +725,8 @@ describe('IncomingDataStreamManager', () => { mimeType: 'text/plain', timestamp: 0n, totalLength: BigInt(bytes.length), - attributes: { - [INLINE_PAYLOAD_ATTRIBUTE]: compressed, - [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW, - }, + compression: DataStream_CompressionType.DEFLATE_RAW, + inlineContent: compressed, contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({}) }, }), }, @@ -745,7 +750,10 @@ describe('IncomingDataStreamManager', () => { // NOTE: mostly incompressible, but the hello world parts repeating should mean that the compressed // contents is smaller than the full uncompressed data. - const text = new Array(30).fill(null).map(() => `hello world${randomText(1_000)}`).join(''); + const text = new Array(30) + .fill(null) + .map(() => `hello world${randomText(1_000)}`) + .join(''); const compressed = await deflateRawCompress(new TextEncoder().encode(text)); @@ -763,7 +771,7 @@ describe('IncomingDataStreamManager', () => { mimeType: 'text/plain', timestamp: 0n, totalLength: BigInt(text.length), - attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, + compression: DataStream_CompressionType.DEFLATE_RAW, contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, }), }, @@ -819,7 +827,8 @@ describe('IncomingDataStreamManager', () => { const text = 'hello world'; const compressed = await deflateRawCompress(new TextEncoder().encode(text)); - let originalCompressionStream: typeof CompressionStream, originalDecompressionStream: typeof DecompressionStream; + let originalCompressionStream: typeof CompressionStream, + originalDecompressionStream: typeof DecompressionStream; try { originalCompressionStream = CompressionStream; (CompressionStream as any) = undefined; @@ -846,7 +855,7 @@ describe('IncomingDataStreamManager', () => { mimeType: 'text/plain', timestamp: 0n, totalLength: BigInt(text.length), - attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, + compression: DataStream_CompressionType.DEFLATE_RAW, contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, }), }, @@ -893,7 +902,8 @@ describe('IncomingDataStreamManager', () => { const bytes = new Uint8Array([0x01, 0x02, 0x03]); const compressed = await deflateRawCompress(bytes); - let originalCompressionStream: typeof CompressionStream, originalDecompressionStream: typeof DecompressionStream; + let originalCompressionStream: typeof CompressionStream, + originalDecompressionStream: typeof DecompressionStream; try { originalCompressionStream = CompressionStream; (CompressionStream as any) = undefined; @@ -920,7 +930,7 @@ describe('IncomingDataStreamManager', () => { mimeType: 'text/plain', timestamp: 0n, totalLength: BigInt(bytes.length), - attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, + compression: DataStream_CompressionType.DEFLATE_RAW, contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, }), }, diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 3f46efdea0..449cf52873 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -1,6 +1,7 @@ import { type DataPacket, DataStream_Chunk, + DataStream_CompressionType, DataStream_Header, DataStream_Trailer, Encryption_Type, @@ -8,13 +9,8 @@ import { import log from '../../../logger'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { type ByteStreamInfo, type StreamController, type TextStreamInfo } from '../../types'; -import { bigIntToNumber, decodeBase64, isCompressionStreamSupported, numberToBigInt } from '../../utils'; +import { bigIntToNumber, isCompressionStreamSupported, numberToBigInt } from '../../utils'; import { deflateRawDecompress, inflateRawStream } from '../compression'; -import { - COMPRESSION_ATTRIBUTE, - COMPRESSION_DEFLATE_RAW, - INLINE_PAYLOAD_ATTRIBUTE, -} from '../constants'; import { type ByteStreamHandler, ByteStreamReader, @@ -165,33 +161,30 @@ export default class IncomingDataStreamManager { }; // Both inline and chunked byte payloads are deflate-raw compressed; inline as a one-shot - // base64 buffer, chunked as a single stream spanning all chunks (mirrors text). - const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_DEFLATE_RAW; + // buffer, chunked as a single stream spanning all chunks (mirrors text). The compression + // flag rides in the header's `compression` field. + const compressed = streamHeader.compression === DataStream_CompressionType.DEFLATE_RAW; if (compressed && !isCompressionStreamSupported()) { // NOTE: this shouldn't really ever happen, if this warning is logged then the sender // isn't properly abiding by the data streams v2 protocol. log.warn( - `Data stream ${streamHeader.streamId} received with ${info.attributes![COMPRESSION_ATTRIBUTE]} compression, but this browser does not have support for DecompressionStream. Dropping...`, + `Data stream ${streamHeader.streamId} received with deflate-raw compression, but this browser does not have support for DecompressionStream. Dropping...`, ); return; } - // Single-packet stream: the entire payload was smuggled into a reserved header attribute. + // Single-packet stream: the entire payload was smuggled into the header's `inlineContent`. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. - const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; - if (typeof inlinePayload !== 'undefined') { - delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; - delete info.attributes![COMPRESSION_ATTRIBUTE]; - // Inline bytes are always base64 (binary isn't a valid attribute string), optionally - // deflate-raw compressed. - const bytes = decodeBase64(inlinePayload); + const inlineContent = streamHeader.inlineContent; + if (typeof inlineContent !== 'undefined') { + // Inline bytes are the raw payload, optionally deflate-raw compressed. streamHandlerCallback( new ByteStreamReader( info, createInlineStream( streamHeader.streamId, - compressed ? deflateRawDecompress(bytes) : bytes, + compressed ? deflateRawDecompress(inlineContent) : inlineContent, ), bigIntToNumber(streamHeader.totalLength), ), @@ -200,10 +193,6 @@ export default class IncomingDataStreamManager { return; } - if (compressed) { - delete info.attributes![COMPRESSION_ATTRIBUTE]; - } - const stream = new ReadableStream({ start: (controller) => { streamController = controller; @@ -261,28 +250,25 @@ export default class IncomingDataStreamManager { }; // Both inline and chunked text payloads are deflate-raw compressed; inline as a one-shot - // buffer, chunked as a single stream spanning all chunks (see COMPRESSION_DEFLATE_RAW). - const compressed = info.attributes![COMPRESSION_ATTRIBUTE] === COMPRESSION_DEFLATE_RAW; + // buffer, chunked as a single stream spanning all chunks. The compression flag rides in the + // header's `compression` field. + const compressed = streamHeader.compression === DataStream_CompressionType.DEFLATE_RAW; if (compressed && !isCompressionStreamSupported()) { // NOTE: this shouldn't really ever happen, if this warning is logged then the sender // isn't properly abiding by the data streams v2 protocol. log.warn( - `Data stream ${streamHeader.streamId} received with ${info.attributes![COMPRESSION_ATTRIBUTE]} compression, but this browser does not have support for DecompressionStream. Dropping...`, + `Data stream ${streamHeader.streamId} received with deflate-raw compression, but this browser does not have support for DecompressionStream. Dropping...`, ); return; } - // Single-packet stream: the entire payload was smuggled into a reserved header attribute. + // Single-packet stream: the entire payload was smuggled into the header's `inlineContent`. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. - const inlinePayload = streamHeader.attributes[INLINE_PAYLOAD_ATTRIBUTE]; - if (typeof inlinePayload !== 'undefined') { - delete info.attributes![INLINE_PAYLOAD_ATTRIBUTE]; - delete info.attributes![COMPRESSION_ATTRIBUTE]; - // Compressed text is base64(deflate-raw(utf-8)); uncompressed text is the raw string. - const content = compressed - ? deflateRawDecompress(decodeBase64(inlinePayload)) - : new TextEncoder().encode(inlinePayload); + const inlineContent = streamHeader.inlineContent; + if (typeof inlineContent !== 'undefined') { + // Inline text is the raw UTF-8 payload, optionally deflate-raw compressed. + const content = compressed ? deflateRawDecompress(inlineContent) : inlineContent; streamHandlerCallback( new TextStreamReader( info, @@ -294,10 +280,6 @@ export default class IncomingDataStreamManager { return; } - if (compressed) { - delete info.attributes![COMPRESSION_ATTRIBUTE]; - } - const stream = new ReadableStream({ start: (controller) => { streamController = controller; diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts index 49d1c01408..ac6a9da785 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts @@ -1,17 +1,16 @@ -import { ClientInfo_Capability, type DataPacket } from '@livekit/protocol'; +import { + ClientInfo_Capability, + type DataPacket, + DataStream_CompressionType, +} from '@livekit/protocol'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import log from '../../../logger'; import { - CLIENT_PROTOCOL_DATA_STREAM_RPC, + CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DATA_STREAM_V2, CLIENT_PROTOCOL_DEFAULT, } from '../../../version'; import type RTCEngine from '../../RTCEngine'; -import { - COMPRESSION_ATTRIBUTE, - COMPRESSION_DEFLATE_RAW, - INLINE_PAYLOAD_ATTRIBUTE, -} from '../constants'; import OutgoingDataStreamManager from './OutgoingDataStreamManager'; /** Builds a low quality random string of the given length. */ @@ -38,7 +37,9 @@ function randomBytes(length: number): Uint8Array { * protocol each advertises. Defaults to a single v2 participant named "bob". */ function createManager( - participants: Record]> = { bob: CLIENT_PROTOCOL_DATA_STREAM_V2 }, + participants: Record]> = { + bob: CLIENT_PROTOCOL_DATA_STREAM_V2, + }, ) { const sentPackets: DataPacket[] = []; const engine = { @@ -52,8 +53,14 @@ function createManager( const manager = new OutgoingDataStreamManager( engine, log, - (identity) => (Array.isArray(participants[identity]) ? participants[identity][0] : participants[identity]) ?? CLIENT_PROTOCOL_DEFAULT, - (identity) => Array.isArray(participants[identity]) ? participants[identity][1] : [ClientInfo_Capability.CAP_COMPRESSION_DEFLATE_RAW], + (identity) => + (Array.isArray(participants[identity]) + ? participants[identity][0] + : participants[identity]) ?? CLIENT_PROTOCOL_DEFAULT, + (identity) => + Array.isArray(participants[identity]) + ? participants[identity][1] + : [ClientInfo_Capability.CAP_COMPRESSION_DEFLATE_RAW], () => Object.keys(participants), ); return { manager, sentPackets }; @@ -154,8 +161,8 @@ describe('OutgoingDataStreamManager', () => { expect(header.contentHeader.case).toBe('textHeader'); for (let i = 0; i < 3; i += 1) { - expect(sentPackets[i+1].value.case).toStrictEqual('streamChunk'); - const chunk = chunkOf(sentPackets[i+1]); + expect(sentPackets[i + 1].value.case).toStrictEqual('streamChunk'); + const chunk = chunkOf(sentPackets[i + 1]); expect(chunk.streamId).toStrictEqual(info.id); expect(chunk.chunkIndex).toStrictEqual(BigInt(i)); expect(chunk.content.every((char) => char === 'A'.charCodeAt(0))).toBeTruthy(); @@ -235,7 +242,7 @@ describe('OutgoingDataStreamManager', () => { expect(header.streamId).toStrictEqual(info.id); expect(header.topic).toStrictEqual('my-topic'); expect(header.contentHeader.case).toBe('byteHeader'); - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); + expect(header.compression).toBe(DataStream_CompressionType.NONE); expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); let chunk = chunkOf(sentPackets[1]); @@ -282,9 +289,11 @@ describe('OutgoingDataStreamManager', () => { expect(header.contentHeader.case).toBe('textHeader'); // Make sure the contents of that packet was compressed - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeTypeOf('string'); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).not.toStrictEqual('hello hello compressible world'); + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); + expect(header.inlineContent).toBeInstanceOf(Uint8Array); + expect(header.inlineContent).not.toStrictEqual( + new TextEncoder().encode('hello hello compressible world'), + ); }); it('should send short TEXT data stream with uncompressible payload in single packet', async () => { const info = await manager.sendText('short', { @@ -304,8 +313,8 @@ describe('OutgoingDataStreamManager', () => { // Make sure the contents of that packet was uncompressed - "short" isn't long enough to // meaningfully compress with DEFLATE - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toStrictEqual('short'); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toStrictEqual(new TextEncoder().encode('short')); }); it('should send short data stream with single packet and NO compression if remote participant does not support compression', async () => { const info = await manager.sendText('hello hello compressible world', { @@ -324,8 +333,10 @@ describe('OutgoingDataStreamManager', () => { expect(header.contentHeader.case).toBe('textHeader'); // Make sure the contents of that packet was NOT compressed - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toStrictEqual('hello hello compressible world'); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toStrictEqual( + new TextEncoder().encode('hello hello compressible world'), + ); }); it('should send long but highly compressible TEXT data stream as single packet', async () => { // A phrase which repeats over and over should compress extremely well. @@ -347,14 +358,19 @@ describe('OutgoingDataStreamManager', () => { expect(header.contentHeader.case).toBe('textHeader'); // Make sure the contents of that packet was compressed - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeTypeOf('string'); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]?.startsWith('hello world')).toBeFalsy(); + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); + expect(header.inlineContent).toBeInstanceOf(Uint8Array); + // Compressed bytes must not begin with the raw UTF-8 prefix of the payload. + const helloWorld = new TextEncoder().encode('hello world'); + expect(header.inlineContent!.slice(0, helloWorld.length)).not.toStrictEqual(helloWorld); }); it('should send long but somewhat compressible data stream as a compressed multi packet data stream', async () => { // Mostly incompressible, but the hello world parts repeating should mean that the compressed // contents is smaller than the full uncompressed data. - const text = new Array(50).fill(null).map(() => `hello world${randomText(1_000)}`).join(''); + const text = new Array(50) + .fill(null) + .map(() => `hello world${randomText(1_000)}`) + .join(''); const info = await manager.sendText(text, { topic: 'my-topic', @@ -374,7 +390,7 @@ describe('OutgoingDataStreamManager', () => { expect(header.contentHeader.case).toBe('textHeader'); // Make sure the contents of that packet was compressed - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); // Verify there are three data packets: expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); @@ -411,7 +427,7 @@ describe('OutgoingDataStreamManager', () => { expect(header.contentHeader.case).toBe('byteHeader'); // Make sure the contents of that packet was NOT compressed - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); // Verify there are four data packets: let totalLength = 0; @@ -463,15 +479,19 @@ describe('OutgoingDataStreamManager', () => { expect(header.topic).toStrictEqual('my-topic'); expect(header.contentHeader.case).toBe('textHeader'); - // Make sure the contents of that packet was compressed - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeTypeOf('string'); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toStrictEqual('hello hello compressible world'); + // Make sure the contents of that packet was NOT compressed (compress: false opt-out) + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toStrictEqual( + new TextEncoder().encode('hello hello compressible world'), + ); }); it('should send long but somewhat compressible data stream but skip compression due to compress: false being passed', async () => { // Mostly incompressible, but the hello world parts repeating should mean that the compressed // contents is smaller than the full uncompressed data. - const text = new Array(50).fill(null).map(() => `hello world${randomText(1_000)}`).join(''); + const text = new Array(50) + .fill(null) + .map(() => `hello world${randomText(1_000)}`) + .join(''); const info = await manager.sendText(text, { topic: 'my-topic', @@ -490,7 +510,7 @@ describe('OutgoingDataStreamManager', () => { expect(header.contentHeader.case).toBe('textHeader'); // Make sure the contents of that packet was uncompressed - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); + expect(header.compression).toBe(DataStream_CompressionType.NONE); // Verify there are four data packets: expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); @@ -524,7 +544,7 @@ describe('OutgoingDataStreamManager', () => { expect(header.streamId).toStrictEqual(writer.info.id); expect(header.topic).toStrictEqual('my-topic'); expect(header.contentHeader.case).toBe('textHeader'); - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); // Make sure compression is disabled + expect(header.compression).toBe(DataStream_CompressionType.NONE); // Make sure compression is disabled await writer.write('hello world'); @@ -556,7 +576,7 @@ describe('OutgoingDataStreamManager', () => { expect(header.streamId).toStrictEqual(writer.info.id); expect(header.topic).toStrictEqual('my-topic'); expect(header.contentHeader.case).toBe('byteHeader'); - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); // Make sure compression is disabled + expect(header.compression).toBe(DataStream_CompressionType.NONE); // Make sure compression is disabled await writer.write(new Uint8Array([0x00, 0x01, 0x02, 0x03])); @@ -584,7 +604,7 @@ describe('OutgoingDataStreamManager', () => { }); // Should be a multi-packet result - // + // // Sending single packet data streams for files is tricky because it's really difficult to // determine ahead of time if a file can fit into a single packet without a ton of ahead of // time in memory buffering. @@ -597,7 +617,7 @@ describe('OutgoingDataStreamManager', () => { expect(header.contentHeader.case).toBe('byteHeader'); // Make sure the contents of that packet was NOT compressed - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); let chunk = chunkOf(sentPackets[1]); @@ -628,8 +648,8 @@ describe('OutgoingDataStreamManager', () => { const header = headerOf(sentPackets[0]); expect(header.streamId).toStrictEqual(info.id); expect(header.contentHeader.case).toBe('byteHeader'); - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeUndefined(); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toBeUndefined(); expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); const chunk = chunkOf(sentPackets[1]); @@ -654,7 +674,7 @@ describe('OutgoingDataStreamManager', () => { expect(header.streamId).toStrictEqual(info.id); expect(header.contentHeader.case).toBe('byteHeader'); expect(header.totalLength).toStrictEqual(0n); - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); const last = sentPackets[sentPackets.length - 1]; expect(last.value.case).toStrictEqual('streamTrailer'); @@ -717,9 +737,11 @@ describe('OutgoingDataStreamManager', () => { expect(header.contentHeader.case).toBe('textHeader'); // Make sure the contents of that packet was compressed - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toStrictEqual(COMPRESSION_DEFLATE_RAW); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toBeTypeOf('string'); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).not.toStrictEqual('hello hello compressible world'); + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); + expect(header.inlineContent).toBeInstanceOf(Uint8Array); + expect(header.inlineContent).not.toStrictEqual( + new TextEncoder().encode('hello hello compressible world'), + ); }); it('should send data stream using data stream v2 format but NO compression when only sending to a subset of participants where one does NOT support compression', async () => { const info = await manager.sendText('hello hello compressible world', { @@ -738,8 +760,10 @@ describe('OutgoingDataStreamManager', () => { expect(header.contentHeader.case).toBe('textHeader'); // Make sure the contents of that packet was compressed - expect(header.attributes?.[COMPRESSION_ATTRIBUTE]).toBeUndefined(); - expect(header.attributes?.[INLINE_PAYLOAD_ATTRIBUTE]).toStrictEqual('hello hello compressible world'); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toStrictEqual( + new TextEncoder().encode('hello hello compressible world'), + ); }); }); }); diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 3f51bc378a..60fc6d35ec 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -3,6 +3,7 @@ import { ClientInfo_Capability, DataPacket, DataStream_Chunk, + DataStream_CompressionType, DataStream_Trailer, Encryption_Type, } from '@livekit/protocol'; @@ -21,7 +22,6 @@ import type { TextStreamInfo, } from '../../types'; import { - encodeBase64, isCompressionStreamSupported, numberToBigInt, readBytesInChunks, @@ -29,12 +29,7 @@ import { splitUtf8, } from '../../utils'; import { deflateRawCompress, deflateRawCompressReadable } from '../compression'; -import { - COMPRESSION_ATTRIBUTE, - COMPRESSION_DEFLATE_RAW, - INLINE_PAYLOAD_ATTRIBUTE, - STREAM_CHUNK_SIZE_BYTES, -} from '../constants'; +import { STREAM_CHUNK_SIZE_BYTES } from '../constants'; import { ByteStreamWriter, TextStreamWriter } from './StreamWriter'; import { buildByteStreamHeader, @@ -105,23 +100,25 @@ export default class OutgoingDataStreamManager { // Phase 1: Try to send as a single packet data stream const noAttachments = !options?.attachments || options.attachments.length === 0; if (noAttachments && this.allRecipientsSupportV2(options?.destinationIdentities)) { - // Compress when the runtime supports it, but only keep the result if it actually shrinks the - // payload (deflate framing plus the base64 expansion makes tiny strings larger). Uncompressed - // inline payloads stay as the raw string; compressed ones are base64'd and flagged via an - // attribute. - const inlineAttributes: Record = { - ...info.attributes, - [INLINE_PAYLOAD_ATTRIBUTE]: text, - }; - if (compress && isCompressionStreamSupported() && this.allRecipientsSupportCompression(options?.destinationIdentities)) { + // The payload rides in the header's `inlineContent` (raw bytes). Compress when the runtime + // supports it, but only keep the result if it actually shrinks the payload (deflate framing + // makes tiny strings larger). The compression flag is carried in the header's `compression` + // field; user attributes are left untouched. + let inlineContent: Uint8Array = textInBytes; + let compression = DataStream_CompressionType.NONE; + if ( + compress && + isCompressionStreamSupported() && + this.allRecipientsSupportCompression(options?.destinationIdentities) + ) { const compressed = await deflateRawCompress(textInBytes); if (compressed.byteLength < textInBytes.byteLength) { - inlineAttributes[INLINE_PAYLOAD_ATTRIBUTE] = encodeBase64(compressed); - inlineAttributes[COMPRESSION_ATTRIBUTE] = COMPRESSION_DEFLATE_RAW; + inlineContent = compressed; + compression = DataStream_CompressionType.DEFLATE_RAW; } } - const header = buildTextStreamHeader({ ...info, attributes: inlineAttributes }); + const header = buildTextStreamHeader(info, undefined, { compression, inlineContent }); const packet = createStreamHeaderPacket(header, options?.destinationIdentities); if (packet.toBinary().byteLength <= STREAM_CHUNK_SIZE_BYTES) { @@ -148,10 +145,11 @@ export default class OutgoingDataStreamManager { this.allRecipientsSupportV2(options?.destinationIdentities) && this.allRecipientsSupportCompression(options?.destinationIdentities) ) { - info.attributes = { ...info.attributes, [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }; info.attachedStreamIds = fileIds; - const header = buildTextStreamHeader(info); + const header = buildTextStreamHeader(info, undefined, { + compression: DataStream_CompressionType.DEFLATE_RAW, + }); const packet = createStreamHeaderPacket(header, options?.destinationIdentities); await this.sendChunkedByteStream( packet, @@ -383,13 +381,14 @@ export default class OutgoingDataStreamManager { topic: options?.topic ?? '', timestamp: Date.now(), size: file.size, - attributes: { [COMPRESSION_ATTRIBUTE]: COMPRESSION_DEFLATE_RAW }, encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled ? Encryption_Type.GCM : Encryption_Type.NONE, }; - const header = buildByteStreamHeader(info); + const header = buildByteStreamHeader(info, { + compression: DataStream_CompressionType.DEFLATE_RAW, + }); const packet = createStreamHeaderPacket(header, destinationIdentities); await this.sendChunkedByteStream( packet, diff --git a/src/room/data-stream/outgoing/header-utils.ts b/src/room/data-stream/outgoing/header-utils.ts index e49a3ff108..301f38bcad 100644 --- a/src/room/data-stream/outgoing/header-utils.ts +++ b/src/room/data-stream/outgoing/header-utils.ts @@ -1,6 +1,7 @@ import { DataPacket, DataStream_ByteHeader, + DataStream_CompressionType, DataStream_Header, DataStream_OperationType, DataStream_TextHeader, @@ -8,10 +9,21 @@ import { import type { ByteStreamInfo, StreamTextOptions, TextStreamInfo } from '../../types'; import { numberToBigInt } from '../../utils'; +/** The data-streams-v2 wire signals carried directly on the header: the compression flag and the + * inline single-packet payload. Both used to live in reserved header attributes; they are now + * first-class protobuf fields on `DataStream.Header`. */ +export interface StreamHeaderV2Fields { + /** Compression applied to the inline/chunked payload. Defaults to `NONE` when omitted. */ + compression?: DataStream_CompressionType; + /** The full payload smuggled into the header for single-packet (inline) sends. */ + inlineContent?: Uint8Array; +} + /** Builds the `DataStream_Header` for a text stream from its info and stream options. */ export function buildTextStreamHeader( info: TextStreamInfo, options?: Pick, + v2?: StreamHeaderV2Fields, ): DataStream_Header { return new DataStream_Header({ streamId: info.id, @@ -20,6 +32,8 @@ export function buildTextStreamHeader( timestamp: numberToBigInt(info.timestamp), totalLength: numberToBigInt(info.size), attributes: info.attributes, + compression: v2?.compression ?? DataStream_CompressionType.NONE, + inlineContent: v2?.inlineContent, contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({ @@ -36,7 +50,10 @@ export function buildTextStreamHeader( } /** Builds the `DataStream_Header` for a byte stream from its info. */ -export function buildByteStreamHeader(info: ByteStreamInfo): DataStream_Header { +export function buildByteStreamHeader( + info: ByteStreamInfo, + v2?: StreamHeaderV2Fields, +): DataStream_Header { return new DataStream_Header({ streamId: info.id, mimeType: info.mimeType, @@ -44,6 +61,8 @@ export function buildByteStreamHeader(info: ByteStreamInfo): DataStream_Header { timestamp: numberToBigInt(info.timestamp), totalLength: numberToBigInt(info.size), attributes: info.attributes, + compression: v2?.compression ?? DataStream_CompressionType.NONE, + inlineContent: v2?.inlineContent, contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({ diff --git a/src/room/participant/RemoteParticipant.ts b/src/room/participant/RemoteParticipant.ts index 34d6b4b22a..da6181fafd 100644 --- a/src/room/participant/RemoteParticipant.ts +++ b/src/room/participant/RemoteParticipant.ts @@ -83,10 +83,7 @@ export default class RemoteParticipant extends Participant { return new RemoteDataTrack(info, manager, { publisherIdentity: pi.identity }); }), pi.clientProtocol, - // FIXME: ParticipantInfo does not yet carry client capabilities. Until the - // protocol/server propagates `capabilities` onto ParticipantInfo, mock every remote as - // advertising deflate-raw compression support so compression stays enabled. - [ClientInfo_Capability.CAP_COMPRESSION_DEFLATE_RAW], + pi.capabilities, ); } From 5464880cc089fa57950d954bd85e5f89668c45d0 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 23 Jun 2026 16:02:43 -0400 Subject: [PATCH 35/68] feat: commit initial data streams spec --- DATA_STREAMS_SPEC.md | 684 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 684 insertions(+) create mode 100644 DATA_STREAMS_SPEC.md diff --git a/DATA_STREAMS_SPEC.md b/DATA_STREAMS_SPEC.md new file mode 100644 index 0000000000..45a00f22f5 --- /dev/null +++ b/DATA_STREAMS_SPEC.md @@ -0,0 +1,684 @@ +# Data Streams v2 Specification + +## Overview + +Data streams let a participant send a finite or open-ended sequence of bytes to one or more other +participants in a LiveKit room over the reliable data channel, routed by a **topic** to a registered +handler on the receiving side. There are two flavors: + +- **Text streams** — UTF-8 string content (chat, LLM/agent transcriptions, RPC v2 payloads). +- **Byte streams** — arbitrary binary content (files, blobs). + +A stream is always three logical parts on the wire: a **header** (opens the stream, carries +metadata), zero or more **chunks** (the content, split to fit the MTU), and a **trailer** (closes +the stream). All three are `DataPacket`s on the **reliable** channel. + +**Data streams v2** adds three optimizations/safeguards on top of the base stream protocol, +negotiated per-recipient via the participant's advertised `clientProtocol` and client capabilities: + +1. **Single-packet (inline) sends** — small finite payloads are smuggled entirely into the header + packet, skipping the chunk/trailer packets (1 packet instead of 3). Gated on `clientProtocol`. +2. **Compression** — finite, fully-known payloads (`sendText`/`sendFile`) are deflate-raw + compressed; incremental writers (`streamText`/`streamBytes`) are never compressed. Gated on the + `CAP_COMPRESSION_DEFLATE_RAW` capability (separately from `clientProtocol`). +3. **Header size limit** — the header packet must fit the MTU budget, bounding attribute size and + closing a DoS / oversized-packet vector. + +All v2 behavior is invisible to the user-facing API and falls back gracefully when a recipient does +not support it. A v2 sender must interoperate with pre-v2 receivers by sending uncompressed, +multi-packet streams. + +--- + +## Part 1: Client protocol and capabilities + +Two independent signals gate v2 features, and they are deliberately separate: + +- **`clientProtocol`** — a monotonic integer version. Crossing `>= 2` is a *baseline* commitment: + the client guarantees it understands inline single-packet streams. There is no opting out of + baseline v2 behavior. +- **client capabilities** — a set of *optional* feature flags, each advertised independently. They + cover features a client may or may not be able to do depending on platform/runtime, rather than + protocol-level invariants. Compression is one of these: a v2 client might still lack a deflate-raw + codec. + +### `clientProtocol` + +An integer each participant advertises in its `ParticipantInfo` over the signaling channel (the +same field used by RPC v2). Distinct from the signaling `protocol` version. + +| Value | Constant name | Meaning | +|-------|---------------|---------| +| `0` | `CLIENT_PROTOCOL_DEFAULT` | Legacy client. No v2 data-stream features. | +| `1` | `CLIENT_PROTOCOL_DATA_STREAM_RPC` | RPC v2 (see RPC spec). No v2 data-stream features. | +| `2` | `CLIENT_PROTOCOL_DATA_STREAM_V2` | Understands **inline single-packet** data streams. | + +### Client capabilities + +A set of optional feature flags a client advertises in its `ClientInfo.capabilities` (a repeated +enum) during the join handshake. + +| Value | Constant name | Meaning | +|-------|---------------|---------| +| `2` | `CAP_COMPRESSION_DEFLATE_RAW` | The client can **decompress** a deflate-raw compressed stream. | + +(Other capability values exist for unrelated features, e.g. `CAP_PACKET_TRAILER`.) + +### What SDKs need to do + +1. **Advertise `clientProtocol`**: set it to at least `2` (`CLIENT_PROTOCOL_DATA_STREAM_V2`) in the + join handshake. +2. **Advertise capabilities**: include `CAP_COMPRESSION_DEFLATE_RAW` in `ClientInfo.capabilities` + when the runtime can decompress deflate-raw (i.e. it has the codec the receive path needs). +3. **Read both, per remote**: store every remote participant's advertised `clientProtocol` (absent + ⇒ `0`) **and** its advertised capabilities (absent ⇒ empty). Expose a per-participant + capabilities accessor for the send path. +4. **Use**: before sending a finite stream, gate inline on recipients' `clientProtocol` and + compression on recipients' capabilities (see "recipient eligibility"). + +### Recipient eligibility + +Eligibility is evaluated over **every** recipient: the named destination identities for a targeted +send, or every remote participant in the room for a broadcast. An **empty room** (no recipients) is +considered eligible. The two v2 features gate independently: + +- **Inline single-packet** is eligible when every recipient advertises `clientProtocol >= 2`. +- **Compression** is eligible when every recipient advertises `clientProtocol >= 2` **and** + `CAP_COMPRESSION_DEFLATE_RAW`, **and** the local runtime can compress deflate-raw. + +So a v2 recipient that does not advertise the compression capability still receives **inline** +single-packet sends, but its chunked streams are sent **uncompressed**. If any recipient is pre-v2, +the send falls back to an uncompressed, multi-packet stream — which all clients understand. + +--- + +## Part 2: Wire protocol + +All packets are `DataPacket`s sent on the **reliable** data channel. A `DataPacket` carries optional +`destinationIdentities` (empty ⇒ broadcast) and a `value` oneof that, for data streams, is one of +`streamHeader` / `streamChunk` / `streamTrailer`. + +### `DataStream.Header` + +Opens a stream. Fields: + +| Field | Type | Meaning | +|-------|------|---------| +| `streamId` | string | UUID identifying this stream; used to correlate chunks and the trailer. | +| `timestamp` | int64 | Creation time (ms). | +| `topic` | string | Routes the stream to a handler registered for this topic. | +| `mimeType` | string | `text/plain` for text; the file/blob MIME type for bytes. | +| `totalLength` | optional int64 | Total **uncompressed** content byte length for finite streams; absent for unknown-length (incremental) streams. | +| `encryptionType` | enum | `NONE` or `GCM` (see E2EE below). | +| `attributes` | map | Caller-supplied metadata only (v2 carries its own signals in dedicated fields, not here). | +| `inlineContent` | optional bytes | The full payload smuggled into the header for single-packet (inline) sends; absent for chunked streams. Deflate-raw compressed iff `compression` is `DEFLATE_RAW`. | +| `compression` | enum (`DataStream.CompressionType`) | `NONE` or `DEFLATE_RAW`; applies to the inline or chunked payload. | +| `contentHeader` | oneof | `textHeader` (`DataStream.TextHeader`) or `byteHeader` (`DataStream.ByteHeader`). | + +`DataStream.TextHeader`: `operationType` (`CREATE`/`UPDATE`/`DELETE`/`REACTION`), `version` (int, +for supersede-by-version updates), `replyToStreamId`, `attachedStreamIds` (stream IDs of attached +byte streams, e.g. file attachments to a text message), `generated` (bool, marks +machine/agent-generated text such as transcriptions). + +`DataStream.ByteHeader`: `name` (file/blob name). + +### `DataStream.Chunk` + +One slice of content. Fields: + +| Field | Type | Meaning | +|-------|------|---------| +| `streamId` | string | The stream this chunk belongs to. | +| `chunkIndex` | int64 | 0-based, **contiguous** index for ordering and dedup. | +| `content` | bytes | The chunk payload (for text, a UTF-8 slice; for compressed streams, compressed bytes). | +| `version` | int | For text stream updates: supersedes a previously-received chunk at the same `chunkIndex` when higher. Otherwise `0`. | +| `iv` | optional bytes (deprecated) | Initialization vector when the chunk is E2EE-encrypted. | + +### `DataStream.Trailer` + +Closes a stream. Fields: `streamId`, `reason` (string; empty on normal close), `attributes` +(map; merged into the stream's attributes on the receiver at close — lets a sender +append metadata known only after the content, e.g. a final checksum). A conforming v2 sender sends a +trailer carrying only `streamId` on normal close; `reason`/`attributes` are optional extensions. + +### Stream lifecycle (multi-packet) + +``` +Sender Receiver + | | + |--- streamHeader (topic, attrs) ----->| (looks up handler by topic, creates a reader) + |--- streamChunk (index 0) ----------->| (delivers content as it arrives) + |--- streamChunk (index 1) ----------->| + | ... | + |--- streamTrailer -------------------->| (merges trailer attrs, closes the reader) + | | +``` + +- The receiver routes on `topic`: if no handler is registered for that topic, the stream is ignored + (chunks/trailer for an unhandled header are dropped). +- Content is delivered incrementally as chunks arrive — a receiver must not wait for the trailer to + begin yielding content. +- Chunk content larger than the MTU budget is split across multiple chunks with contiguous indices. + +### Topics and handlers + +The receiving SDK exposes registration of one handler per topic, separately for text and byte +streams. A handler is invoked once per incoming stream (when its header arrives) with a **reader** +object and the sending participant's identity. Handlers must be registered **before** connecting / +before the stream arrives, or the stream is dropped. + +### Readers + +A reader exposes the stream's `info` (id, topic, mimeType, size, attributes, name for bytes, +encryptionType, etc.) and lets the consumer either: + +- **read incrementally** — iterate chunks as they arrive (text yields decoded string pieces; bytes + yield byte arrays), or +- **read to completion** — await the full concatenated content (string for text, list/array of byte + arrays for bytes). + +The reader counts received content bytes against `totalLength` (when present) and surfaces an error +if the stream ends short, or if more bytes than declared arrive. + +### E2EE + +When data-channel encryption is enabled, the header's `encryptionType` is `GCM` and each chunk +carries an `iv`. The receiver MUST enforce a consistent `encryptionType` across a stream's +header and chunks; a mismatch errors the stream. `totalLength` and `attributes` semantics are +unchanged by encryption. + +--- + +## Part 3: Send APIs + +Four send operations, two finite (full payload known up front) and two incremental: + +| API | Content | Payload known up front? | Eligible for inline? | Eligible for compression? | +|-----|---------|-------------------------|----------------------|---------------------------| +| `sendText(text, opts)` | text | yes | yes | yes | +| `sendFile(file, opts)` | bytes | yes (streamed from disk) | **no** | yes | +| `streamText(opts) -> writer` | text | no (incremental writes) | no | **no** | +| `streamBytes(opts) -> writer` | bytes | no (incremental writes) | no | **no** | + +Common options: `topic`, `destinationIdentities` (omit ⇒ broadcast), `attributes` +(map), and for the finite APIs a `compress` boolean (default `true`, opt-out). +`sendText` additionally supports `attachments` (each becomes an attached byte stream referenced by +`attachedStreamIds` in the text header). `streamText` additionally supports `type` +(`create`/`update`) and `version` for streaming edits/updates of a prior stream. + +The v2 signals are carried in dedicated header fields, **not** attributes: `inlineContent` (the +single-packet payload, as raw bytes) and `compression` (`NONE` / `DEFLATE_RAW`). `attributes` carries +only caller-supplied metadata. (Earlier drafts smuggled these into reserved `lk.inline_payload` / +`lk.compression` attributes; that is gone — there is no attribute fallback.) + +### `sendText` send algorithm + +1. Compute the UTF-8 byte length as `totalLength`. +2. **Inline attempt** (only when there are no attachments and all recipients are v2): build a header + carrying the UTF-8 payload bytes in `inlineContent` with `compression = NONE`. If `compress` and + the runtime supports compression, deflate-raw the payload and, **only if the compressed form is + smaller**, put the compressed bytes in `inlineContent` and set `compression = DEFLATE_RAW`. If the + resulting serialized header packet is `<= STREAM_CHUNK_SIZE_BYTES`, send it as a single packet and + finish. Otherwise fall through. +3. **Chunked compressed** (when the send is compression-eligible — see Part 1 § Recipient + eligibility: `compress` set, runtime can compress, and every recipient is v2 **and** advertises + `CAP_COMPRESSION_DEFLATE_RAW`): send a header with `compression = DEFLATE_RAW` and `totalLength` + = uncompressed length, then the compressed content as chunks, then the trailer. +4. **Chunked uncompressed** (fallback): send the payload as a normal multi-packet text stream + (UTF-8-boundary-split chunks), header has `compression = NONE`. +5. Send each attachment as its own byte stream (`sendFile` semantics), referenced by + `attachedStreamIds` in the text header. + +### `sendFile` send algorithm + +`sendFile` is fully streamed from the file's byte stream and is **never** sent inline (file uploads +are an edge case; the inline single-packet optimization is intentionally dropped for them): + +1. Compress iff the send is compression-eligible (Part 1 § Recipient eligibility): the `compress` + option is set (default true), the runtime can compress, and every recipient is v2 **and** + advertises `CAP_COMPRESSION_DEFLATE_RAW`. +2. Send a byte-stream header with `totalLength` = file size and `compression = DEFLATE_RAW` iff + compressing. +3. Stream the file's bytes → (deflate-raw if compressing) → chunk packets, then the trailer. The + whole file is never buffered in memory at once. + +Even a tiny file is sent as header + chunk(s) + trailer — there is no inline single-packet fast +path for `sendFile`, because deciding inline-eligibility would require buffering and compressing the +whole file up front. An empty file still produces a well-formed stream (`totalLength` 0 + trailer). + +### `streamText` / `streamBytes` (incremental) + +Open a header immediately (unknown `totalLength`), then the caller writes content over time; each +write is split into chunks and sent, and `close()` sends the trailer. **Incremental writers are +never compressed**: the platform stream compressor cannot flush mid-stream, and per-write flushing +costs more than it saves at typical write sizes (validated against agent-transcription workloads, +where it *expanded* the wire data). Text writes are split on UTF-8 character boundaries so each chunk +decodes independently. + +--- + +## Part 4: Single-packet (inline) optimization + +For small finite text payloads, the entire content is smuggled into the header's `inlineContent` +field (raw bytes) and sent as **one** packet (no chunks, no trailer). The decision is made by +**serializing the candidate header packet and checking its byte length against +`STREAM_CHUNK_SIZE_BYTES`** — if it fits, send inline; if not, fall back to the chunked path. This +naturally accounts for attributes, topic, framing, and (when used) the compressed payload all +together. + +- Inline applies to `sendText` only (not `sendFile`, not incremental writers), and only when all + recipients are v2 and there are no attachments. +- The receiver detects an inline stream by the presence of `inlineContent` on the header. It + synthesizes an already-complete stream from those bytes (decompressing first if `compression` is + `DEFLATE_RAW`) and never waits for chunk/trailer packets. + +--- + +## Part 5: Compression + +Compression is **deflate-raw** (raw DEFLATE, no zlib/gzip wrapper). It is applied only by the finite +send APIs (`sendText`/`sendFile`), where the full payload is known up front. Two forms: + +### Inline payload compression (single packet) + +One-shot deflate-raw of the full payload, written as raw bytes into `inlineContent`, flagged +`compression = DEFLATE_RAW`. **Kept only if it actually shrinks** the payload (deflate framing can +make tiny payloads larger). For uncompressed inline (text or byte), `inlineContent` holds the raw +payload bytes with `compression = NONE`; because `inlineContent` is a binary field there is no base64 +round-trip. + +### Chunked stream compression (multi packet) + +The full payload is compressed as a **single deflate-raw stream whose bytes are spread across the +chunk contents in `chunkIndex` order**, terminated by the DEFLATE final block before the trailer. +The header is flagged `compression = DEFLATE_RAW` and carries `totalLength` = the **pre-compression** +(decompressed) byte length. Chunk packets carry **no** compression metadata. + +### Receiver decompression + +The receiver detects `compression = DEFLATE_RAW` on the header and feeds all chunk contents (in +`chunkIndex` order) through **one** deflate-raw decompressor for the whole stream, emitting +decompressed content as it is produced. Because the decompressor is stateful and order-sensitive: + +- **Duplicate** chunk indices (index ≤ last processed) are dropped with a warning (reliable delivery + is expected, but reconnect logic may replay). +- A **gap** in chunk indices is a hard error (the stream cannot continue decompressing). + +The receiver counts **decompressed** bytes against the header's `totalLength`. For text, decompressed +bytes are re-framed on UTF-8 character boundaries so each delivered piece decodes independently. + +If the receiver has no deflate-raw decompressor (no platform `DecompressionStream`), an incoming +compressed stream is **ignored** — the topic handler is never invoked. A conforming receiver only +advertises `CAP_COMPRESSION_DEFLATE_RAW` when it can decompress, so a conforming sender never sends +it a compressed stream; the drop is a defensive backstop against a non-conforming peer. + +### Forward-compatibility note (context takeover) + +The receive path is deliberately more general than the current send path: a deflate-raw stream that +is sync-flushed at write boundaries (permessage-deflate "context takeover") also decodes +incrementally through the same single-decompressor path. This means a future incremental sender +(compressed `streamText`/`streamBytes`) could be introduced **without a `clientProtocol` bump** — +existing v2 receivers already decode that wire format. + +### Eligibility recap + +Compression is used iff `compress` is requested (default true) AND the local runtime provides a +deflate-raw compressor AND every recipient advertises `clientProtocol >= 2` AND every recipient +advertises the `CAP_COMPRESSION_DEFLATE_RAW` capability. Otherwise the stream is sent uncompressed. +A pre-v2 recipient — or a v2 recipient that does not advertise the capability — therefore always +receives uncompressed, multi-packet streams (it still receives inline single-packet sends, which +are gated on `clientProtocol` alone). + +--- + +## Part 6: Header size limit (MTU) + +A `DataStream.Header` is a single `DataPacket`; one larger than the MTU cannot be reliably sent, and +unbounded `attributes` are both a correctness hazard and a DoS vector. Therefore: + +**When sending any stream header on the chunked path, the SDK serializes the header packet and, if +its byte length exceeds `STREAM_CHUNK_SIZE_BYTES`, throws an error (`HeaderTooLarge`) instead of +emitting the packet.** This bounds attributes + topic + framing together against the MTU. + +- This is a **breaking change**: previously oversized attributes were accepted; they now error. +- The **inline** path keeps its existing graceful behavior — if the inline header exceeds the + budget it falls back to the chunked path (no throw); the chunked header send is what enforces the + hard limit. So a large *payload* with small attributes falls back and sends fine, while large + *attributes* (whose chunked header still exceeds the MTU) throw. +- Enforcement is **send-side only**. Receivers are not required to reject oversized incoming headers + (interop with other/older SDKs). + +### Constants + +| Constant | Value | Meaning | +|----------|-------|---------| +| `STREAM_CHUNK_SIZE_BYTES` | `15000` | Max chunk content size AND the header-packet MTU budget. Kept below the ~16 KB data-channel MTU for protocol/E2EE framing headroom. | + +--- + +## Part 7: Receive-side semantics + +1. **Header** → look up the topic handler (text or byte). If none, ignore the stream. Otherwise + build the stream `info` (stripping reserved attributes), detect inline / compression, create the + reader, register it by `streamId`, and invoke the handler with the reader + sender identity. + Reject a duplicate `streamId` whose stream is already open. +2. **Chunk** → route by `streamId` to the open stream; enforce consistent `encryptionType`; deliver + content (through the decompressor for compressed streams). Empty chunks are ignored. +3. **Trailer** → merge `trailer.attributes` into the stream `info`, then close the reader. Drop the + stream's registration. +4. **Length validation** → the reader compares received content bytes against the header's + `totalLength` (when present): short ⇒ "incomplete" error at close; over ⇒ "length exceeded" + error. +5. **Abnormal end** → if a sending participant disconnects while it has streams in flight to this + receiver, those open readers are errored ("abnormal end"). +6. **Text updates** → a later chunk at an existing `chunkIndex` with a higher `version` supersedes + the earlier one (used with `TextHeader.operationType = UPDATE`). +7. **Connection gating** → packets received before the receiver is marked connected are buffered and + replayed in order once it connects (streams can begin arriving during the join handshake). +8. **Unsupported compression** → a compressed stream is ignored (its handler is never invoked) when + the receiver has no deflate-raw decompressor (see Part 5). + +--- + +## Recommended naming + +In the reference implementation: + +- The entity that builds and sends streams is `OutgoingDataStreamManager`. +- The entity that receives, routes, and exposes streams to handlers is `IncomingDataStreamManager`. +- Client-protocol constants: `CLIENT_PROTOCOL_DEFAULT` (0), `CLIENT_PROTOCOL_DATA_STREAM_RPC` (1), + `CLIENT_PROTOCOL_DATA_STREAM_V2` (2). +- Capability constant: `CAP_COMPRESSION_DEFLATE_RAW` (2), stored on each remote participant and + exposed via a `getRemoteParticipantCapabilities(identity)` accessor the send path consults. +- V2 header fields: `inlineContent` (bytes), `compression` (`DataStream.CompressionType`: `NONE` / + `DEFLATE_RAW`). +- Chunk/header budget constant: `STREAM_CHUNK_SIZE_BYTES` (15000). + +Use these names unless prior SDK architecture makes it burdensome; if you diverge, explain the +rationale and confirm with the user before continuing. The header fields (`inlineContent`, +`compression`), topics, protobuf field names, `clientProtocol` values, the +`CAP_COMPRESSION_DEFLATE_RAW` capability value, and the `STREAM_CHUNK_SIZE_BYTES` budget are **wire +contract** and must match exactly for cross-SDK interop. + +--- + +## Minimum required test cases + +The two managers are independently testable and a conforming implementation must pass both sets: + +- The **`OutgoingDataStreamManager`** is exercised by calling the send APIs against a captured-packet + engine and a configurable set of remote participants (each with a `clientProtocol` and a capability + set), then asserting on the emitted packets (which case, header attributes, chunk contents/indices, + trailer). Recipient scenarios: a room of all pre-v2 participants, a room of all v2 participants + (one of which advertises **no** compression capability), and a mixed room. +- The **`IncomingDataStreamManager`** is exercised by registering a topic handler, marking the + manager connected, feeding hand-crafted packets, and asserting on what the reader yields or the + error it raises. + +### `OutgoingDataStreamManager` (send side) + +**Test harness.** Construct the manager with an engine that pushes every `sendDataPacket(packet)` +into a `sentPackets` array, plus a configurable map of remote participants → `(clientProtocol, +capabilities)`. Call a send API, then assert on `sentPackets`: each packet's `value.case` +(`streamHeader` / `streamChunk` / `streamTrailer`), the header's `contentHeader.case` +(`textHeader` / `byteHeader`) and `attributes`, each chunk's `chunkIndex` and `content`, and the +trailer's `streamId`. Three participant rooms are used: + +- **all pre-v2** — `alice`, `bob` at `clientProtocol 0`, `jim` at `1` (RPC). +- **all v2** — `alice`, `bob` at `clientProtocol 2` with `CAP_COMPRESSION_DEFLATE_RAW`; `noCompression` + at `clientProtocol 2` with **no** capabilities. +- **mixed** — `alice` (0), `bob`/`jim` (2 + cap), `mallory` (1), `noCompression` (2, no cap). + +#### Sending to a room where every recipient is pre-v2 + +1. **Short text → legacy multi-packet, uncompressed** + - Call `sendText('hello world', { topic })` (broadcast; all recipients pre-v2). + - Expect exactly **3** packets. + - Packet 0 is a `streamHeader` with `contentHeader.case === 'textHeader'`, `streamId === info.id`, + and the given `topic`. + - Packet 1 is a `streamChunk` with `chunkIndex 0` and `content` equal to the raw UTF-8 of + `'hello world'`. + - Packet 2 is a `streamTrailer` with the matching `streamId` and empty `reason`. + - The header's `compression` is `NONE` and `inlineContent` is absent. + +2. **Short bytes → legacy multi-packet, uncompressed** + - Open `streamBytes({ topic })`, `write([0x00,0x01,0x02,0x03])`, `close()`. + - Expect **3** packets: a `byteHeader`, one `streamChunk` (`chunkIndex 0`, `content` equal to the + four raw bytes), and a trailer. + +3. **Long text → uncompressed multi-packet** + - Call `sendText('A'.repeat(40_000), { topic })`. + - Expect **5** packets: header + 3 chunks + trailer. Chunk indices are `0,1,2` (contiguous), each + `content` is all `'A'`, and chunks are split at `STREAM_CHUNK_SIZE_BYTES` (15000, 15000, 10000). + - The header's `compression` is `NONE`. + +4. **Long bytes → uncompressed multi-packet** + - Open `streamBytes({ topic })`, write a 20 000-byte buffer of `0x01` twice, `close()`. + - Expect **6** packets: header + 4 chunks + trailer. Each write produces a 15 000-byte chunk then + a 5 000-byte chunk; chunk indices are `0,1,2,3` (contiguous across writes); content is all `0x01`. + +5. **File → uncompressed multi-packet** + - Call `sendFile(new File([Uint8Array(20_000).fill(0x07)], 'text.txt'), { topic })`. + - Expect **4** packets: a `byteHeader` with `compression = NONE`, a 15 000-byte chunk + (`chunkIndex 0`), a 5 000-byte chunk (`chunkIndex 1`), and a trailer; chunk content is raw `0x07`. + +#### Sending to a room where every recipient is v2 + +6. **Short compressible text → single inline packet, compressed** + - Call `sendText('hello hello compressible world', { topic, destinationIdentities: ['alice','bob'] })`. + - Expect exactly **1** packet: a `streamHeader` (`textHeader`). + - `compression === DEFLATE_RAW`; `inlineContent` is a `Uint8Array` and is **not** the raw UTF-8 + of the text (it is the compressed bytes). + - No `streamChunk` or `streamTrailer` packets. + +7. **Short incompressible text → single inline packet, raw** + - Call `sendText('short', { ..., destinationIdentities: ['alice','bob'] })`. + - Expect **1** packet. `compression` is `NONE` and `inlineContent` equals the raw UTF-8 of + `'short'` — deflate didn't shrink it, so the raw bytes are kept (compression is only applied + when it actually reduces size). + +8. **Short text to a recipient lacking the compression capability → single inline packet, raw** + - Call `sendText('hello hello compressible world', { ..., destinationIdentities: ['noCompression'] })`. + - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the raw UTF-8 of the text. + Inline still happens (gated on `clientProtocol`); compression does not (gated on the capability). + +9. **Large highly-compressible text → single inline packet, compressed** + - Call `sendText('hello world'.repeat(20_000), { ..., destinationIdentities: ['alice','bob'] })`. + - Expect **1** packet. `compression === DEFLATE_RAW`; `inlineContent` is compressed bytes (does + **not** start with the UTF-8 of `'hello world'`). It compresses well under the MTU, so it still + goes inline. + +10. **Large somewhat-compressible text → compressed multi-packet** + - Build a ~50 KB payload of 50 × (`'hello world'` + 1 000 random chars) and `sendText` it to + `['alice','bob']`. + - Expect **5** packets: header (`compression = DEFLATE_RAW`) + **3** chunks + trailer — fewer + than the `ceil(50_000 / 15_000) = 4` chunks an uncompressed send would need. The first chunk's + `content` length is 15 000 (MTU). + +11. **Large incompressible file → compressed multi-packet** + - Call `sendFile(new File([50_000 random bytes], 'text.txt'), { ..., destinationIdentities: ['alice','bob'] })`. + - Expect **6** packets: a `byteHeader` (`compression = DEFLATE_RAW`) + **4** chunks (first is + 15 000 bytes) + trailer. + - The summed chunk content length is **greater** than 50 000 — deflate adds slight overhead on + incompressible data, the accepted trade-off for streaming the file instead of buffering it. + +12. **`compress: false`, short payload → single inline packet, raw** + - Call `sendText('hello hello compressible world', { ..., destinationIdentities: ['alice','bob'], compress: false })`. + - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the raw UTF-8 of the text + (inline still applies; the opt-out only disables compression). + +13. **`compress: false`, large payload → uncompressed multi-packet** + - `sendText` the same ~50 KB somewhat-compressible payload to `['alice','bob']` with `compress: false`. + - Expect **6** packets: header (`compression = NONE`) + **4** chunks (first is 15 000 bytes) + + trailer (uncompressed → `ceil(50_000 / 15_000) = 4` chunks). + +14. **`streamText` never compresses or inlines** + - Open `streamText({ topic, destinationIdentities: ['noCompression'] })`; after open, expect **1** + packet — a `textHeader` with `compression = NONE`. + - `write('hello world')` → expect **2** packets total; the new `streamChunk` content equals the + raw UTF-8 of `'hello world'`. + - `close()` → expect **3** packets total; the last is a `streamTrailer`. + +15. **`streamBytes` never compresses or inlines** + - Open `streamBytes({ topic, destinationIdentities: ['noCompression'] })`; expect **1** packet — a + `byteHeader` with `compression = NONE`. + - `write([0x00,0x01,0x02,0x03])` → expect **2** packets; the chunk content equals the raw bytes. + - `close()` → expect **3** packets; the last is a trailer. + +16. **`sendFile` never sends a single inline packet** + - Call `sendFile(new File([Uint8Array(10_000).fill(0x01)], 'text.txt'), { ..., destinationIdentities: ['alice','bob'] })` + (highly compressible). + - Expect **3** packets (header + 1 chunk + trailer), **not** 1. The header is a `byteHeader` with + `compression = DEFLATE_RAW`; the chunk `content` length is **less** than 10 000 (compressed). + `sendFile` never uses the inline path regardless of how small/compressible the file is. + +17. **File to a recipient lacking the compression capability → uncompressed multi-packet** + - Call `sendFile(new File([Uint8Array(10_000).fill(0x07)], 'text.txt'), { ..., destinationIdentities: ['noCompression'] })`. + - Expect **3** packets: a `byteHeader` with `compression = NONE` and **no** `inlineContent`, + one 10 000-byte chunk (`chunkIndex 0`, all `0x07`, uncompressed, under the MTU), and a trailer. + +18. **Empty file** + - Call `sendFile(new File([], 'empty.bin'), { ..., destinationIdentities: ['alice','bob'] })`. + - Packet 0 is a `byteHeader` with `totalLength === 0` and `compression = DEFLATE_RAW`. + - The last packet is a `streamTrailer` with the matching `streamId`. (A well-formed stream is + still produced — the deflate stream's final block plus the trailer.) + +#### Sending to a mixed room (some pre-v2, some v2) + +19. **Broadcast falls back to legacy** + - Call `sendText('hello world', { topic })` (broadcast) in the mixed room (contains pre-v2 + participants). + - Expect **3** packets: `textHeader` + chunk (`content` = raw `'hello world'`) + trailer. No + `inlineContent`; `compression = NONE`. + +20. **Targeted send to an all-v2, all-capable subset → single inline packet, compressed** + - Call `sendText('hello hello compressible world', { ..., destinationIdentities: ['bob','jim'] })` + (both v2 + capability). + - Expect **1** packet with `compression = DEFLATE_RAW` and `inlineContent` being compressed bytes + (not the raw text). Restricting the send to capable recipients re-enables inline + compression. + +21. **Targeted send to a subset where one lacks the capability → inline, uncompressed** + - Call `sendText('hello hello compressible world', { ..., destinationIdentities: ['bob','jim','noCompression'] })`. + - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the raw UTF-8 of the text — + inline still happens (all three are v2) but compression is gated off by `noCompression`. + +### `IncomingDataStreamManager` (receive side) + +**Test harness.** Construct the manager, register a text or byte stream handler for a topic that +resolves a promise with the delivered `reader`, mark the manager connected (`setConnected(true)`, +except where buffering is under test), then feed hand-crafted packets via +`handleDataStreamPacket(packet, encryptionType)`. Assertions are on the reader: `await +reader.readAll()` (the full string for text, the array of byte chunks for bytes), `reader.info` +(attributes, `attachedStreamIds`), or the error `readAll()` rejects with. The sending participant +identity is `'alice'` throughout. + +#### Receiving v1 (legacy multi-packet) streams + +1. **Text stream round-trips** + - Feed a `textHeader` (`totalLength` = the byte length, `attributes: { foo: 'bar' }`), a chunk + (`chunkIndex 0`, raw UTF-8), and a trailer (matching `streamId`). + - `await reader.readAll()` equals the text; `reader.info.attributes.foo === 'bar'`. + +2. **Byte stream round-trips** + - Feed a `byteHeader` (`totalLength 4`, `attributes: { foo: 'bar' }`), a chunk + (`content [0x01,0x02,0x03,0x04]`), and a trailer. + - `await reader.readAll()` equals `[Uint8Array([1,2,3,4])]`; `reader.info.attributes.foo === 'bar'`. + +3. **Text stream with attachments** + - Register both a text and a byte handler for the topic. + - Feed a `textHeader` whose `contentHeader.attachedStreamIds` references an attachment stream id + (with an inline payload for the text body), then a separate byte stream for that attachment id + (header + chunk + trailer). + - Both handlers fire: the text reader yields the body, the byte reader yields the attachment + bytes, and `textReader.info.attachedStreamIds` has length 1. + +4. **Buffers packets received while disconnected** + - `setConnected(false)`. Feed header + chunk + trailer. + - Assert the handler has **not** fired yet (the reader promise is still pending). + - `setConnected(true)`. The handler now fires and `await reader.readAll()` equals the text. + +5. **Merges trailer attributes** + - Feed a header with `attributes: { foo: 'bar', baz: 'quux' }`, a chunk, and a trailer with + `attributes: { hello: 'world', foo: 'updated' }`. + - After close, `reader.info.attributes` has `baz === 'quux'` (header), `hello === 'world'` + (trailer), and `foo === 'updated'` (trailer overrides header). + +6. **Drops packets with a mismatched `encryptionType`** + - Feed the header with `Encryption_Type.NONE`, then feed a chunk with `Encryption_Type.GCM`. + - `await reader.readAll()` rejects with an `EncryptionTypeMismatch` error ("Encryption type + mismatch"). + +7. **Errors when too few bytes arrive** + - Feed a header declaring `totalLength 5`, a single 1-byte chunk, and a trailer. + - `await reader.readAll()` rejects with "Not enough chunk(s)" (raised when the stream closes). + +8. **Errors when too many bytes arrive** + - Feed a header declaring `totalLength 3`, then a 5-byte chunk (and a trailer). + - `await reader.readAll()` rejects with "Extra chunk(s)" (raised as the over-budget chunk is + processed, before the trailer matters). + +9. **Errors on sender disconnect mid-stream** + - Feed a header declaring `totalLength 10` and a 5-byte chunk (no trailer). + - Call `manager.validateParticipantHasNoActiveDataStreams('alice')` (the room calls this on + disconnect). + - `await reader.readAll()` rejects with "Participant alice unexpectedly disconnected in the middle + of sending data". + +#### Receiving v2 streams + +10. **Inline uncompressed text** + - Feed a single `textHeader` with `inlineContent` = the raw UTF-8 of the text, `attributes: + { foo: 'bar' }`, and **no** chunk or trailer packets. + - `await reader.readAll()` equals the text; `reader.info.attributes.foo === 'bar'` (the v2 signals + were never in `attributes`, so nothing is stripped). + +11. **Inline uncompressed byte** + - Feed a single `byteHeader` (`totalLength 3`) with `inlineContent` = `[0x01,0x02,0x03]` and + `compression = NONE`. + - `await reader.readAll()` equals `[Uint8Array([1,2,3])]`. + +12. **Inline compressed text** + - Feed a single `textHeader` (`totalLength` = the *uncompressed* byte length) with `inlineContent` + = `deflateRaw(text)`, `compression = DEFLATE_RAW`, `attributes: { foo: 'bar' }`. + - `await reader.readAll()` equals the decompressed text; `reader.info.attributes.foo === 'bar'`. + +13. **Inline compressed byte** + - Feed a single `byteHeader` (`totalLength` = the uncompressed length) with `inlineContent` = + `deflateRaw(bytes)`, `compression = DEFLATE_RAW`. + - `await reader.readAll()` equals the original bytes (decompressed). + +14. **Multi-packet compressed text** + - Compress a ~30 KB somewhat-compressible text with one deflate-raw pass (verify the compressed + output is under `2 × STREAM_CHUNK_SIZE_BYTES`). + - Feed a `textHeader` (`totalLength` = the uncompressed length, `compression = DEFLATE_RAW`), + then split the compressed bytes into two chunks at `STREAM_CHUNK_SIZE_BYTES` (`chunkIndex 0` and + `1`), then a trailer. + - `await reader.readAll()` equals the original text. + +15. **Ignores a compressed stream when no decompressor is available** + - IMPORTANT NOTE: this test is only relevant if the client relies on platform support for compression / + decompression. If the client always supports encryption (ie, maybe via a library which is + always available) this test can be skipped. + - Temporarily make the platform `CompressionStream`/`DecompressionStream` unavailable. + - Feed a compressed (`compression = DEFLATE_RAW`) **text** stream (header + chunk + trailer); + assert the handler is **never** invoked (the reader promise stays pending) — the stream is + dropped. + - Repeat with a compressed **byte** stream; same result. Restore the globals afterward. + +--- + +## Benchmarking + +Implementing a benchmark is optional but useful for validating correctness and performance under +realistic conditions. Two reference shapes from `client-sdk-js`: + +- **Throughput grid** — connect a sender and receiver to one room; for a grid of payload sizes × + simulated network conditions, send finite streams for a fixed window and measure + received-stream throughput, one-way latency percentiles (p50/p95/p99), and integrity + (checksum) mismatches. +- **Agent-transcription scenario** — model a voice-agent transcript: stream short word-sized writes + at a realistic cadence (median ~350 ms between writes, occasional bursts) over a long-lived + `streamText` stream, optionally alongside competing "junk" reliable data packets on the same + channel, and measure **per-chunk staleness** (receiver-arrival minus sender-write time) + percentiles plus chunks delivered vs sent. This workload is what established that incremental + compression is not worthwhile at word granularity. From a7780dec586f35a4145fc6c075a012bddf08110c Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 23 Jun 2026 17:01:11 -0400 Subject: [PATCH 36/68] fix: remove unused tests --- .../data-stream/compression-roundtrip.test.ts | 371 ------------------ src/room/data-stream/compression.test.ts | 111 ------ 2 files changed, 482 deletions(-) delete mode 100644 src/room/data-stream/compression-roundtrip.test.ts delete mode 100644 src/room/data-stream/compression.test.ts diff --git a/src/room/data-stream/compression-roundtrip.test.ts b/src/room/data-stream/compression-roundtrip.test.ts deleted file mode 100644 index 65615e20d2..0000000000 --- a/src/room/data-stream/compression-roundtrip.test.ts +++ /dev/null @@ -1,371 +0,0 @@ -import { type DataPacket, type DataStream_Chunk, Encryption_Type } from '@livekit/protocol'; -import { describe, expect, it, vi } from 'vitest'; -import log from '../../logger'; -import { CLIENT_PROTOCOL_DATA_STREAM_RPC, CLIENT_PROTOCOL_DATA_STREAM_V2 } from '../../version'; -import type RTCEngine from '../RTCEngine'; -import IncomingDataStreamManager from './incoming/IncomingDataStreamManager'; -import type { ByteStreamReader, TextStreamReader } from './incoming/StreamReader'; -import OutgoingDataStreamManager from './outgoing/OutgoingDataStreamManager'; - -const RECEIVER = 'bob'; -const hasCompression = typeof CompressionStream !== 'undefined'; - -/** High-entropy text: too big to inline even after compression, forcing the chunked fallback. */ -function randomText(length: number): string { - let s = ''; - while (s.length < length) { - s += Math.random().toString(36).slice(2); - } - return s.slice(0, length); -} - -/** High-entropy multibyte text (random CJK), so compressed output chunk boundaries fall - * mid-character and the payload stays over the inline budget even compressed. */ -function randomMultibyteText(chars: number): string { - let s = ''; - for (let i = 0; i < chars; i += 1) { - s += String.fromCharCode(0x4e00 + Math.floor(Math.random() * 0x51a5)); - } - return s; -} - -/** Total streamChunk content bytes across the captured packets. */ -function chunkContentBytes(packets: DataPacket[]): number { - return packets - .filter((p) => p.value.case === 'streamChunk') - .reduce((sum, p) => sum + (p.value.value as DataStream_Chunk).content.byteLength, 0); -} - -/** An OutgoingDataStreamManager whose engine captures every sent packet. */ -function createSender(recipientProtocol = CLIENT_PROTOCOL_DATA_STREAM_V2) { - const sentPackets: DataPacket[] = []; - const engine = { - sendDataPacket: vi.fn(async (packet: DataPacket) => { - sentPackets.push(packet); - }), - e2eeManager: undefined, - once: vi.fn(), - off: vi.fn(), - } as unknown as RTCEngine; - const manager = new OutgoingDataStreamManager( - engine, - log, - () => recipientProtocol, - () => [RECEIVER], - ); - return { manager, sentPackets }; -} - -/** Replays captured outgoing packets into a receiver and returns the resulting text. */ -async function receiveText(packets: DataPacket[], topic: string): Promise { - const incoming = new IncomingDataStreamManager(); - incoming.setConnected(true); - const readerPromise = new Promise((resolve) => { - incoming.registerTextStreamHandler(topic, (reader) => resolve(reader)); - }); - for (const packet of packets) { - incoming.handleDataStreamPacket(packet, Encryption_Type.NONE); - } - return readerPromise; -} - -async function receiveBytes(packets: DataPacket[], topic: string): Promise { - const incoming = new IncomingDataStreamManager(); - incoming.setConnected(true); - const readerPromise = new Promise((resolve) => { - incoming.registerByteStreamHandler(topic, (reader) => resolve(reader)); - }); - for (const packet of packets) { - incoming.handleDataStreamPacket(packet, Encryption_Type.NONE); - } - return readerPromise; -} - -function flatten(chunks: Array): Uint8Array { - const total = chunks.reduce((n, c) => n + c.byteLength, 0); - const out = new Uint8Array(total); - let offset = 0; - for (const c of chunks) { - out.set(c, offset); - offset += c.byteLength; - } - return out; -} - -describe.skipIf(!hasCompression)('data stream compression round-trip', () => { - it('round-trips a small compressible inline text payload', async () => { - const { manager, sentPackets } = createSender(); - const payload = 'compress me '.repeat(50); // > raw, compresses well, fits inline - - await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); - - expect(sentPackets).toHaveLength(1); // single inline header packet - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe(payload); - }); - - it('round-trips a large chunked compressed text payload (multi-packet)', async () => { - const { manager, sentPackets } = createSender(); - // High entropy → too big to inline even compressed → chunked + compressed. - const payload = randomText(60_000); - - await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); - - expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe(payload); - }); - - it('round-trips chunked compressed text with multibyte UTF-8 (reframing on char boundaries)', async () => { - const { manager, sentPackets } = createSender(); - // High-entropy CJK so compressed output chunk boundaries fall mid-character. - const payload = randomMultibyteText(30_000); - - await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); - - expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe(payload); - }); - - it('round-trips a chunked compressed byte stream', async () => { - const { manager, sentPackets } = createSender(); - const payload = new Uint8Array(50_000); - for (let i = 0; i < payload.length; i += 1) { - payload[i] = i % 256; - } - - const writer = await manager.streamBytes({ topic: 'b', destinationIdentities: [RECEIVER] }); - await writer.write(payload); - await writer.close(); - - expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); - const reader = await receiveBytes(sentPackets, 'b'); - expect(Array.from(flatten(await reader.readAll()))).toEqual(Array.from(payload)); - }); - - it('round-trips text written across multiple writes (uncompressed streamText)', async () => { - const { manager, sentPackets } = createSender(); - const parts = ['first part ', '日本語🚀 second ', 'x'.repeat(20_000), ' tail']; - - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - for (const part of parts) { - await writer.write(part); - } - await writer.close(); - - expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe(parts.join('')); - }); - - it('round-trips bytes written across multiple writes (multi-member gzip)', async () => { - const { manager, sentPackets } = createSender(); - const parts = [ - new Uint8Array([1, 2, 3]), - new Uint8Array(20_000).fill(7), - new Uint8Array([9, 8, 7, 6]), - ]; - const expected = parts.flatMap((p) => Array.from(p)); - - const writer = await manager.streamBytes({ topic: 'b', destinationIdentities: [RECEIVER] }); - for (const part of parts) { - await writer.write(part); - } - await writer.close(); - - const reader = await receiveBytes(sentPackets, 'b'); - expect(Array.from(flatten(await reader.readAll()))).toEqual(expected); - }); - - it('marks the chunked compressed sendText fallback with the deflate-raw attribute', async () => { - const { manager, sentPackets } = createSender(); - - await manager.sendText(randomText(60_000), { topic: 't', destinationIdentities: [RECEIVER] }); - - const header = sentPackets.find((p) => p.value.case === 'streamHeader'); - const headerValue = header!.value.value as Extract< - DataPacket['value'], - { case: 'streamHeader' } - >['value']; - expect(headerValue.attributes['lk.compression']).toBe('deflate-raw'); - // Chunk packets carry no smuggled metadata. - for (const packet of sentPackets) { - if (packet.value.case === 'streamChunk') { - expect(packet.value.value.iv).toBeUndefined(); - expect(packet.value.value.version).toBe(0); - } - } - }); - - it('emits each write to the receiver as its packets arrive, before the stream ends', async () => { - const { manager, sentPackets } = createSender(); - const incoming = new IncomingDataStreamManager(); - incoming.setConnected(true); - const readerPromise = new Promise((resolve) => { - incoming.registerTextStreamHandler('t', (reader) => resolve(reader)); - }); - - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - let fed = 0; - const feedNewPackets = () => { - for (const packet of sentPackets.slice(fed)) { - incoming.handleDataStreamPacket(packet, Encryption_Type.NONE); - } - fed = sentPackets.length; - }; - - const writes = ['first write ', 'second write, repeating first write words ', 'third']; - let iterator: AsyncIterator | undefined; - for (const write of writes) { - await writer.write(write); - feedNewPackets(); - if (!iterator) { - const reader = await readerPromise; - iterator = reader[Symbol.asyncIterator](); - } - // The write's full text must be readable now - no trailer has been sent yet. - let got = ''; - while (got.length < write.length) { - const { done, value } = await iterator.next(); - expect(done).toBeFalsy(); - got += value; - } - expect(got).toBe(write); - } - - await writer.close(); - feedNewPackets(); - const end = await iterator!.next(); - expect(end.done).toBe(true); - }); - - it('ignores a duplicated chunk packet with a warning', async () => { - const { manager, sentPackets } = createSender(); - const warnSpy = vi.spyOn(log, 'warn').mockImplementation(() => {}); - - try { - // Compressed chunked fallback: the stateful inflater is what needs dup protection. - const payload = randomText(60_000); - await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); - - // Replay the first chunk packet again right after the original. - const firstChunkIdx = sentPackets.findIndex((p) => p.value.case === 'streamChunk'); - const packets = [...sentPackets]; - packets.splice(firstChunkIdx + 1, 0, packets[firstChunkIdx]); - - const reader = await receiveText(packets, 't'); - expect(await reader.readAll()).toBe(payload); - expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('duplicate chunk')); - } finally { - warnSpy.mockRestore(); - } - }); - - it('errors the stream when a chunk goes missing (gap in chunk indices)', async () => { - const { manager, sentPackets } = createSender(); - - // Compressed chunked fallback: the stateful inflater cannot tolerate gaps. - await manager.sendText(randomText(60_000), { topic: 't', destinationIdentities: [RECEIVER] }); - - // Drop the first chunk packet entirely. - const firstChunkIdx = sentPackets.findIndex((p) => p.value.case === 'streamChunk'); - const packets = sentPackets.filter((_, i) => i !== firstChunkIdx); - - const reader = await receiveText(packets, 't'); - await expect(reader.readAll()).rejects.toThrow(/[Mm]issing chunk/); - }); - - it('sends a long stream of many small writes uncompressed (transcription pattern)', async () => { - const { manager, sentPackets } = createSender(); - const writes = Array.from( - { length: 500 }, - (_, i) => `transcription segment number ${i} with some repeated filler words. `, - ); - - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - for (const write of writes) { - await writer.write(write); - } - await writer.close(); - - const expected = writes.join(''); - // streamText never compresses: the chunk contents are exactly the payload bytes. - expect(chunkContentBytes(sentPackets)).toBe(new TextEncoder().encode(expected).byteLength); - - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe(expected); - }); - - it('streamText never compresses, even for v2 recipients', async () => { - const { manager, sentPackets } = createSender(); - - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - await writer.write('one '); - await writer.write('two'); - await writer.close(); - - // No compression attribute on the header, and chunk contents are plain UTF-8. - const header = sentPackets.find((p) => p.value.case === 'streamHeader'); - const headerValue = header!.value.value as Extract< - DataPacket['value'], - { case: 'streamHeader' } - >['value']; - expect(headerValue.attributes['lk.compression']).toBeUndefined(); - const contents = sentPackets - .filter((p) => p.value.case === 'streamChunk') - .map((p) => new TextDecoder().decode((p.value.value as DataStream_Chunk).content)); - expect(contents.join('')).toBe('one two'); - - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe('one two'); - }); - - it('actually shrinks a compressible chunked sendText payload on the wire', async () => { - const { manager, sentPackets } = createSender(); - // Moderately compressible (small vocabulary) but high enough entropy that the compressed - // output still exceeds the inline budget → chunked fallback with real compression win. - const vocabulary = randomText(2_000).match(/.{1,8}/g)!; - let payload = ''; - while (payload.length < 200_000) { - payload += vocabulary[Math.floor(Math.random() * vocabulary.length)] + ' '; - } - - await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); - - expect(sentPackets.some((p) => p.value.case === 'streamChunk')).toBe(true); - expect(chunkContentBytes(sentPackets)).toBeLessThan(payload.length / 2); - - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe(payload); - }); - - it('round-trips a streamText stream closed without any writes', async () => { - const { manager, sentPackets } = createSender(); - - const writer = await manager.streamText({ topic: 't', destinationIdentities: [RECEIVER] }); - await writer.close(); - - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe(''); - }); - - it('does not compress sendText for a pre-v2 recipient (uncompressed round-trip)', async () => { - const { manager, sentPackets } = createSender(CLIENT_PROTOCOL_DATA_STREAM_RPC); - const payload = 'plain text '.repeat(2_000); - - await manager.sendText(payload, { topic: 't', destinationIdentities: [RECEIVER] }); - - // No compression attribute on the header, and the payload went out as raw bytes. - const header = sentPackets.find((p) => p.value.case === 'streamHeader'); - const headerValue = header!.value.value as Extract< - DataPacket['value'], - { case: 'streamHeader' } - >['value']; - expect(headerValue.attributes['lk.compression']).toBeUndefined(); - expect(chunkContentBytes(sentPackets)).toBe(new TextEncoder().encode(payload).byteLength); - - const reader = await receiveText(sentPackets, 't'); - expect(await reader.readAll()).toBe(payload); - }); -}); diff --git a/src/room/data-stream/compression.test.ts b/src/room/data-stream/compression.test.ts deleted file mode 100644 index 148706fd80..0000000000 --- a/src/room/data-stream/compression.test.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { - deflateRawCompress, - deflateRawCompressStream, - deflateRawDecompress, - inflateRawStream, -} from './compression'; - -function bytes(str: string): Uint8Array { - return new TextEncoder().encode(str); -} - -function text(buf: Uint8Array): string { - return new TextDecoder().decode(buf); -} - -/** A readable stream that emits the given byte arrays in order. */ -function streamOf(...parts: Uint8Array[]): ReadableStream { - return new ReadableStream({ - start(controller) { - for (const part of parts) { - controller.enqueue(part); - } - controller.close(); - }, - }); -} - -async function collect(stream: ReadableStream): Promise { - const reader = stream.getReader(); - const chunks: Uint8Array[] = []; - for (;;) { - const { done, value } = await reader.read(); - if (done) { - break; - } - chunks.push(value); - } - const total = chunks.reduce((n, c) => n + c.byteLength, 0); - const out = new Uint8Array(total); - let offset = 0; - for (const c of chunks) { - out.set(c, offset); - offset += c.byteLength; - } - return out; -} - -describe('data-stream buffered deflate-raw helpers (inline payloads)', () => { - it('round-trips a buffered payload', async () => { - const original = bytes('the quick brown fox '.repeat(500)); - const restored = await deflateRawDecompress(await deflateRawCompress(original)); - expect(text(restored)).toBe(text(original)); - }); - - it('actually compresses repetitive data', async () => { - const original = bytes('A'.repeat(50_000)); - const compressed = await deflateRawCompress(original); - expect(compressed.byteLength).toBeLessThan(original.byteLength); - }); - - it('round-trips an empty payload', async () => { - const restored = await deflateRawDecompress(await deflateRawCompress(new Uint8Array(0))); - expect(restored.byteLength).toBe(0); - }); - - it('streams decompression of a one-shot payload split across many input chunks', async () => { - const original = bytes('hello compressed world '.repeat(2_000)); - const compressed = await deflateRawCompress(original); - - // Feed the compressed bytes in small slices to exercise incremental decompression - a one-shot - // buffer is also a valid input for the streaming decompressor. - const slices: Uint8Array[] = []; - for (let i = 0; i < compressed.byteLength; i += 100) { - slices.push(compressed.slice(i, i + 100)); - } - const restored = await collect(inflateRawStream(streamOf(...slices))); - expect(text(restored)).toBe(text(original)); - }); -}); - -describe('deflateRawCompressStream + inflateRawStream', () => { - it('round-trips a one-shot payload through the streaming decompressor', async () => { - const original = 'first part 日本語🚀 ' + 'repeated filler '.repeat(2_000); - const compressed = await collect(deflateRawCompressStream(bytes(original))); - const restored = await collect(inflateRawStream(streamOf(compressed))); - expect(text(restored)).toBe(original); - }); - - it('actually compresses repetitive data', async () => { - const original = bytes('the quick brown fox '.repeat(2_000)); - const compressed = await collect(deflateRawCompressStream(original)); - expect(compressed.byteLength).toBeLessThan(original.byteLength / 3); - }); - - it('round-trips incompressible input', async () => { - const original = new Uint8Array(10_000); - for (let i = 0; i < original.length; i += 1) { - original[i] = Math.floor(Math.random() * 256); - } - const compressed = await collect(deflateRawCompressStream(original)); - const restored = await collect(inflateRawStream(streamOf(compressed))); - expect(Array.from(restored)).toEqual(Array.from(original)); - }); - - it('round-trips an empty payload', async () => { - const compressed = await collect(deflateRawCompressStream(new Uint8Array(0))); - const restored = await collect(inflateRawStream(streamOf(compressed))); - expect(restored.byteLength).toBe(0); - }); -}); From 0611aba0e407b5c4423311ce15bd6813f0a3468e Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 25 Jun 2026 11:56:24 -0400 Subject: [PATCH 37/68] fix: remove earlier drafts mention --- DATA_STREAMS_SPEC.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DATA_STREAMS_SPEC.md b/DATA_STREAMS_SPEC.md index 45a00f22f5..8bfb003451 100644 --- a/DATA_STREAMS_SPEC.md +++ b/DATA_STREAMS_SPEC.md @@ -208,8 +208,7 @@ Common options: `topic`, `destinationIdentities` (omit ⇒ broadcast), `attribut The v2 signals are carried in dedicated header fields, **not** attributes: `inlineContent` (the single-packet payload, as raw bytes) and `compression` (`NONE` / `DEFLATE_RAW`). `attributes` carries -only caller-supplied metadata. (Earlier drafts smuggled these into reserved `lk.inline_payload` / -`lk.compression` attributes; that is gone — there is no attribute fallback.) +only caller-supplied metadata. ### `sendText` send algorithm From 78fe01535501ab5ac9b408f20e7c346274ad6c2a Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 25 Jun 2026 12:02:34 -0400 Subject: [PATCH 38/68] fix: remove more out of date E2EE stuff and note about utf-8 chunk boundaries --- DATA_STREAMS_SPEC.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/DATA_STREAMS_SPEC.md b/DATA_STREAMS_SPEC.md index 8bfb003451..82f1366c45 100644 --- a/DATA_STREAMS_SPEC.md +++ b/DATA_STREAMS_SPEC.md @@ -159,6 +159,9 @@ Sender Receiver - Content is delivered incrementally as chunks arrive — a receiver must not wait for the trailer to begin yielding content. - Chunk content larger than the MTU budget is split across multiple chunks with contiguous indices. + - Text data streams should ALWAYS split text at valid UTF-8 boundaries, which should already be + implemented for data streams v1. For an example of this behavior, see the relevant web sdk + code: https://github.com/livekit/client-sdk-js/blob/23326f9c9b85d6babb562d54b6a663f132189880/src/room/utils.ts#L758 ### Topics and handlers @@ -180,13 +183,6 @@ encryptionType, etc.) and lets the consumer either: The reader counts received content bytes against `totalLength` (when present) and surfaces an error if the stream ends short, or if more bytes than declared arrive. -### E2EE - -When data-channel encryption is enabled, the header's `encryptionType` is `GCM` and each chunk -carries an `iv`. The receiver MUST enforce a consistent `encryptionType` across a stream's -header and chunks; a mismatch errors the stream. `totalLength` and `attributes` semantics are -unchanged by encryption. - --- ## Part 3: Send APIs From c551b4a656063c0c71e496f4c21f66434972778d Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 25 Jun 2026 12:05:17 -0400 Subject: [PATCH 39/68] fix: update misleading comments --- .../data-stream/incoming/IncomingDataStreamManager.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 449cf52873..60dad8bf46 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -161,8 +161,7 @@ export default class IncomingDataStreamManager { }; // Both inline and chunked byte payloads are deflate-raw compressed; inline as a one-shot - // buffer, chunked as a single stream spanning all chunks (mirrors text). The compression - // flag rides in the header's `compression` field. + // buffer, chunked as a single stream spanning all chunks (mirrors text). const compressed = streamHeader.compression === DataStream_CompressionType.DEFLATE_RAW; if (compressed && !isCompressionStreamSupported()) { @@ -174,7 +173,7 @@ export default class IncomingDataStreamManager { return; } - // Single-packet stream: the entire payload was smuggled into the header's `inlineContent`. + // Single-packet stream: the entire payload was packaged into the header's `inlineContent`. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. const inlineContent = streamHeader.inlineContent; if (typeof inlineContent !== 'undefined') { @@ -250,8 +249,7 @@ export default class IncomingDataStreamManager { }; // Both inline and chunked text payloads are deflate-raw compressed; inline as a one-shot - // buffer, chunked as a single stream spanning all chunks. The compression flag rides in the - // header's `compression` field. + // buffer, chunked as a single stream spanning all chunks. const compressed = streamHeader.compression === DataStream_CompressionType.DEFLATE_RAW; if (compressed && !isCompressionStreamSupported()) { @@ -530,7 +528,8 @@ function inflateRawChunkStream( } const text = decodeOrThrow(value.content); if (text.length > 0) { - controller.enqueue(makeChunk(streamId, outIndex++, encoder.encode(text))); + controller.enqueue(makeChunk(streamId, outIndex, encoder.encode(text))); + outIndex += 1; return; } // Everything so far was a partial codepoint; keep pulling. From 00ba872a65d0ffbd37d8a5c41ec1c1cc1454a365 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 25 Jun 2026 12:46:18 -0400 Subject: [PATCH 40/68] fix: add sendBytes method as the main entrypoint for sending inline bytes data streams --- DATA_STREAMS_SPEC.md | 75 ++++++- src/index.ts | 2 + .../OutgoingDataStreamManager.test.ts | 188 ++++++++++++++++++ .../outgoing/OutgoingDataStreamManager.ts | 164 ++++++++++----- src/room/participant/LocalParticipant.ts | 13 ++ src/room/types.ts | 10 + 6 files changed, 393 insertions(+), 59 deletions(-) diff --git a/DATA_STREAMS_SPEC.md b/DATA_STREAMS_SPEC.md index 82f1366c45..9f2c5d6385 100644 --- a/DATA_STREAMS_SPEC.md +++ b/DATA_STREAMS_SPEC.md @@ -18,7 +18,7 @@ negotiated per-recipient via the participant's advertised `clientProtocol` and c 1. **Single-packet (inline) sends** — small finite payloads are smuggled entirely into the header packet, skipping the chunk/trailer packets (1 packet instead of 3). Gated on `clientProtocol`. -2. **Compression** — finite, fully-known payloads (`sendText`/`sendFile`) are deflate-raw +2. **Compression** — finite, fully-known payloads (`sendText`/`sendBytes`/`sendFile`) are deflate-raw compressed; incremental writers (`streamText`/`streamBytes`) are never compressed. Gated on the `CAP_COMPRESSION_DEFLATE_RAW` capability (separately from `clientProtocol`). 3. **Header size limit** — the header packet must fit the MTU budget, bounding attribute size and @@ -187,15 +187,22 @@ if the stream ends short, or if more bytes than declared arrive. ## Part 3: Send APIs -Four send operations, two finite (full payload known up front) and two incremental: +Five send operations, three finite (full payload known up front) and two incremental: | API | Content | Payload known up front? | Eligible for inline? | Eligible for compression? | |-----|---------|-------------------------|----------------------|---------------------------| | `sendText(text, opts)` | text | yes | yes | yes | +| `sendBytes(bytes, opts)` | bytes | yes | yes | yes | | `sendFile(file, opts)` | bytes | yes (streamed from disk) | **no** | yes | | `streamText(opts) -> writer` | text | no (incremental writes) | no | **no** | | `streamBytes(opts) -> writer` | bytes | no (incremental writes) | no | **no** | +`sendBytes` is the byte-stream analogue of `sendText`: the whole payload is already in memory, so it +gets the same inline single-packet fast path and one-shot compression. `sendFile` is the special +case for files — the payload is streamed from disk (never buffered) and so does **not** get the +inline path. `sendBytes` does not infer a `name`/`mimeType` from its input; its byte-stream header +defaults to `name = "unknown"` and `mimeType = "application/octet-stream"`. + Common options: `topic`, `destinationIdentities` (omit ⇒ broadcast), `attributes` (map), and for the finite APIs a `compress` boolean (default `true`, opt-out). `sendText` additionally supports `attachments` (each becomes an attached byte stream referenced by @@ -224,6 +231,21 @@ only caller-supplied metadata. 5. Send each attachment as its own byte stream (`sendFile` semantics), referenced by `attachedStreamIds` in the text header. +### `sendBytes` send algorithm + +`sendBytes` mirrors `sendText` for an in-memory `Uint8Array` (byte-stream header, no attachments): + +1. `totalLength` is the payload's byte length. +2. **Inline attempt** (when all recipients are v2): build a byte-stream header carrying the raw + payload in `inlineContent` with `compression = NONE`. If `compress` and the runtime supports + compression, deflate-raw the payload and, **only if the compressed form is smaller**, put the + compressed bytes in `inlineContent` and set `compression = DEFLATE_RAW`. If the serialized header + packet is `<= STREAM_CHUNK_SIZE_BYTES`, send it as a single packet and finish. Otherwise fall + through. +3. **Chunked** (fallback): send a byte-stream header (`compression = DEFLATE_RAW` iff + compression-eligible, else `NONE`) then the payload — deflate-raw compressed when eligible — as + chunk packets, then the trailer. + ### `sendFile` send algorithm `sendFile` is fully streamed from the file's byte stream and is **never** sent inline (file uploads @@ -261,8 +283,8 @@ field (raw bytes) and sent as **one** packet (no chunks, no trailer). The decisi naturally accounts for attributes, topic, framing, and (when used) the compressed payload all together. -- Inline applies to `sendText` only (not `sendFile`, not incremental writers), and only when all - recipients are v2 and there are no attachments. +- Inline applies to `sendText` and `sendBytes` (not `sendFile`, not incremental writers), and only + when all recipients are v2 (and, for `sendText`, there are no attachments). - The receiver detects an inline stream by the presence of `inlineContent` on the header. It synthesizes an already-complete stream from those bytes (decompressing first if `compression` is `DEFLATE_RAW`) and never waits for chunk/trailer packets. @@ -272,7 +294,7 @@ together. ## Part 5: Compression Compression is **deflate-raw** (raw DEFLATE, no zlib/gzip wrapper). It is applied only by the finite -send APIs (`sendText`/`sendFile`), where the full payload is known up front. Two forms: +send APIs (`sendText`/`sendBytes`/`sendFile`), where the full payload is known up front. Two forms: ### Inline payload compression (single packet) @@ -557,6 +579,49 @@ trailer's `streamId`. Three participant rooms are used: - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the raw UTF-8 of the text — inline still happens (all three are v2) but compression is gated off by `noCompression`. +#### `sendBytes` (in-memory byte payloads) + +`sendBytes` mirrors `sendText`'s inline + compression behavior on a `byteHeader`. Every emitted +header has `contentHeader.case === 'byteHeader'`, and the returned `info` defaults `name` to +`'unknown'` and `mimeType` to `'application/octet-stream'` (and threads `topic`/`attributes`/`size`). + +22. **Short compressible bytes → single inline packet, compressed** (all-v2 room) + - Call `sendBytes(utf8('hello hello compressible world'), { topic, attributes: { foo: 'bar' }, destinationIdentities: ['alice','bob'] })`. + - Expect **1** packet: a `byteHeader` with `compression === DEFLATE_RAW`; `inlineContent` is a + `Uint8Array` that is **not** the raw payload. `info.name === 'unknown'`, + `info.mimeType === 'application/octet-stream'`, `info.size === payload.byteLength`, + `info.attributes === { foo: 'bar' }`. + +23. **Short incompressible bytes → single inline packet, raw** (all-v2 room) + - Call `sendBytes([0x00,0x01,0x02,0x03], { ..., destinationIdentities: ['alice','bob'] })`. + - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the four raw bytes. + +24. **Bytes to a recipient lacking the compression capability → single inline packet, raw** + - Call `sendBytes(utf8('hello hello compressible world'), { ..., destinationIdentities: ['noCompression'] })`. + - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the raw payload (inline is + gated on `clientProtocol`; compression on the capability). + +25. **Large highly-compressible bytes → single inline packet, compressed** (all-v2 room) + - Call `sendBytes(Uint8Array(50_000).fill(0x01), { ..., destinationIdentities: ['alice','bob'] })`. + - Expect **1** packet. `compression === DEFLATE_RAW`; `inlineContent` byte length is **less** than + 50 000 (compresses well under the MTU, so it still goes inline). + +26. **Large somewhat-compressible bytes → compressed multi-packet** (all-v2 room) + - `sendBytes` a ~50 KB payload of 50 × (`'hello world'` + 1 000 random chars) (UTF-8 encoded) to + `['alice','bob']`. + - Expect **5** packets: `byteHeader` (`compression = DEFLATE_RAW`, **no** `inlineContent`) + **3** + chunks (first is 15 000 bytes) + trailer. + +27. **`compress: false`, large bytes → uncompressed multi-packet** (all-v2 room) + - Call `sendBytes(Uint8Array(40_000).fill(0x07), { ..., destinationIdentities: ['alice','bob'], compress: false })`. + - Expect **5** packets: `byteHeader` (`compression = NONE`, no `inlineContent`) + **3** chunks + (15 000, 15 000, 10 000; all `0x07`) + trailer. + +28. **Bytes to a pre-v2 room → legacy uncompressed multi-packet** + - Call `sendBytes([0x00,0x01,0x02,0x03], { topic })` with only pre-v2 recipients. + - Expect **3** packets: `byteHeader` (`compression = NONE`, no `inlineContent`) + one chunk + (`chunkIndex 0`, the four raw bytes) + trailer. Never inline, never compressed. + ### `IncomingDataStreamManager` (receive side) **Test harness.** Construct the manager, register a text or byte stream handler for a topic that diff --git a/src/index.ts b/src/index.ts index 1d218cd89d..be7a70b0f3 100644 --- a/src/index.ts +++ b/src/index.ts @@ -104,6 +104,8 @@ export type { TranscriptionSegment, ChatMessage, SendTextOptions, + SendBytesOptions, + ByteStreamInfo, } from './room/types'; export * from './version'; export { diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts index ac6a9da785..fd64cfed04 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts @@ -766,4 +766,192 @@ describe('OutgoingDataStreamManager', () => { ); }); }); + + describe('sendBytes', () => { + describe('v2 -> room of all v2', () => { + let manager: OutgoingDataStreamManager, sentPackets: Array; + beforeEach(() => { + const result = createManager({ + alice: CLIENT_PROTOCOL_DATA_STREAM_V2, + bob: CLIENT_PROTOCOL_DATA_STREAM_V2, + noCompression: [CLIENT_PROTOCOL_DATA_STREAM_V2, []], + }); + manager = result.manager; + sentPackets = result.sentPackets; + }); + + it('should send short compressible BYTE payload as a single compressed packet (happy path)', async () => { + const bytes = new TextEncoder().encode('hello hello compressible world'); + const info = await manager.sendBytes(bytes, { + topic: 'my-topic', + attributes: { foo: 'bar' }, + destinationIdentities: ['alice', 'bob'], + }); + + expect(sentPackets).toHaveLength(1); + expect(sentPackets[0].value.case).toBe('streamHeader'); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.topic).toStrictEqual('my-topic'); + expect(header.contentHeader.case).toBe('byteHeader'); + + // Compressed inline payload + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); + expect(header.inlineContent).toBeInstanceOf(Uint8Array); + expect(header.inlineContent).not.toStrictEqual(bytes); + + // Returned info uses the byte-stream defaults + expect(info.name).toStrictEqual('unknown'); + expect(info.mimeType).toStrictEqual('application/octet-stream'); + expect(info.size).toStrictEqual(bytes.byteLength); + expect(info.attributes).toStrictEqual({ foo: 'bar' }); + }); + + it('should send short uncompressible BYTE payload inline without compression', async () => { + const bytes = new Uint8Array([0x00, 0x01, 0x02, 0x03]); + const info = await manager.sendBytes(bytes, { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + }); + + expect(sentPackets).toHaveLength(1); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.contentHeader.case).toBe('byteHeader'); + // Tiny payload doesn't shrink under DEFLATE framing, so it's sent raw. + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toStrictEqual(bytes); + }); + + it('should send BYTE payload inline with NO compression if a recipient does not support compression', async () => { + const bytes = new TextEncoder().encode('hello hello compressible world'); + const info = await manager.sendBytes(bytes, { + topic: 'my-topic', + destinationIdentities: ['noCompression'], + }); + + // Inline still applies (gated on v2 alone), but compression is skipped. + expect(sentPackets).toHaveLength(1); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.contentHeader.case).toBe('byteHeader'); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toStrictEqual(bytes); + }); + + it('should send long, highly compressible BYTE payload as a single compressed packet', async () => { + // Repeating bytes compress well under the 15k MTU even though the raw payload is 50k. + const bytes = new Uint8Array(50_000).fill(0x01); + const info = await manager.sendBytes(bytes, { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + }); + + expect(sentPackets).toHaveLength(1); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.contentHeader.case).toBe('byteHeader'); + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); + expect(header.inlineContent).toBeInstanceOf(Uint8Array); + expect(header.inlineContent!.byteLength).toBeLessThan(bytes.byteLength); + }); + + it('should send long but somewhat compressible BYTE payload as a compressed multi packet stream', async () => { + const bytes = new TextEncoder().encode( + new Array(50) + .fill(null) + .map(() => `hello world${randomText(1_000)}`) + .join(''), + ); + const info = await manager.sendBytes(bytes, { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + }); + + // 1 header + 3 chunks + 1 trailer (compressed ~45k -> 3 MTU chunks) + expect(sentPackets).toHaveLength(5); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.contentHeader.case).toBe('byteHeader'); + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); + expect(header.inlineContent).toBeUndefined(); + + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + const chunk = chunkOf(sentPackets[1]); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toHaveLength(15_000); // MTU + expect(sentPackets[2].value.case).toStrictEqual('streamChunk'); + expect(sentPackets[3].value.case).toStrictEqual('streamChunk'); + expect(sentPackets[4].value.case).toStrictEqual('streamTrailer'); + expect(trailerOf(sentPackets[4]).streamId).toStrictEqual(info.id); + }); + + it('should skip compression for an inline payload when compress: false', async () => { + const bytes = new TextEncoder().encode('hello hello compressible world'); + const info = await manager.sendBytes(bytes, { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + compress: false, + }); + + expect(sentPackets).toHaveLength(1); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toStrictEqual(bytes); + }); + + it('should send an uncompressed multi packet stream when compress: false and payload exceeds the MTU', async () => { + const bytes = new Uint8Array(40_000).fill(0x07); + const info = await manager.sendBytes(bytes, { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + compress: false, + }); + + // 40k -> 15k + 15k + 10k chunks. 1 header + 3 chunks + 1 trailer = 5 packets. + expect(sentPackets).toHaveLength(5); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.contentHeader.case).toBe('byteHeader'); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toBeUndefined(); + + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + let chunk = chunkOf(sentPackets[1]); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toHaveLength(15_000); + expect(chunk.content.every((byte) => byte === 0x07)).toBeTruthy(); + chunk = chunkOf(sentPackets[3]); + expect(chunk.content).toHaveLength(10_000); + expect(sentPackets[4].value.case).toStrictEqual('streamTrailer'); + }); + }); + + describe('v2 -> room of all v1', () => { + it('should send a legacy uncompressed multi packet stream', async () => { + const { manager, sentPackets } = createManager({ + alice: CLIENT_PROTOCOL_DEFAULT, + jim: CLIENT_PROTOCOL_DATA_STREAM_RPC, + }); + const bytes = new Uint8Array([0x00, 0x01, 0x02, 0x03]); + const info = await manager.sendBytes(bytes, { topic: 'my-topic' }); + + // No v2 recipients: header + chunk + trailer, never inline, never compressed. + expect(sentPackets).toHaveLength(3); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.contentHeader.case).toBe('byteHeader'); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toBeUndefined(); + + expect(sentPackets[1].value.case).toStrictEqual('streamChunk'); + const chunk = chunkOf(sentPackets[1]); + expect(chunk.chunkIndex).toStrictEqual(0n); + expect(chunk.content).toStrictEqual(bytes); + expect(sentPackets[2].value.case).toStrictEqual('streamTrailer'); + expect(trailerOf(sentPackets[2]).streamId).toStrictEqual(info.id); + }); + }); + }); }); diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 60fc6d35ec..cc547a5c7b 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -15,6 +15,7 @@ import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { EngineEvent } from '../../events'; import type { ByteStreamInfo, + SendBytesOptions, SendFileOptions, SendTextOptions, StreamBytesOptions, @@ -195,6 +196,83 @@ export default class OutgoingDataStreamManager { return info; } + /** + * Sends a complete in-memory byte payload. Mirrors {@link sendText}'s semantics: when every + * recipient supports data streams v2 the payload rides inline in a single header packet + * (optionally deflate-raw compressed), otherwise it is sent as a (optionally compressed) + * chunked byte stream. Unlike {@link sendFile}, the whole payload is already in memory, so the + * inline single-packet fast path applies. + */ + async sendBytes(bytes: Uint8Array, options?: SendBytesOptions): Promise { + const streamId = crypto.randomUUID(); + const destinationIdentities = options?.destinationIdentities; + const compress = options?.compress ?? true; + + const info: ByteStreamInfo = { + id: streamId, + name: 'unknown', + mimeType: 'application/octet-stream', + timestamp: Date.now(), + topic: options?.topic ?? '', + size: bytes.byteLength, // NOTE: size is always the pre-compression byte length + attributes: options?.attributes, + encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled + ? Encryption_Type.GCM + : Encryption_Type.NONE, + }; + + // Phase 1: Try to send as a single packet data stream + if (this.allRecipientsSupportV2(destinationIdentities)) { + // The payload rides in the header's `inlineContent` (raw bytes). Compress when the runtime + // supports it, but only keep the result if it actually shrinks the payload (deflate framing + // makes tiny payloads larger). The compression flag is carried in the header's `compression` + // field; user attributes are left untouched. + let inlineContent: Uint8Array = bytes; + let compression = DataStream_CompressionType.NONE; + if ( + compress && + isCompressionStreamSupported() && + this.allRecipientsSupportCompression(destinationIdentities) + ) { + const compressed = await deflateRawCompress(bytes); + if (compressed.byteLength < bytes.byteLength) { + inlineContent = compressed; + compression = DataStream_CompressionType.DEFLATE_RAW; + } + } + + const header = buildByteStreamHeader(info, { compression, inlineContent }); + const packet = createStreamHeaderPacket(header, destinationIdentities); + + if (packet.toBinary().byteLength <= STREAM_CHUNK_SIZE_BYTES) { + await this.engine.sendDataPacket(packet, DataChannelKind.RELIABLE); + options?.onProgress?.(1); + return info; + } + } + + // Phase 2/3: header + (optionally compressed) chunk packets + trailer. + const compressEligible = + compress && + isCompressionStreamSupported() && + this.allRecipientsSupportV2(destinationIdentities) && + this.allRecipientsSupportCompression(destinationIdentities); + + const header = buildByteStreamHeader(info, { + compression: compressEligible + ? DataStream_CompressionType.DEFLATE_RAW + : DataStream_CompressionType.NONE, + }); + const packet = createStreamHeaderPacket(header, destinationIdentities); + const source = compressEligible + ? deflateRawCompressReadable(readableFromBytes(bytes)) + : readableFromBytes(bytes); + await this.sendChunkedByteStream(packet, streamId, destinationIdentities, source); + options?.onProgress?.(1); + + return info; + } + /** * Returns true only if every recipient is known to support data streams v2 (single-packet inline * streams and compression). For a targeted send this checks the named destination identities; for @@ -350,8 +428,9 @@ export default class OutgoingDataStreamManager { /** * Streams a file as a chunked byte stream, compressed (deflate-raw) when the runtime supports it * and every recipient is on data streams v2. The file is piped `file.stream()` → - * (`CompressionStream`) → chunk packets, so it is never fully buffered in memory — unlike - * `sendText`, there is no inline single-packet fast path for files. + * (`CompressionStream`) → chunk packets via {@link sendChunkedByteStream}, so it is never fully + * buffered in memory — unlike {@link sendBytes}, there is no inline single-packet fast path for + * files (the compressed size can't be known up front without buffering the whole file). */ private async _sendFile( streamId: string, @@ -359,7 +438,23 @@ export default class OutgoingDataStreamManager { options?: SendFileOptions, ): Promise { const destinationIdentities = options?.destinationIdentities; - const compress = options?.compress ?? true; + const compress = + (options?.compress ?? true) && + isCompressionStreamSupported() && + this.allRecipientsSupportV2(destinationIdentities) && + this.allRecipientsSupportCompression(destinationIdentities); + + const info: ByteStreamInfo = { + id: streamId, + name: file.name, + mimeType: options?.mimeType ?? file.type, + topic: options?.topic ?? '', + timestamp: Date.now(), + size: file.size, + encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled + ? Encryption_Type.GCM + : Encryption_Type.NONE, + }; // Phase 1: Try to send as a single packet data stream // @@ -367,58 +462,19 @@ export default class OutgoingDataStreamManager { // time how well the file contents will compress (and whether the total output will be under the // MTU). Revisit this in the future though. - // Phase 2: Try to send a multi packet data stream with compressed bytes - if ( - compress && - isCompressionStreamSupported() && - this.allRecipientsSupportV2(destinationIdentities) && - this.allRecipientsSupportCompression(destinationIdentities) - ) { - const info: ByteStreamInfo = { - id: streamId, - name: file.name, - mimeType: options?.mimeType ?? file.type, - topic: options?.topic ?? '', - timestamp: Date.now(), - size: file.size, - encryptionType: this.engine.e2eeManager?.isDataChannelEncryptionEnabled - ? Encryption_Type.GCM - : Encryption_Type.NONE, - }; - - const header = buildByteStreamHeader(info, { - compression: DataStream_CompressionType.DEFLATE_RAW, - }); - const packet = createStreamHeaderPacket(header, destinationIdentities); - await this.sendChunkedByteStream( - packet, - streamId, - destinationIdentities, - deflateRawCompressReadable(file.stream()), - ); - - return info; - } - - // Phase 3 / fallback: header + plain uncompressed chunk packets + trailer. - const writer = await this.streamBytes({ - streamId, - totalSize: file.size, - name: file.name, - mimeType: options?.mimeType ?? file.type, - topic: options?.topic, - destinationIdentities: options?.destinationIdentities, + // Phase 2 (compressed) / Phase 3 (uncompressed fallback): header + chunk packets + trailer. Both + // funnel through the shared sendChunkedByteStream primitive, differing only by whether the file + // stream is wrapped in the deflate-raw compressor; the file is never fully buffered in memory. + const header = buildByteStreamHeader(info, { + compression: compress + ? DataStream_CompressionType.DEFLATE_RAW + : DataStream_CompressionType.NONE, }); - const reader = file.stream().getReader(); - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - await writer.write(value); - } - await writer.close(); - return writer.info; + const packet = createStreamHeaderPacket(header, destinationIdentities); + const source = compress ? deflateRawCompressReadable(file.stream()) : file.stream(); + await this.sendChunkedByteStream(packet, streamId, destinationIdentities, source); + + return info; } async streamBytes(options?: StreamBytesOptions) { diff --git a/src/room/participant/LocalParticipant.ts b/src/room/participant/LocalParticipant.ts index fae8d8934f..1993a1d154 100644 --- a/src/room/participant/LocalParticipant.ts +++ b/src/room/participant/LocalParticipant.ts @@ -82,8 +82,10 @@ import { sourceToKind, } from '../track/utils'; import { + type ByteStreamInfo, type ChatMessage, type DataPublishOptions, + type SendBytesOptions, type SendFileOptions, type SendTextOptions, type StreamBytesOptions, @@ -1851,6 +1853,17 @@ export default class LocalParticipant extends Participant { return this.roomOutgoingDataStreamManager.sendFile(file, options); } + /** + * Sends the given bytes to participants in the room via the data channel. + * For files, consider using {@link sendFile}; for longer/incremental payloads, {@link streamBytes}. + * + * @param bytes The byte payload + * @param options.topic Topic identifier used to route the stream to appropriate handlers. + */ + async sendBytes(bytes: Uint8Array, options?: SendBytesOptions): Promise { + return this.roomOutgoingDataStreamManager.sendBytes(bytes, options); + } + /** * Stream bytes incrementally to participants in the room via the data channel. * For sending files, consider using {@link sendFile} instead. diff --git a/src/room/types.ts b/src/room/types.ts index f3f1f1c21b..7cc7ef9da6 100644 --- a/src/room/types.ts +++ b/src/room/types.ts @@ -26,6 +26,16 @@ export interface SendTextOptions { compress?: boolean; } +export interface SendBytesOptions { + topic?: string; + destinationIdentities?: Array; + attributes?: Record; + onProgress?: (progress: number) => void; + /** Whether to compress the payload (deflate-raw). Defaults to true. Compression is only applied + * when every recipient supports data streams v2 and the runtime can compress. */ + compress?: boolean; +} + export interface StreamTextOptions { topic?: string; destinationIdentities?: Array; From cc53c78d43fae7d2c1629e23d7f8009d99561f84 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 25 Jun 2026 12:52:16 -0400 Subject: [PATCH 41/68] fix: make sendFile optional in spec --- DATA_STREAMS_SPEC.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/DATA_STREAMS_SPEC.md b/DATA_STREAMS_SPEC.md index 9f2c5d6385..e2498c5eb6 100644 --- a/DATA_STREAMS_SPEC.md +++ b/DATA_STREAMS_SPEC.md @@ -193,7 +193,7 @@ Five send operations, three finite (full payload known up front) and two increme |-----|---------|-------------------------|----------------------|---------------------------| | `sendText(text, opts)` | text | yes | yes | yes | | `sendBytes(bytes, opts)` | bytes | yes | yes | yes | -| `sendFile(file, opts)` | bytes | yes (streamed from disk) | **no** | yes | +| `sendFile(file, opts)` _(optional)_ | bytes | yes (streamed from disk) | **no** | yes | | `streamText(opts) -> writer` | text | no (incremental writes) | no | **no** | | `streamBytes(opts) -> writer` | bytes | no (incremental writes) | no | **no** | @@ -203,6 +203,16 @@ case for files — the payload is streamed from disk (never buffered) and so doe inline path. `sendBytes` does not infer a `name`/`mimeType` from its input; its byte-stream header defaults to `name = "unknown"` and `mimeType = "application/octet-stream"`. +**`sendBytes`, `sendText`, `streamText`, and `streamBytes` are the required send APIs; `sendFile` is +optional and platform-dependent.** Since `sendBytes` already covers sending an in-memory byte +payload, `sendFile` is only worth adding on platforms that have a native streamable file type whose +contents can be read without buffering the whole thing into memory (e.g. the `File`/`Blob` object in +browser JS, which exposes a `ReadableStream`). On such platforms `sendFile` is a thin convenience +wrapper that derives the `name`/`mimeType` from the file and streams its contents. On platforms with +no such type, omit `sendFile` entirely — callers read the bytes themselves and use `sendBytes`. The +two produce identical wire output for the same bytes (a `byteHeader` stream), so there is no +interop concern in omitting it. + Common options: `topic`, `destinationIdentities` (omit ⇒ broadcast), `attributes` (map), and for the finite APIs a `compress` boolean (default `true`, opt-out). `sendText` additionally supports `attachments` (each becomes an attached byte stream referenced by @@ -248,6 +258,10 @@ only caller-supplied metadata. ### `sendFile` send algorithm +> `sendFile` is **optional** (see the note under the API table): implement it only where the +> platform has a native streamable file type. Where present, it behaves as below; where absent, +> callers use `sendBytes` instead. + `sendFile` is fully streamed from the file's byte stream and is **never** sent inline (file uploads are an edge case; the inline single-packet optimization is intentionally dropped for them): @@ -446,6 +460,9 @@ trailer's `streamId`. Three participant rooms are used: at `clientProtocol 2` with **no** capabilities. - **mixed** — `alice` (0), `bob`/`jim` (2 + cap), `mallory` (1), `noCompression` (2, no cap). +The `sendFile` cases below apply only to SDKs that implement the optional `sendFile` API (see Part 3); +omit them on platforms without a native streamable file type. + #### Sending to a room where every recipient is pre-v2 1. **Short text → legacy multi-packet, uncompressed** From 9cc7a28282100d864b4e8211266f7caaac495d13 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 25 Jun 2026 12:58:50 -0400 Subject: [PATCH 42/68] fix: run npm run format --- src/room/Room.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/room/Room.ts b/src/room/Room.ts index a243fa8d33..acabb86c38 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -2513,7 +2513,10 @@ class Room extends (EventEmitter as new () => TypedEmitter) roomOptions: InternalRoomOptions, ): Array { const capabilities: Array = []; - if (isFrameMetadataSupported(roomOptions.frameMetadata ?? roomOptions.packetTrailer) || !!this.e2eeManager) { + if ( + isFrameMetadataSupported(roomOptions.frameMetadata ?? roomOptions.packetTrailer) || + !!this.e2eeManager + ) { capabilities.push(ClientInfo_Capability.CAP_PACKET_TRAILER); } // Advertise deflate-raw decompression support so peers know they can send us compressed data From 66f2811ae77159f9c840f12e775cc301aca1c330 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 25 Jun 2026 13:02:01 -0400 Subject: [PATCH 43/68] fix: address remaining lint errors --- .../incoming/IncomingDataStreamManager.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts index 67e9421c28..e54a1c9885 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -831,9 +831,9 @@ describe('IncomingDataStreamManager', () => { originalDecompressionStream: typeof DecompressionStream; try { originalCompressionStream = CompressionStream; - (CompressionStream as any) = undefined; + (globalThis as any).CompressionStream = undefined; originalDecompressionStream = DecompressionStream; - (DecompressionStream as any) = undefined; + (globalThis as any).DecompressionStream = undefined; const manager = new IncomingDataStreamManager(); manager.setConnected(true); @@ -893,8 +893,8 @@ describe('IncomingDataStreamManager', () => { Promise.race([readerPromise, Promise.resolve('still pending')]), ).resolves.toStrictEqual('still pending'); } finally { - CompressionStream = originalCompressionStream!; - DecompressionStream = originalDecompressionStream!; + (globalThis as any).CompressionStream = originalCompressionStream!; + (globalThis as any).DecompressionStream = originalDecompressionStream!; } }); @@ -906,9 +906,9 @@ describe('IncomingDataStreamManager', () => { originalDecompressionStream: typeof DecompressionStream; try { originalCompressionStream = CompressionStream; - (CompressionStream as any) = undefined; + (globalThis as any).CompressionStream = undefined; originalDecompressionStream = DecompressionStream; - (DecompressionStream as any) = undefined; + (globalThis as any).DecompressionStream = undefined; const manager = new IncomingDataStreamManager(); manager.setConnected(true); @@ -968,8 +968,8 @@ describe('IncomingDataStreamManager', () => { Promise.race([readerPromise, Promise.resolve('still pending')]), ).resolves.toStrictEqual('still pending'); } finally { - CompressionStream = originalCompressionStream!; - DecompressionStream = originalDecompressionStream!; + (globalThis as any).CompressionStream = originalCompressionStream!; + (globalThis as any).DecompressionStream = originalDecompressionStream!; } }); }); From 72ed4f4c8d9bf8ed50270b5d7fbe2b2e68b00221 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 25 Jun 2026 13:31:23 -0400 Subject: [PATCH 44/68] fix: add missing changeset --- .changeset/slick-geese-happen.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/slick-geese-happen.md diff --git a/.changeset/slick-geese-happen.md b/.changeset/slick-geese-happen.md new file mode 100644 index 0000000000..bb06ec43ec --- /dev/null +++ b/.changeset/slick-geese-happen.md @@ -0,0 +1,5 @@ +--- +'livekit-client': patch +--- + +Add support for data streams v2 From e4b07a563e1faa7dc65a10823253aefa41851215 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 29 Jun 2026 15:41:06 -0400 Subject: [PATCH 45/68] fix: add log when unknown compression scheme encountered --- .../incoming/IncomingDataStreamManager.ts | 70 +++++++++++++------ 1 file changed, 48 insertions(+), 22 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 60dad8bf46..0175afa1e2 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -160,17 +160,30 @@ export default class IncomingDataStreamManager { encryptionType, }; - // Both inline and chunked byte payloads are deflate-raw compressed; inline as a one-shot - // buffer, chunked as a single stream spanning all chunks (mirrors text). - const compressed = streamHeader.compression === DataStream_CompressionType.DEFLATE_RAW; - - if (compressed && !isCompressionStreamSupported()) { - // NOTE: this shouldn't really ever happen, if this warning is logged then the sender - // isn't properly abiding by the data streams v2 protocol. - log.warn( - `Data stream ${streamHeader.streamId} received with deflate-raw compression, but this browser does not have support for DecompressionStream. Dropping...`, - ); - return; + // Determine if the byte payload needs to be decompressed. + let compressed; + switch (streamHeader.compression) { + case DataStream_CompressionType.DEFLATE_RAW: + if (!isCompressionStreamSupported()) { + // NOTE: this shouldn't really ever happen, if this warning is logged then the sender + // isn't properly abiding by the data streams v2 protocol. + log.warn( + `Data stream ${streamHeader.streamId} received with deflate-raw compression, but this browser does not have support for DecompressionStream. Dropping...`, + ); + return; + } + compressed = true; + break; + case DataStream_CompressionType.NONE: + compressed = false; + break; + default: + // NOTE: this shouldn't really ever happen, if this warning is logged then the sender + // isn't properly abiding by the data streams v2 protocol. + log.warn( + `Data stream ${streamHeader.streamId} received with unknown compression type ${streamHeader.compression}, dropping...`, + ); + return; } // Single-packet stream: the entire payload was packaged into the header's `inlineContent`. @@ -248,17 +261,30 @@ export default class IncomingDataStreamManager { attachedStreamIds: streamHeader.contentHeader.value.attachedStreamIds, }; - // Both inline and chunked text payloads are deflate-raw compressed; inline as a one-shot - // buffer, chunked as a single stream spanning all chunks. - const compressed = streamHeader.compression === DataStream_CompressionType.DEFLATE_RAW; - - if (compressed && !isCompressionStreamSupported()) { - // NOTE: this shouldn't really ever happen, if this warning is logged then the sender - // isn't properly abiding by the data streams v2 protocol. - log.warn( - `Data stream ${streamHeader.streamId} received with deflate-raw compression, but this browser does not have support for DecompressionStream. Dropping...`, - ); - return; + // Determine if the byte payload needs to be decompressed. + let compressed; + switch (streamHeader.compression) { + case DataStream_CompressionType.DEFLATE_RAW: + if (!isCompressionStreamSupported()) { + // NOTE: this shouldn't really ever happen, if this warning is logged then the sender + // isn't properly abiding by the data streams v2 protocol. + log.warn( + `Data stream ${streamHeader.streamId} received with deflate-raw compression, but this browser does not have support for DecompressionStream. Dropping...`, + ); + return; + } + compressed = true; + break; + case DataStream_CompressionType.NONE: + compressed = false; + break; + default: + // NOTE: this shouldn't really ever happen, if this warning is logged then the sender + // isn't properly abiding by the data streams v2 protocol. + log.warn( + `Data stream ${streamHeader.streamId} received with unknown compression type ${streamHeader.compression}, dropping...`, + ); + return; } // Single-packet stream: the entire payload was smuggled into the header's `inlineContent`. From 2b034a71ba42cc5b4705f4e3d49138d5fdbfc061 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 29 Jun 2026 16:21:04 -0400 Subject: [PATCH 46/68] fix: convert intermedite streams to transformstream --- src/room/data-stream/compression.ts | 51 ++-- .../incoming/IncomingDataStreamManager.ts | 220 +++++++++--------- 2 files changed, 130 insertions(+), 141 deletions(-) diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index ead2eefd8f..500a0d4fc7 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -1,22 +1,26 @@ /** * Compression helpers for data streams. The buffered deflate-raw variant ({@link deflateRawCompress}) * is for the inline (single-packet) case where the payload is small and bounded; - * {@link deflateRawCompressReadable} / {@link inflateRawStream} serve the chunked (multi-packet) - * `sendText`/`sendFile` paths, streaming the compressed bytes through without buffering the whole - * payload. + * {@link deflateRawCompressReadable} / {@link inflateRawTransform} serve the chunked (multi-packet) + * `sendText`/`sendBytes`/`sendFile` paths, streaming the compressed bytes through without buffering + * the whole payload. * * These operate on bytes (not strings) so a single set of helpers serves both text and byte streams; * the `TextEncoder`/`TextDecoder` boundary lives at the manager/reader edges. * - * Like the rest of the SDK, the platform-stream helpers drive `getWriter()`/`getReader()` directly - * instead of `pipeThrough`, which sidesteps the `CompressionStream` lib-type mismatches. + * Note the asymmetry between the two platform-codec wrappers: the inflate side + * ({@link inflateRawTransform}) is exposed as a `ReadableWritablePair` so it drops straight into the + * receive-side `pipeThrough` chain, at the cost of one localized cast to bridge a DOM lib-type + * mismatch (see that function). The compress side ({@link deflateRawCompressReadable}) is consumed as + * a plain readable on the send path, so it drives `getWriter()`/`getReader()` directly and needs no + * cast. * * @internal */ /** * Pipes a byte stream through `CompressionStream('deflate-raw')`, exposing the compressed output as - * a readable — the compression counterpart of {@link inflateRawStream}. Drives the source into the + * a readable — the compression counterpart of {@link inflateRawTransform}. Drives the source into the * compressor in the background (forwarding source errors via `abort`), so callers can forward the * compressed output incrementally without buffering the whole payload. Used for the chunked * `sendText`/`sendFile` paths, where the full payload is known up front but is streamed (e.g. from @@ -47,33 +51,16 @@ export function deflateRawCompressReadable( } /** - * Streams raw-deflate input through a single `DecompressionStream('deflate-raw')` for the lifetime - * of the stream (inverse of {@link StreamingDeflate}). Inflate emits output greedily, so as long as - * the sender flushed at write boundaries, each write's content is produced as soon as its - * compressed bytes are written. Source errors are forwarded by aborting the decompression input. + * A `deflate-raw` decompression transform (inverse of {@link deflateRawCompressReadable}): pipe a + * stream of compressed bytes through it to get the decompressed bytes. Inflate emits output greedily, + * so as long as the sender flushed at write boundaries each write's content is produced as soon as + * its compressed bytes arrive. */ -export function inflateRawStream(input: ReadableStream): ReadableStream { - const ds = new DecompressionStream('deflate-raw'); - const writer = ds.writable.getWriter(); - // Drive compressed input into the decompressor in the background; the IIFE handles its own errors - // by aborting the writable (which surfaces on the readable side), so this never rejects. - const pipe = (async () => { - const reader = input.getReader(); - try { - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - await writer.write(value as NonSharedUint8Array); - } - await writer.close(); - } catch (err) { - await writer.abort(err).catch(() => {}); - } - })(); - pipe.catch(() => {}); - return ds.readable; +export function inflateRawTransform(): ReadableWritablePair { + return new DecompressionStream('deflate-raw') as unknown as ReadableWritablePair< + Uint8Array, + Uint8Array + >; } /** deflate-raw compresses a byte array in full. Use for inline payloads; prefer the streaming diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 0175afa1e2..1f56f5f551 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -10,7 +10,7 @@ import log from '../../../logger'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { type ByteStreamInfo, type StreamController, type TextStreamInfo } from '../../types'; import { bigIntToNumber, isCompressionStreamSupported, numberToBigInt } from '../../utils'; -import { deflateRawDecompress, inflateRawStream } from '../compression'; +import { deflateRawDecompress, inflateRawTransform } from '../compression'; import { type ByteStreamHandler, ByteStreamReader, @@ -429,103 +429,73 @@ function createInlineStream( } /** - * Unwraps a stream of compressed `DataStream_Chunk`s to their compressed bytes (in `chunkIndex` - * order), guarding ordering for the stateful decompressor that consumes them. A stateful - * decompressor silently corrupts on duplicated or out-of-order input, so duplicates are dropped - * (with a warning - in-order delivery is expected on the reliable channel, but reconnect handling - * may replay) and a gap is a hard error. Shared by the text and byte deflate-raw decoders. + * Validates that chunks are received in order, dropping duplicates and throwing if gaps are found. + * + * A stateful decompressor silently corrupts on duplicated or out-of-order input, so duplicates are + * dropped (with a warning - in-order delivery is expected on the reliable channel, but reconnect + * handling may replay) and a gap is a hard error. Shared by the text and byte deflate-raw decoders. */ -function orderedCompressedBytes( - srcReader: ReadableStreamDefaultReader, - streamId: string, -): ReadableStream { +function ensureOrderedChunks(streamId: string): TransformStream { let lastChunkIndex = -1; - return new ReadableStream({ - pull: async (controller) => { - while (true) { - const { done, value } = await srcReader.read(); - if (done) { - controller.close(); - return; - } - const index = bigIntToNumber(value.chunkIndex); - if (index <= lastChunkIndex) { - log.warn( - `ignoring duplicate chunk ${index} for compressed data stream ${streamId} (last processed: ${lastChunkIndex})`, - ); - continue; - } - if (index > lastChunkIndex + 1) { - throw new DataStreamError( - `Missing chunk(s) ${lastChunkIndex + 1}..${index - 1} for compressed data stream ${streamId} - cannot continue decompressing`, - DataStreamErrorReason.Incomplete, - ); - } - lastChunkIndex = index; - controller.enqueue(value.content); + return new TransformStream({ + transform: (value, controller) => { + const index = bigIntToNumber(value.chunkIndex); + if (index <= lastChunkIndex) { + log.warn( + `ignoring duplicate chunk ${index} for compressed data stream ${streamId} (last processed: ${lastChunkIndex})`, + ); return; } + if (index > lastChunkIndex + 1) { + throw new DataStreamError( + `Missing chunk(s) ${lastChunkIndex + 1}..${index - 1} for compressed data stream ${streamId} - cannot continue decompressing`, + DataStreamErrorReason.Incomplete, + ); + } + lastChunkIndex = index; + controller.enqueue(value); }, - cancel: (reason) => srcReader.cancel(reason), }); } -/** - * Transforms a stream of deflate-raw-compressed byte `DataStream_Chunk`s into a stream of - * decompressed chunks, so `ByteStreamReader` consumes it unchanged. All chunk contents are fed (in - * `chunkIndex` order) through ONE raw-deflate decompressor for the stream's lifetime; decompressed - * output is re-wrapped as chunks as soon as it is produced. The sender (sendFile) compresses the - * whole payload in one shot, but the format also supports a single context-takeover stream - * sync-flushed at write boundaries, so a future incremental streamBytes could compress with no - * protocol change. Errors on the source stream propagate to the reader. - */ -function inflateRawByteChunkStream( - raw: ReadableStream, - streamId: string, -): ReadableStream { - const srcReader = raw.getReader(); - const decompressedReader = inflateRawStream( - orderedCompressedBytes(srcReader, streamId), - ).getReader(); +/** Unwraps compressed `DataStream_Chunk`s to their compressed bytes (in `chunkIndex` order), */ +function chunksToBytes(): TransformStream { + return new TransformStream({ + transform: (value, controller) => { + controller.enqueue(value.content); + }, + }); +} +/** Re-wraps decompressed bytes into contiguous `DataStream_Chunk`s, skipping inflate's empty reads. */ +function bytesToChunks(streamId: string): TransformStream { let outIndex = 0; - return new ReadableStream({ - pull: async (controller) => { - while (true) { - const { done, value } = await decompressedReader.read(); - if (done) { - controller.close(); - return; - } - if (value.byteLength > 0) { - controller.enqueue(makeChunk(streamId, outIndex++, value)); - return; - } - // Inflate can emit empty reads; keep pulling until there is content or the stream ends. + return new TransformStream({ + transform: (value, controller) => { + // Inflate can emit empty reads; only synthesize a chunk when there is content. + if (value.byteLength > 0) { + controller.enqueue( + new DataStream_Chunk({ + streamId, + chunkIndex: numberToBigInt(outIndex), + content: value, + }) + ); + outIndex += 1; } }, - cancel: (reason) => { - decompressedReader.cancel(reason).catch(() => {}); - srcReader.cancel(reason).catch(() => {}); - }, }); } /** - * Transforms a stream of deflate-raw-compressed text `DataStream_Chunk`s into a stream of - * decompressed chunks, so `TextStreamReader` consumes it unchanged. Builds on - * {@link inflateRawByteChunkStream} (single decompressor + ordering guard) and adds a streaming - * `TextDecoder` that reframes the decompressed bytes on UTF-8 character boundaries (a write larger - * than the MTU spans several packets, which may split a codepoint) so each synthesized chunk - * decodes independently. Errors on the source stream propagate to the reader. + * Reframes decompressed byte chunks onto UTF-8 character boundaries via a streaming `TextDecoder` + * (a write larger than the MTU spans several packets, which may split a codepoint), so each + * synthesized text chunk decodes independently. The `flush` emits the decoder's trailing bytes. */ -function inflateRawChunkStream( - raw: ReadableStream, - streamId: string, -): ReadableStream { - const byteReader = inflateRawByteChunkStream(raw, streamId).getReader(); - +function bytesToDecodedUtf8(streamId: string): TransformStream { const decoder = new TextDecoder('utf-8', { fatal: true }); + const encoder = new TextEncoder(); + let outIndex = 0; const decodeOrThrow = (bytes?: Uint8Array): string => { try { @@ -538,39 +508,71 @@ function inflateRawChunkStream( } }; - const encoder = new TextEncoder(); - return new ReadableStream({ - pull: async (controller) => { - while (true) { - const { done, value } = await byteReader.read(); - if (done) { - const tail = decodeOrThrow(); - if (tail.length > 0) { - controller.enqueue(makeChunk(streamId, outIndex, encoder.encode(tail))); - outIndex += 1; - } - controller.close(); - return; - } - const text = decodeOrThrow(value.content); - if (text.length > 0) { - controller.enqueue(makeChunk(streamId, outIndex, encoder.encode(text))); - outIndex += 1; - return; - } - // Everything so far was a partial codepoint; keep pulling. + return new TransformStream({ + transform: (value, controller) => { + const text = decodeOrThrow(value); + // Everything so far may have been a partial codepoint; only emit once we have characters. + if (text.length > 0) { + controller.enqueue( + new DataStream_Chunk({ + streamId, + chunkIndex: numberToBigInt(outIndex), + content: encoder.encode(text), + }) + ); + outIndex += 1; } }, - cancel: (reason) => { - byteReader.cancel(reason).catch(() => {}); + flush: (controller) => { + const tail = decodeOrThrow(); + if (tail.length > 0) { + controller.enqueue( + new DataStream_Chunk({ + streamId, + chunkIndex: numberToBigInt(outIndex), + content: encoder.encode(tail), + }) + ); + outIndex += 1; + } }, }); } -function makeChunk(streamId: string, chunkIndex: number, content: Uint8Array): DataStream_Chunk { - return new DataStream_Chunk({ - streamId, - chunkIndex: numberToBigInt(chunkIndex), - content: content as NonSharedUint8Array, - }); +/** + * Transforms a stream of deflate-raw-compressed byte `DataStream_Chunk`s into a stream of + * decompressed chunks, so `ByteStreamReader` consumes it unchanged. All chunk contents are fed (in + * `chunkIndex` order) through ONE raw-deflate decompressor for the stream's lifetime; decompressed + * output is re-wrapped as chunks as soon as it is produced. The sender (sendFile) compresses the + * whole payload in one shot, but the format also supports a single context-takeover stream + * sync-flushed at write boundaries, so a future incremental streamBytes could compress with no + * protocol change. Errors and cancellation propagate through the pipe chain. + */ +function inflateRawByteChunkStream( + raw: ReadableStream, + streamId: string, +): ReadableStream { + return raw + .pipeThrough(ensureOrderedChunks(streamId)) + .pipeThrough(chunksToBytes()) + .pipeThrough(inflateRawTransform()) + .pipeThrough(bytesToChunks(streamId)); +} + +/** + * Transforms a stream of deflate-raw-compressed text `DataStream_Chunk`s into a stream of + * decompressed chunks, so `TextStreamReader` consumes it unchanged. Builds on + * {@link inflateRawByteChunkStream} (single decompressor + ordering guard) and adds a streaming + * `TextDecoder` that reframes the decompressed bytes on UTF-8 character boundaries so each + * synthesized chunk decodes independently. Errors and cancellation propagate through the pipe chain. + */ +function inflateRawChunkStream( + raw: ReadableStream, + streamId: string, +): ReadableStream { + return raw + .pipeThrough(ensureOrderedChunks(streamId)) + .pipeThrough(chunksToBytes()) + .pipeThrough(inflateRawTransform()) + .pipeThrough(bytesToDecodedUtf8(streamId)); } From 2280836c1ba3812e9ac708cb62d6aa69b8f1e910 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 29 Jun 2026 17:10:27 -0400 Subject: [PATCH 47/68] fix: port compression to use TransformStream --- src/room/data-stream/compression.ts | 56 ++++++------------- .../incoming/IncomingDataStreamManager.ts | 10 ++-- .../outgoing/OutgoingDataStreamManager.ts | 8 +-- 3 files changed, 28 insertions(+), 46 deletions(-) diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index 500a0d4fc7..66210b468c 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -1,57 +1,37 @@ /** * Compression helpers for data streams. The buffered deflate-raw variant ({@link deflateRawCompress}) * is for the inline (single-packet) case where the payload is small and bounded; - * {@link deflateRawCompressReadable} / {@link inflateRawTransform} serve the chunked (multi-packet) - * `sendText`/`sendBytes`/`sendFile` paths, streaming the compressed bytes through without buffering - * the whole payload. + * {@link deflateRawTransform} / {@link inflateRawTransform} serve the chunked (multi-packet) + * `sendText`/`sendBytes`/`sendFile` paths, streaming the bytes through without buffering the whole + * payload. * * These operate on bytes (not strings) so a single set of helpers serves both text and byte streams; * the `TextEncoder`/`TextDecoder` boundary lives at the manager/reader edges. * - * Note the asymmetry between the two platform-codec wrappers: the inflate side - * ({@link inflateRawTransform}) is exposed as a `ReadableWritablePair` so it drops straight into the - * receive-side `pipeThrough` chain, at the cost of one localized cast to bridge a DOM lib-type - * mismatch (see that function). The compress side ({@link deflateRawCompressReadable}) is consumed as - * a plain readable on the send path, so it drives `getWriter()`/`getReader()` directly and needs no - * cast. + * Both streaming variants are exposed as `ReadableWritablePair`s so they drop straight into a + * `pipeThrough` chain. Each needs one localized cast to bridge a DOM lib-type mismatch: the platform + * `CompressionStream`/`DecompressionStream` type their `writable` as `WritableStream` + * (a wider element type than `Uint8Array`), and `WritableStream` is covariant in `W`, so neither + * is structurally a `ReadableWritablePair` without help. * * @internal */ /** - * Pipes a byte stream through `CompressionStream('deflate-raw')`, exposing the compressed output as - * a readable — the compression counterpart of {@link inflateRawTransform}. Drives the source into the - * compressor in the background (forwarding source errors via `abort`), so callers can forward the - * compressed output incrementally without buffering the whole payload. Used for the chunked - * `sendText`/`sendFile` paths, where the full payload is known up front but is streamed (e.g. from - * `file.stream()`) rather than held in memory. + * A `deflate-raw` compression transform (inverse of {@link inflateRawTransform}): pipe a byte stream + * through it to get the compressed bytes without buffering the whole payload. Used for the chunked + * `sendText`/`sendBytes`/`sendFile` paths, where the full payload is known up front but is streamed + * (e.g. from `file.stream()`) rather than held in memory. */ -export function deflateRawCompressReadable( - input: ReadableStream, -): ReadableStream { - const cs = new CompressionStream('deflate-raw'); - const writer = cs.writable.getWriter(); - const pipe = (async () => { - const reader = input.getReader(); - try { - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - await writer.write(value as NonSharedUint8Array); - } - await writer.close(); - } catch (err) { - await writer.abort(err).catch(() => {}); - } - })(); - pipe.catch(() => {}); - return cs.readable; +export function deflateRawTransform(): ReadableWritablePair { + return new CompressionStream('deflate-raw') as unknown as ReadableWritablePair< + Uint8Array, + Uint8Array + >; } /** - * A `deflate-raw` decompression transform (inverse of {@link deflateRawCompressReadable}): pipe a + * A `deflate-raw` decompression transform (inverse of {@link deflateRawTransform}): pipe a * stream of compressed bytes through it to get the decompressed bytes. Inflate emits output greedily, * so as long as the sender flushed at write boundaries each write's content is produced as soon as * its compressed bytes arrive. diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 1f56f5f551..617c8bfda5 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -435,7 +435,9 @@ function createInlineStream( * dropped (with a warning - in-order delivery is expected on the reliable channel, but reconnect * handling may replay) and a gap is a hard error. Shared by the text and byte deflate-raw decoders. */ -function ensureOrderedChunks(streamId: string): TransformStream { +function ensureOrderedChunks( + streamId: string, +): TransformStream { let lastChunkIndex = -1; return new TransformStream({ transform: (value, controller) => { @@ -479,7 +481,7 @@ function bytesToChunks(streamId: string): TransformStream Date: Wed, 1 Jul 2026 10:34:56 -0400 Subject: [PATCH 48/68] feat: add zero length single packet data stream tests --- .../IncomingDataStreamManager.test.ts | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts index e54a1c9885..73df5988d7 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -972,5 +972,73 @@ describe('IncomingDataStreamManager', () => { (globalThis as any).DecompressionStream = originalDecompressionStream!; } }); + + it('should receive a v2 SINGLE PACKET text data stream with zero length', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + totalLength: 0n, + attributes: { foo: 'bar' }, + inlineContent: new Uint8Array(0), // Empty buffer that is 0 bytes long + contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual(''); + expect(reader.info.attributes?.foo).toStrictEqual('bar'); + }); + + it('should receive a v2 SINGLE PACKET byte data stream with zero length', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerByteStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + manager.handleDataStreamPacket( + new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'byte/plain', + timestamp: 0n, + totalLength: 0n, + attributes: { foo: 'bar' }, + inlineContent: new Uint8Array(0), // Empty buffer that is 0 bytes long + contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({}) }, + }), + }, + }), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual([new Uint8Array(0)]); + expect(reader.info.attributes?.foo).toStrictEqual('bar'); + }); }); }); From 1536956edb2ed47d372d77189a6ea01c2dd6a824 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 1 Jul 2026 10:39:38 -0400 Subject: [PATCH 49/68] fix: swap break for return --- src/room/data-stream/incoming/IncomingDataStreamManager.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 617c8bfda5..1e22087587 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -236,7 +236,7 @@ export default class IncomingDataStreamManager { identity: participantIdentity, }, ); - break; + return; } case 'textHeader': { const streamHandlerCallback = this.textStreamHandlers.get(streamHeader.topic); @@ -333,7 +333,7 @@ export default class IncomingDataStreamManager { ), { identity: participantIdentity }, ); - break; + return; } } } From 21426e2b77a0b97e5888918c1bc31abcbf446603 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 1 Jul 2026 11:32:50 -0400 Subject: [PATCH 50/68] fix: ensure data stream payload will only ever be compressed once --- src/room/data-stream/compression.ts | 6 +- .../outgoing/OutgoingDataStreamManager.ts | 124 ++++++++++++------ 2 files changed, 85 insertions(+), 45 deletions(-) diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index 66210b468c..e6ba20e9d5 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -62,12 +62,12 @@ export async function deflateRawDecompress(data: Uint8Array): Promise): Promise { +/** Drains a byte stream, concatenating all of its chunks into a single array. */ +export async function collect(stream: ReadableStream): Promise { const reader = stream.getReader(); const chunks: Uint8Array[] = []; let total = 0; - for (;;) { + while (true) { const { done, value } = await reader.read(); if (done) { break; diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 156d8fe528..d467962f66 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -29,7 +29,7 @@ import { readableFromBytes, splitUtf8, } from '../../utils'; -import { deflateRawCompress, deflateRawTransform } from '../compression'; +import { collect, deflateRawTransform } from '../compression'; import { STREAM_CHUNK_SIZE_BYTES } from '../constants'; import { ByteStreamWriter, TextStreamWriter } from './StreamWriter'; import { @@ -98,23 +98,30 @@ export default class OutgoingDataStreamManager { : Encryption_Type.NONE, }; + const compressEligible = + compress && + isCompressionStreamSupported() && + this.allRecipientsSupportV2(options?.destinationIdentities) && + this.allRecipientsSupportCompression(options?.destinationIdentities); + let compressedStream = compressEligible + ? CompressedStreamState.fromStream( + readableFromBytes(textInBytes).pipeThrough(deflateRawTransform()), + ) + : null; + // Phase 1: Try to send as a single packet data stream const noAttachments = !options?.attachments || options.attachments.length === 0; if (noAttachments && this.allRecipientsSupportV2(options?.destinationIdentities)) { - // The payload rides in the header's `inlineContent` (raw bytes). Compress when the runtime - // supports it, but only keep the result if it actually shrinks the payload (deflate framing - // makes tiny strings larger). The compression flag is carried in the header's `compression` - // field; user attributes are left untouched. + // The payload rides in the header's `inlineContent` (raw bytes). Keep the compressed form only + // if it actually shrinks the payload (deflate framing makes tiny strings larger). The + // compression flag is carried in the header's `compression` field; user attributes are left + // untouched. let inlineContent: Uint8Array = textInBytes; let compression = DataStream_CompressionType.NONE; - if ( - compress && - isCompressionStreamSupported() && - this.allRecipientsSupportCompression(options?.destinationIdentities) - ) { - const compressed = await deflateRawCompress(textInBytes); - if (compressed.byteLength < textInBytes.byteLength) { - inlineContent = compressed; + if (compressedStream) { + const collectedBytes = await compressedStream.collect(); + if (collectedBytes.byteLength < textInBytes.byteLength) { + inlineContent = collectedBytes; compression = DataStream_CompressionType.DEFLATE_RAW; } } @@ -140,12 +147,7 @@ export default class OutgoingDataStreamManager { }; // Phase 2: Try to send a multi packet data stream with compressed bytes - if ( - compress && - isCompressionStreamSupported() && - this.allRecipientsSupportV2(options?.destinationIdentities) && - this.allRecipientsSupportCompression(options?.destinationIdentities) - ) { + if (compressedStream) { info.attachedStreamIds = fileIds; const header = buildTextStreamHeader(info, undefined, { @@ -156,7 +158,7 @@ export default class OutgoingDataStreamManager { packet, streamId, options?.destinationIdentities, - readableFromBytes(textEncoder.encode(text)).pipeThrough(deflateRawTransform()), + compressedStream.stream(), ); // set text part of progress to 1 @@ -221,22 +223,29 @@ export default class OutgoingDataStreamManager { : Encryption_Type.NONE, }; + const compressEligible = + compress && + isCompressionStreamSupported() && + this.allRecipientsSupportV2(destinationIdentities) && + this.allRecipientsSupportCompression(destinationIdentities); + let compressedStream = compressEligible + ? CompressedStreamState.fromStream( + readableFromBytes(bytes).pipeThrough(deflateRawTransform()), + ) + : null; + // Phase 1: Try to send as a single packet data stream if (this.allRecipientsSupportV2(destinationIdentities)) { - // The payload rides in the header's `inlineContent` (raw bytes). Compress when the runtime - // supports it, but only keep the result if it actually shrinks the payload (deflate framing - // makes tiny payloads larger). The compression flag is carried in the header's `compression` - // field; user attributes are left untouched. + // The payload rides in the header's `inlineContent` (raw bytes). Keep the compressed form only + // if it actually shrinks the payload (deflate framing makes tiny payloads larger). The + // compression flag is carried in the header's `compression` field; user attributes are left + // untouched. let inlineContent: Uint8Array = bytes; let compression = DataStream_CompressionType.NONE; - if ( - compress && - isCompressionStreamSupported() && - this.allRecipientsSupportCompression(destinationIdentities) - ) { - const compressed = await deflateRawCompress(bytes); - if (compressed.byteLength < bytes.byteLength) { - inlineContent = compressed; + if (compressedStream) { + const collectedBytes = await compressedStream.collect(); + if (collectedBytes.byteLength < bytes.byteLength) { + inlineContent = collectedBytes; compression = DataStream_CompressionType.DEFLATE_RAW; } } @@ -252,21 +261,13 @@ export default class OutgoingDataStreamManager { } // Phase 2/3: header + (optionally compressed) chunk packets + trailer. - const compressEligible = - compress && - isCompressionStreamSupported() && - this.allRecipientsSupportV2(destinationIdentities) && - this.allRecipientsSupportCompression(destinationIdentities); - const header = buildByteStreamHeader(info, { - compression: compressEligible + compression: compressedStream ? DataStream_CompressionType.DEFLATE_RAW : DataStream_CompressionType.NONE, }); const packet = createStreamHeaderPacket(header, destinationIdentities); - const source = compressEligible - ? readableFromBytes(bytes).pipeThrough(deflateRawTransform()) - : readableFromBytes(bytes); + const source = compressedStream ? compressedStream.stream() : readableFromBytes(bytes); await this.sendChunkedByteStream(packet, streamId, destinationIdentities, source); options?.onProgress?.(1); @@ -550,6 +551,45 @@ export default class OutgoingDataStreamManager { } } +/** A utility to handle lazily calling {@link collect} on a stream of compressed bytes only when + * requested, to avoid buffering lots of data into memory until the program needs it. + */ +class CompressedStreamState { + private state: + | { type: 'stream'; stream: ReadableStream } + | { type: 'collected'; bytes: Uint8Array }; + + private constructor(state: typeof this.state) { + this.state = state; + } + + static fromStream(stream: ReadableStream) { + return new CompressedStreamState({ type: 'stream', stream }); + } + + /** Collect data from the stream into memory and return as a Uint8Array. */ + async collect() { + switch (this.state.type) { + case 'stream': + const bytes = await collect(this.state.stream); + this.state = { type: 'collected', bytes }; + return bytes; + case 'collected': + return this.state.bytes; + } + } + + /** Pass wrapped stream through to downstream consumer. */ + stream() { + switch (this.state.type) { + case 'stream': + return this.state.stream; + case 'collected': + return readableFromBytes(this.state.bytes); + } + } +} + /** * Sends a stream `streamHeader` packet, enforcing that it fits the MTU budget. The header carries * the user attributes (plus topic/framing), and a single `DataPacket` larger than the MTU can't be From 7f488819f4025dc8487a0e6fde4fb6b646ef9d36 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 1 Jul 2026 11:39:56 -0400 Subject: [PATCH 51/68] fix: address lint error --- src/room/data-stream/outgoing/OutgoingDataStreamManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index d467962f66..068920d6ad 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -557,7 +557,7 @@ export default class OutgoingDataStreamManager { class CompressedStreamState { private state: | { type: 'stream'; stream: ReadableStream } - | { type: 'collected'; bytes: Uint8Array }; + | { type: 'collected'; bytes: Uint8Array }; private constructor(state: typeof this.state) { this.state = state; From b4326850112bcb91603f245a2dca686ff4c7798c Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 1 Jul 2026 11:59:49 -0400 Subject: [PATCH 52/68] feat: add name / mimeType to new sendBytes method --- src/room/data-stream/outgoing/OutgoingDataStreamManager.ts | 4 ++-- src/room/types.ts | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 068920d6ad..b5e33d7555 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -212,8 +212,8 @@ export default class OutgoingDataStreamManager { const info: ByteStreamInfo = { id: streamId, - name: 'unknown', - mimeType: 'application/octet-stream', + name: options?.name ?? 'unknown', + mimeType: options?.mimeType ?? 'application/octet-stream', timestamp: Date.now(), topic: options?.topic ?? '', size: bytes.byteLength, // NOTE: size is always the pre-compression byte length diff --git a/src/room/types.ts b/src/room/types.ts index 7cc7ef9da6..d12579afbc 100644 --- a/src/room/types.ts +++ b/src/room/types.ts @@ -34,6 +34,8 @@ export interface SendBytesOptions { /** Whether to compress the payload (deflate-raw). Defaults to true. Compression is only applied * when every recipient supports data streams v2 and the runtime can compress. */ compress?: boolean; + name?: string; + mimeType?: string; } export interface StreamTextOptions { From 1edcb689e49ecef4a92ad1ca2da5825d04dea63e Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 1 Jul 2026 12:03:16 -0400 Subject: [PATCH 53/68] docs: update CompressedStreamState docstring --- .../outgoing/OutgoingDataStreamManager.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index b5e33d7555..e4a3b6823f 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -551,8 +551,21 @@ export default class OutgoingDataStreamManager { } } -/** A utility to handle lazily calling {@link collect} on a stream of compressed bytes only when - * requested, to avoid buffering lots of data into memory until the program needs it. +/** + * Wraps a stream of compressed bytes and defers buffering it into memory until (and only if) the + * full length is actually needed. + * + * The ideal way to send a compressed stream is incrementally: compress each chunk and send it as + * soon as it's ready, without ever holding the whole payload in memory. But to decide whether a + * payload is small enough to fit in a single-packet data stream, we have to know the compressed + * length — and the only way to learn that is to compress everything and add up the bytes. That + * forces us to buffer the entire compressed output before we can make the decision. + * + * This class lets us have it both ways. It holds the compressed stream unread by default and only + * collects it into memory ({@link collect}) when a caller needs the length for the single-packet + * size check. If that check never happens, the stream is instead {@link stream|passed straight + * through} to the downstream consumer without ever being fully buffered. Once collected, later + * calls reuse the buffered bytes rather than re-collecting. */ class CompressedStreamState { private state: From a3ffb95b281941c1a94215fd7a6bea41fc1075bf Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 1 Jul 2026 12:25:52 -0400 Subject: [PATCH 54/68] feat: add compressedStream.lengthLargerThan to allow for partial stream materialization --- .../outgoing/OutgoingDataStreamManager.ts | 131 ++++++++++++++++-- 1 file changed, 117 insertions(+), 14 deletions(-) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index e4a3b6823f..b45129d63b 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -119,10 +119,17 @@ export default class OutgoingDataStreamManager { let inlineContent: Uint8Array = textInBytes; let compression = DataStream_CompressionType.NONE; if (compressedStream) { - const collectedBytes = await compressedStream.collect(); - if (collectedBytes.byteLength < textInBytes.byteLength) { - inlineContent = collectedBytes; - compression = DataStream_CompressionType.DEFLATE_RAW; + // The compressed form is only worth inlining if it's both smaller than the raw payload and + // small enough to fit in a single packet. Probe just up to that budget so a large payload + // isn't fully buffered here — if it's over budget it stays streamable for Phase 2. + const inlineBudget = Math.min(textInBytes.byteLength, STREAM_CHUNK_SIZE_BYTES); + const tooLong = await compressedStream.lengthLargerThan(inlineBudget); + if (!tooLong) { + const collectedBytes = await compressedStream.collect(); + if (collectedBytes.byteLength < textInBytes.byteLength) { + inlineContent = collectedBytes; + compression = DataStream_CompressionType.DEFLATE_RAW; + } } } @@ -243,10 +250,17 @@ export default class OutgoingDataStreamManager { let inlineContent: Uint8Array = bytes; let compression = DataStream_CompressionType.NONE; if (compressedStream) { - const collectedBytes = await compressedStream.collect(); - if (collectedBytes.byteLength < bytes.byteLength) { - inlineContent = collectedBytes; - compression = DataStream_CompressionType.DEFLATE_RAW; + // The compressed form is only worth inlining if it's both smaller than the raw payload and + // small enough to fit in a single packet. Probe just up to that budget so a large payload + // isn't fully buffered here — if it's over budget it stays streamable for Phase 2. + const inlineBudget = Math.min(bytes.byteLength, STREAM_CHUNK_SIZE_BYTES); + const tooLong = await compressedStream.lengthLargerThan(inlineBudget); + if (!tooLong) { + const collectedBytes = await compressedStream.collect(); + if (collectedBytes.byteLength < bytes.byteLength) { + inlineContent = collectedBytes; + compression = DataStream_CompressionType.DEFLATE_RAW; + } } } @@ -561,15 +575,22 @@ export default class OutgoingDataStreamManager { * length — and the only way to learn that is to compress everything and add up the bytes. That * forces us to buffer the entire compressed output before we can make the decision. * - * This class lets us have it both ways. It holds the compressed stream unread by default and only - * collects it into memory ({@link collect}) when a caller needs the length for the single-packet - * size check. If that check never happens, the stream is instead {@link stream|passed straight - * through} to the downstream consumer without ever being fully buffered. Once collected, later - * calls reuse the buffered bytes rather than re-collecting. + * This class lets us have it both ways. It holds the compressed stream unread by default. When a + * caller needs to know whether the payload fits in a single packet, {@link lengthLargerThan} reads + * only far enough to answer that (stopping the moment the length exceeds the budget) rather than + * buffering — and waiting for — the whole stream. If the stream turns out to be small it ends up + * fully {@link collect|collected}; if it's too large the read prefix is retained and the remainder + * can still be {@link stream|streamed through} incrementally to the downstream consumer, so nothing + * is buffered twice or dropped. Once fully collected, later calls reuse the buffered bytes. */ class CompressedStreamState { private state: | { type: 'stream'; stream: ReadableStream } + | { + type: 'partial'; + buffered: Uint8Array; + reader: ReadableStreamDefaultReader; + } | { type: 'collected'; bytes: Uint8Array }; private constructor(state: typeof this.state) { @@ -580,6 +601,58 @@ class CompressedStreamState { return new CompressedStreamState({ type: 'stream', stream }); } + /** + * Reads from the compressed stream only far enough to decide whether its total length exceeds + * `n`, stopping early the moment it does. Avoids buffering (and waiting for) the whole stream just + * to learn it is too large to inline. + * + * Returns true if the stream is longer than `n` bytes. On a `false` result the stream was shorter, + * so it is now fully `collected`; on a `true` result the read prefix is retained in a `partial` + * state and the rest can still be streamed via {@link stream} or drained via {@link collect}. + * Intended to be called once, before {@link collect}/{@link stream}. + */ + async lengthLargerThan(n: number): Promise { + switch (this.state.type) { + case 'collected': + return this.state.bytes.byteLength > n; + case 'partial': + // Already read past `n` to get here (that's the only way we enter this state). + return true; + case 'stream': { + const reader = this.state.stream.getReader(); + const chunks: Array = []; + let total = 0; + let exceeded = false; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + chunks.push(value); + total += value.byteLength; + if (total > n) { + exceeded = true; + break; + } + } + // Merge whatever we've read into one contiguous buffer. + const buffered = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + buffered.set(chunk, offset); + offset += chunk.byteLength; + } + if (exceeded) { + this.state = { type: 'partial', buffered, reader }; + return true; + } + reader.releaseLock(); + this.state = { type: 'collected', bytes: buffered }; + return buffered.byteLength > n; + } + } + } + /** Collect data from the stream into memory and return as a Uint8Array. */ async collect() { switch (this.state.type) { @@ -587,18 +660,48 @@ class CompressedStreamState { const bytes = await collect(this.state.stream); this.state = { type: 'collected', bytes }; return bytes; + case 'partial': { + // Drain the reconstructed stream (buffered prefix + rest of the reader) in one pass. + const collectedBytes = await collect(this.stream()); + this.state = { type: 'collected', bytes: collectedBytes }; + return collectedBytes; + } case 'collected': return this.state.bytes; } } /** Pass wrapped stream through to downstream consumer. */ - stream() { + stream(): ReadableStream { switch (this.state.type) { case 'stream': return this.state.stream; case 'collected': return readableFromBytes(this.state.bytes); + case 'partial': { + // Emit the already-read prefix as one chunk, then resume pumping the retained reader. + const { buffered, reader } = this.state; + let prefixSent = false; + return new ReadableStream({ + async pull(controller) { + if (!prefixSent) { + prefixSent = true; + controller.enqueue(buffered); + return; + } + const { done, value } = await reader.read(); + if (done) { + reader.releaseLock(); + controller.close(); + } else { + controller.enqueue(value); + } + }, + cancel(reason) { + return reader.cancel(reason); + }, + }); + } } } } From 27f4afdb54e2abf1ccae9acbfe77ae8e9b98d502 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 7 Jul 2026 14:43:24 -0400 Subject: [PATCH 55/68] Revert "feat: add compressedStream.lengthLargerThan to allow for partial stream materialization" This reverts commit 25af9bcbf0b1558f58f41a31fbcf89f14300ca5c. --- .../outgoing/OutgoingDataStreamManager.ts | 131 ++---------------- 1 file changed, 14 insertions(+), 117 deletions(-) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index b45129d63b..e4a3b6823f 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -119,17 +119,10 @@ export default class OutgoingDataStreamManager { let inlineContent: Uint8Array = textInBytes; let compression = DataStream_CompressionType.NONE; if (compressedStream) { - // The compressed form is only worth inlining if it's both smaller than the raw payload and - // small enough to fit in a single packet. Probe just up to that budget so a large payload - // isn't fully buffered here — if it's over budget it stays streamable for Phase 2. - const inlineBudget = Math.min(textInBytes.byteLength, STREAM_CHUNK_SIZE_BYTES); - const tooLong = await compressedStream.lengthLargerThan(inlineBudget); - if (!tooLong) { - const collectedBytes = await compressedStream.collect(); - if (collectedBytes.byteLength < textInBytes.byteLength) { - inlineContent = collectedBytes; - compression = DataStream_CompressionType.DEFLATE_RAW; - } + const collectedBytes = await compressedStream.collect(); + if (collectedBytes.byteLength < textInBytes.byteLength) { + inlineContent = collectedBytes; + compression = DataStream_CompressionType.DEFLATE_RAW; } } @@ -250,17 +243,10 @@ export default class OutgoingDataStreamManager { let inlineContent: Uint8Array = bytes; let compression = DataStream_CompressionType.NONE; if (compressedStream) { - // The compressed form is only worth inlining if it's both smaller than the raw payload and - // small enough to fit in a single packet. Probe just up to that budget so a large payload - // isn't fully buffered here — if it's over budget it stays streamable for Phase 2. - const inlineBudget = Math.min(bytes.byteLength, STREAM_CHUNK_SIZE_BYTES); - const tooLong = await compressedStream.lengthLargerThan(inlineBudget); - if (!tooLong) { - const collectedBytes = await compressedStream.collect(); - if (collectedBytes.byteLength < bytes.byteLength) { - inlineContent = collectedBytes; - compression = DataStream_CompressionType.DEFLATE_RAW; - } + const collectedBytes = await compressedStream.collect(); + if (collectedBytes.byteLength < bytes.byteLength) { + inlineContent = collectedBytes; + compression = DataStream_CompressionType.DEFLATE_RAW; } } @@ -575,22 +561,15 @@ export default class OutgoingDataStreamManager { * length — and the only way to learn that is to compress everything and add up the bytes. That * forces us to buffer the entire compressed output before we can make the decision. * - * This class lets us have it both ways. It holds the compressed stream unread by default. When a - * caller needs to know whether the payload fits in a single packet, {@link lengthLargerThan} reads - * only far enough to answer that (stopping the moment the length exceeds the budget) rather than - * buffering — and waiting for — the whole stream. If the stream turns out to be small it ends up - * fully {@link collect|collected}; if it's too large the read prefix is retained and the remainder - * can still be {@link stream|streamed through} incrementally to the downstream consumer, so nothing - * is buffered twice or dropped. Once fully collected, later calls reuse the buffered bytes. + * This class lets us have it both ways. It holds the compressed stream unread by default and only + * collects it into memory ({@link collect}) when a caller needs the length for the single-packet + * size check. If that check never happens, the stream is instead {@link stream|passed straight + * through} to the downstream consumer without ever being fully buffered. Once collected, later + * calls reuse the buffered bytes rather than re-collecting. */ class CompressedStreamState { private state: | { type: 'stream'; stream: ReadableStream } - | { - type: 'partial'; - buffered: Uint8Array; - reader: ReadableStreamDefaultReader; - } | { type: 'collected'; bytes: Uint8Array }; private constructor(state: typeof this.state) { @@ -601,58 +580,6 @@ class CompressedStreamState { return new CompressedStreamState({ type: 'stream', stream }); } - /** - * Reads from the compressed stream only far enough to decide whether its total length exceeds - * `n`, stopping early the moment it does. Avoids buffering (and waiting for) the whole stream just - * to learn it is too large to inline. - * - * Returns true if the stream is longer than `n` bytes. On a `false` result the stream was shorter, - * so it is now fully `collected`; on a `true` result the read prefix is retained in a `partial` - * state and the rest can still be streamed via {@link stream} or drained via {@link collect}. - * Intended to be called once, before {@link collect}/{@link stream}. - */ - async lengthLargerThan(n: number): Promise { - switch (this.state.type) { - case 'collected': - return this.state.bytes.byteLength > n; - case 'partial': - // Already read past `n` to get here (that's the only way we enter this state). - return true; - case 'stream': { - const reader = this.state.stream.getReader(); - const chunks: Array = []; - let total = 0; - let exceeded = false; - while (true) { - const { done, value } = await reader.read(); - if (done) { - break; - } - chunks.push(value); - total += value.byteLength; - if (total > n) { - exceeded = true; - break; - } - } - // Merge whatever we've read into one contiguous buffer. - const buffered = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - buffered.set(chunk, offset); - offset += chunk.byteLength; - } - if (exceeded) { - this.state = { type: 'partial', buffered, reader }; - return true; - } - reader.releaseLock(); - this.state = { type: 'collected', bytes: buffered }; - return buffered.byteLength > n; - } - } - } - /** Collect data from the stream into memory and return as a Uint8Array. */ async collect() { switch (this.state.type) { @@ -660,48 +587,18 @@ class CompressedStreamState { const bytes = await collect(this.state.stream); this.state = { type: 'collected', bytes }; return bytes; - case 'partial': { - // Drain the reconstructed stream (buffered prefix + rest of the reader) in one pass. - const collectedBytes = await collect(this.stream()); - this.state = { type: 'collected', bytes: collectedBytes }; - return collectedBytes; - } case 'collected': return this.state.bytes; } } /** Pass wrapped stream through to downstream consumer. */ - stream(): ReadableStream { + stream() { switch (this.state.type) { case 'stream': return this.state.stream; case 'collected': return readableFromBytes(this.state.bytes); - case 'partial': { - // Emit the already-read prefix as one chunk, then resume pumping the retained reader. - const { buffered, reader } = this.state; - let prefixSent = false; - return new ReadableStream({ - async pull(controller) { - if (!prefixSent) { - prefixSent = true; - controller.enqueue(buffered); - return; - } - const { done, value } = await reader.read(); - if (done) { - reader.releaseLock(); - controller.close(); - } else { - controller.enqueue(value); - } - }, - cancel(reason) { - return reader.cancel(reason); - }, - }); - } } } } From 9dca376a2542a1bba3a1a0e06f5c5014f4238af8 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Tue, 7 Jul 2026 14:48:19 -0400 Subject: [PATCH 56/68] refactor: rename CompressedStreamState -> MaybeCollectedStream --- .../data-stream/outgoing/OutgoingDataStreamManager.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index e4a3b6823f..f0d97b39dc 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -104,7 +104,7 @@ export default class OutgoingDataStreamManager { this.allRecipientsSupportV2(options?.destinationIdentities) && this.allRecipientsSupportCompression(options?.destinationIdentities); let compressedStream = compressEligible - ? CompressedStreamState.fromStream( + ? MaybeCollectedStream.fromStream( readableFromBytes(textInBytes).pipeThrough(deflateRawTransform()), ) : null; @@ -229,9 +229,7 @@ export default class OutgoingDataStreamManager { this.allRecipientsSupportV2(destinationIdentities) && this.allRecipientsSupportCompression(destinationIdentities); let compressedStream = compressEligible - ? CompressedStreamState.fromStream( - readableFromBytes(bytes).pipeThrough(deflateRawTransform()), - ) + ? MaybeCollectedStream.fromStream(readableFromBytes(bytes).pipeThrough(deflateRawTransform())) : null; // Phase 1: Try to send as a single packet data stream @@ -567,7 +565,7 @@ export default class OutgoingDataStreamManager { * through} to the downstream consumer without ever being fully buffered. Once collected, later * calls reuse the buffered bytes rather than re-collecting. */ -class CompressedStreamState { +class MaybeCollectedStream { private state: | { type: 'stream'; stream: ReadableStream } | { type: 'collected'; bytes: Uint8Array }; @@ -577,7 +575,7 @@ class CompressedStreamState { } static fromStream(stream: ReadableStream) { - return new CompressedStreamState({ type: 'stream', stream }); + return new MaybeCollectedStream({ type: 'stream', stream }); } /** Collect data from the stream into memory and return as a Uint8Array. */ From 4a6e2bb36ca87f5c8f20a99dd7c7333619b12159 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 17 Jul 2026 10:09:55 -0400 Subject: [PATCH 57/68] feat: adjust progress event to report results more reliably in all contexts It's still not perfect - the progress events are reported pre-compressor right now so the compressor lag isn't taken into account directly (other than the side effect of it on future data "pulls" through the stream). But I think it's an improvement over the previous state of just 0 -> 1. --- src/room/data-stream/incoming/StreamReader.ts | 6 ++ .../outgoing/OutgoingDataStreamManager.ts | 79 ++++++++++++++++--- 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/src/room/data-stream/incoming/StreamReader.ts b/src/room/data-stream/incoming/StreamReader.ts index 9057c22772..bd700285f0 100644 --- a/src/room/data-stream/incoming/StreamReader.ts +++ b/src/room/data-stream/incoming/StreamReader.ts @@ -105,6 +105,9 @@ export class ByteStreamReader extends BaseStreamReader { ); if (result.done) { this.validateBytesReceived(true); + if (typeof this.totalByteSize === 'number') { + this.onProgress?.(1); + } return { done: true, value: undefined as any }; } else { this.handleChunkReceived(result.value); @@ -233,6 +236,9 @@ export class TextStreamReader extends BaseStreamReader { ); if (result.done) { this.validateBytesReceived(true); + if (typeof this.totalByteSize === 'number') { + this.onProgress?.(1); + } return { done: true, value: undefined }; } else { this.handleChunkReceived(result.value); diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index f0d97b39dc..03f7493869 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -138,12 +138,15 @@ export default class OutgoingDataStreamManager { const fileIds = options?.attachments?.map(() => crypto.randomUUID()); - const progresses = new Array(fileIds ? fileIds.length + 1 : 1).fill(0); + // Progress is split evenly across the text part (slot 0) and one slot per attachment, then + // normalized to a [0,1] fraction. Each slot climbs monotonically to 1, so the aggregate ends at + // exactly 1 once every part has completed. + const parts = fileIds ? fileIds.length + 1 : 1; + const progresses = new Array(parts).fill(0); const handleProgress = (progress: number, idx: number) => { progresses[idx] = progress; - const totalProgress = progresses.reduce((acc, val) => acc + val, 0); - options?.onProgress?.(totalProgress); + options?.onProgress?.(progresses.reduce((acc, val) => acc + val, 0) / parts); }; // Phase 2: Try to send a multi packet data stream with compressed bytes @@ -158,11 +161,17 @@ export default class OutgoingDataStreamManager { packet, streamId, options?.destinationIdentities, - compressedStream.stream(), + compressedStream + .stream() + .pipeThrough( + progressReportingStream(textInBytes.length, (progress) => handleProgress(progress, 0)), + ), ); - // set text part of progress to 1 - handleProgress(1, 0); + // Ensure there's always a 100% progress event fired, even if the string is zero bytes long + if (textInBytes.length === 0) { + handleProgress(1, 0); + } } else { // Phase 3 / fallback: header + plain uncompressed chunk packets + trailer. const writer = await this.streamText({ @@ -223,13 +232,19 @@ export default class OutgoingDataStreamManager { : Encryption_Type.NONE, }; + const progressMonitorTap = progressReportingStream(bytes.length, options?.onProgress); + const compressEligible = compress && isCompressionStreamSupported() && this.allRecipientsSupportV2(destinationIdentities) && this.allRecipientsSupportCompression(destinationIdentities); let compressedStream = compressEligible - ? MaybeCollectedStream.fromStream(readableFromBytes(bytes).pipeThrough(deflateRawTransform())) + ? MaybeCollectedStream.fromStream( + readableFromBytes(bytes) + .pipeThrough(progressMonitorTap) + .pipeThrough(deflateRawTransform()), + ) : null; // Phase 1: Try to send as a single packet data stream @@ -265,9 +280,15 @@ export default class OutgoingDataStreamManager { : DataStream_CompressionType.NONE, }); const packet = createStreamHeaderPacket(header, destinationIdentities); - const source = compressedStream ? compressedStream.stream() : readableFromBytes(bytes); + const source = compressedStream + ? compressedStream.stream() + : readableFromBytes(bytes).pipeThrough(progressMonitorTap); await this.sendChunkedByteStream(packet, streamId, destinationIdentities, source); - options?.onProgress?.(1); + + // Ensure there's always a 100% progress event fired, even if the buffer is zero bytes long + if (bytes.length === 0) { + options?.onProgress?.(1); + } return info; } @@ -470,9 +491,18 @@ export default class OutgoingDataStreamManager { : DataStream_CompressionType.NONE, }); const packet = createStreamHeaderPacket(header, destinationIdentities); - const source = compress ? file.stream().pipeThrough(deflateRawTransform()) : file.stream(); + + const tapped = file + .stream() + .pipeThrough(progressReportingStream(file.size, options?.onProgress)); + const source = compress ? tapped.pipeThrough(deflateRawTransform()) : tapped; await this.sendChunkedByteStream(packet, streamId, destinationIdentities, source); + // Ensure there's always a 100% progress event fired, even if the file is zero bytes long + if (file.size === 0) { + options?.onProgress?.(1); + } + return info; } @@ -601,6 +631,35 @@ class MaybeCollectedStream { } } +/** + * A pass-through byte transform that reports progress as bytes flow through it, measured against + * `totalLength`. Used as a "tap" to instrument the source of a chunked send without threading a + * progress callback through the send primitive. + * + * IMPORTANT: This should be upstream of any compression step. + * + * Emits nothing when the total is unknown or zero. + */ +function progressReportingStream( + totalPreCompressionLength: number | undefined, + onProgress?: (progress: number) => void, +): ReadableWritablePair { + let sent = 0; + return new TransformStream({ + transform(chunk, controller) { + sent += chunk.byteLength; + if ( + onProgress && + typeof totalPreCompressionLength === 'number' && + totalPreCompressionLength > 0 + ) { + onProgress(Math.min(sent / totalPreCompressionLength, 1)); + } + controller.enqueue(chunk); + }, + }); +} + /** * Sends a stream `streamHeader` packet, enforcing that it fits the MTU budget. The header carries * the user attributes (plus topic/framing), and a single `DataPacket` larger than the MTU can't be From 3d3e26ca490caf0952c53bcbf595df8d0d1c8833 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 17 Jul 2026 13:37:57 -0400 Subject: [PATCH 58/68] fix: address flipped stream controllers check --- src/room/data-stream/incoming/IncomingDataStreamManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 1e22087587..e0e9786303 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -209,7 +209,7 @@ export default class IncomingDataStreamManager { start: (controller) => { streamController = controller; - if (this.textStreamControllers.has(streamHeader.streamId)) { + if (this.byteStreamControllers.has(streamHeader.streamId)) { throw new DataStreamError( `A data stream read is already in progress for a stream with id ${streamHeader.streamId}.`, DataStreamErrorReason.AlreadyOpened, From f45fadcdaccf1ff710e12f4431d71005b23a6086 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 17 Jul 2026 13:38:37 -0400 Subject: [PATCH 59/68] fix: make sure textStreamControllers always cleaned up --- src/room/data-stream/incoming/IncomingDataStreamManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index e0e9786303..452626554e 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -382,8 +382,8 @@ export default class IncomingDataStreamManager { } else { textBuffer.info.attributes = { ...textBuffer.info.attributes, ...trailer.attributes }; textBuffer.controller.close(); - this.textStreamControllers.delete(trailer.streamId); } + this.textStreamControllers.delete(trailer.streamId); } const fileBuffer = this.byteStreamControllers.get(trailer.streamId); From 2c23c340b7718ec486587ffd1622f0432e0e22a2 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Fri, 17 Jul 2026 13:38:53 -0400 Subject: [PATCH 60/68] fix: ensure destinationIdentities / compress forwarded along to attached streams --- src/room/data-stream/outgoing/OutgoingDataStreamManager.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index 03f7493869..a55b697f6e 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -197,6 +197,8 @@ export default class OutgoingDataStreamManager { this._sendFile(fileIds[idx], file, { topic: options.topic, mimeType: file.type, + destinationIdentities: options.destinationIdentities, + compress: options.compress, onProgress: (progress) => { handleProgress(progress, idx + 1); }, From fc53b2394db519b6e1c3ffb703d5aae59ae8a5be Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 20 Jul 2026 13:07:53 -0400 Subject: [PATCH 61/68] feat: add max payload byte size limit to data streams This is set fairly high and will prevent any malicious decompression bomb style payloads --- src/room/data-stream/compression.ts | 39 +++++++++++--- src/room/data-stream/constants.ts | 10 ++++ .../incoming/IncomingDataStreamManager.ts | 54 +++++++++++++++++-- src/room/errors.ts | 3 ++ 4 files changed, 95 insertions(+), 11 deletions(-) diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index e6ba20e9d5..5b36558f5c 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -16,6 +16,7 @@ * * @internal */ +import { DataStreamError, DataStreamErrorReason } from '../errors'; /** * A `deflate-raw` compression transform (inverse of {@link inflateRawTransform}): pipe a byte stream @@ -53,17 +54,34 @@ export async function deflateRawCompress(data: Uint8Array): Promise return collect(cs.readable); } -/** Decompresses a raw-deflate byte array in full (inverse of {@link deflateRawCompress}). */ -export async function deflateRawDecompress(data: Uint8Array): Promise { +/** + * Decompresses a raw-deflate byte array in full (inverse of {@link deflateRawCompress}). + * `maxByteLength` bounds the decompressed output (decompression-bomb guard); exceeding it rejects + * with a `PayloadTooLarge` error. + */ +export async function deflateRawDecompress( + data: Uint8Array, + maxByteLength?: number, +): Promise { const ds = new DecompressionStream('deflate-raw'); const writer = ds.writable.getWriter(); - writer.write(data as NonSharedUint8Array); - writer.close(); - return collect(ds.readable); + // The writer promises are intentionally not awaited (output is consumed via `collect`), but + // they reject when the byte cap cancels the readable mid-stream — swallow that so an enforced + // cap doesn't surface as an unhandled rejection. + writer.write(data as NonSharedUint8Array).catch(() => {}); + writer.close().catch(() => {}); + return collect(ds.readable, maxByteLength); } -/** Drains a byte stream, concatenating all of its chunks into a single array. */ -export async function collect(stream: ReadableStream): Promise { +/** + * Drains a byte stream, concatenating all of its chunks into a single array. When + * `maxByteLength` is given, drops the stream and throws `PayloadTooLarge` as soon as the + * accumulated output exceeds it. + */ +export async function collect( + stream: ReadableStream, + maxByteLength?: number, +): Promise { const reader = stream.getReader(); const chunks: Uint8Array[] = []; let total = 0; @@ -74,6 +92,13 @@ export async function collect(stream: ReadableStream): Promise maxByteLength) { + await reader.cancel(); + throw new DataStreamError( + `Decompressed payload exceeds the maximum payload size of ${maxByteLength} bytes`, + DataStreamErrorReason.PayloadTooLarge, + ); + } } const result = new Uint8Array(total); let offset = 0; diff --git a/src/room/data-stream/constants.ts b/src/room/data-stream/constants.ts index 4c072f3eb0..5a93c6da64 100644 --- a/src/room/data-stream/constants.ts +++ b/src/room/data-stream/constants.ts @@ -6,3 +6,13 @@ * @internal */ export const STREAM_CHUNK_SIZE_BYTES = 15_000; + +/** + * Default cap on the number of decompressed bytes a single incoming compressed data stream may + * produce (5 GB). A tiny compressed payload can inflate to an arbitrarily large output + * (decompression bomb), so the decompressor's output is bounded rather than trusting the wire + * size; streams exceeding the cap error with `PayloadTooLarge`. + * + * @internal + */ +export const DEFAULT_MAX_PAYLOAD_BYTE_LENGTH = 5_000_000_000; diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 452626554e..b83e161195 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -11,6 +11,7 @@ import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { type ByteStreamInfo, type StreamController, type TextStreamInfo } from '../../types'; import { bigIntToNumber, isCompressionStreamSupported, numberToBigInt } from '../../utils'; import { deflateRawDecompress, inflateRawTransform } from '../compression'; +import { DEFAULT_MAX_PAYLOAD_BYTE_LENGTH } from '../constants'; import { type ByteStreamHandler, ByteStreamReader, @@ -21,6 +22,14 @@ import { export default class IncomingDataStreamManager { private log = log; + /** Max number of decompressed bytes an incoming compressed stream may produce before it is + * errored (decompression-bomb guard). */ + private maxPayloadByteLength: number; + + constructor(maxPayloadByteLength: number = DEFAULT_MAX_PAYLOAD_BYTE_LENGTH) { + this.maxPayloadByteLength = maxPayloadByteLength; + } + private byteStreamControllers = new Map>(); private textStreamControllers = new Map>(); @@ -196,7 +205,9 @@ export default class IncomingDataStreamManager { info, createInlineStream( streamHeader.streamId, - compressed ? deflateRawDecompress(inlineContent) : inlineContent, + compressed + ? deflateRawDecompress(inlineContent, this.maxPayloadByteLength) + : inlineContent, ), bigIntToNumber(streamHeader.totalLength), ), @@ -227,7 +238,9 @@ export default class IncomingDataStreamManager { streamHandlerCallback( new ByteStreamReader( info, - compressed ? inflateRawByteChunkStream(stream, streamHeader.streamId) : stream, + compressed + ? inflateRawByteChunkStream(stream, streamHeader.streamId, this.maxPayloadByteLength) + : stream, // `totalLength` is the pre-compression size, and the reader counts decompressed bytes, // so it applies to both paths (mirrors text). bigIntToNumber(streamHeader.totalLength), @@ -292,7 +305,9 @@ export default class IncomingDataStreamManager { const inlineContent = streamHeader.inlineContent; if (typeof inlineContent !== 'undefined') { // Inline text is the raw UTF-8 payload, optionally deflate-raw compressed. - const content = compressed ? deflateRawDecompress(inlineContent) : inlineContent; + const content = compressed + ? deflateRawDecompress(inlineContent, this.maxPayloadByteLength) + : inlineContent; streamHandlerCallback( new TextStreamReader( info, @@ -326,7 +341,9 @@ export default class IncomingDataStreamManager { streamHandlerCallback( new TextStreamReader( info, - compressed ? inflateRawChunkStream(stream, streamHeader.streamId) : stream, + compressed + ? inflateRawChunkStream(stream, streamHeader.streamId, this.maxPayloadByteLength) + : stream, // `totalLength` is the pre-compression size, and the reader sees decompressed bytes, so // it applies to both paths. bigIntToNumber(streamHeader.totalLength), @@ -553,11 +570,13 @@ function bytesToDecodedUtf8(streamId: string): TransformStream, streamId: string, + maxPayloadByteLength: number, ): ReadableStream { return raw .pipeThrough(ensureOrderedChunks(streamId)) .pipeThrough(chunksToBytes()) .pipeThrough(inflateRawTransform()) + .pipeThrough(maxDecompressedLengthGuard(streamId, maxPayloadByteLength)) .pipeThrough(bytesToChunks(streamId)); } @@ -571,10 +590,37 @@ function inflateRawByteChunkStream( function inflateRawChunkStream( raw: ReadableStream, streamId: string, + maxPayloadByteLength: number, ): ReadableStream { return raw .pipeThrough(ensureOrderedChunks(streamId)) .pipeThrough(chunksToBytes()) .pipeThrough(inflateRawTransform()) + .pipeThrough(maxDecompressedLengthGuard(streamId, maxPayloadByteLength)) .pipeThrough(bytesToDecodedUtf8(streamId)); } + +/** + * Caps the total decompressed byte count of a compressed stream (decompression-bomb guard): a + * tiny compressed payload can inflate to an arbitrarily large output, so the decompressor's + * output is bounded rather than trusting the wire size. Exceeding the cap errors the stream with + * `PayloadTooLarge`. + */ +function maxDecompressedLengthGuard( + streamId: string, + maxByteLength: number, +): TransformStream { + let total = 0; + return new TransformStream({ + transform: (value, controller) => { + total += value.byteLength; + if (total > maxByteLength) { + throw new DataStreamError( + `Data stream ${streamId} exceeds the maximum payload size of ${maxByteLength} bytes`, + DataStreamErrorReason.PayloadTooLarge, + ); + } + controller.enqueue(value); + }, + }); +} diff --git a/src/room/errors.ts b/src/room/errors.ts index 666afe9507..087cc6f9a8 100644 --- a/src/room/errors.ts +++ b/src/room/errors.ts @@ -290,6 +290,9 @@ export enum DataStreamErrorReason { // The serialized stream header packet (driven mainly by attributes) exceeds the MTU budget. HeaderTooLarge = 9, + + // The stream's (decompressed) payload exceeds the maximum allowed size. + PayloadTooLarge = 10, } export class DataStreamError extends LivekitReasonedError { From 52734f18725e93c321964735dd3d22285f1dfc23 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 20 Jul 2026 13:09:47 -0400 Subject: [PATCH 62/68] fix: surface an abnormal data stream trailer close reason as an AbnormalEnd --- .../incoming/IncomingDataStreamManager.ts | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index b83e161195..572c1632f2 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -398,7 +398,18 @@ export default class IncomingDataStreamManager { ); } else { textBuffer.info.attributes = { ...textBuffer.info.attributes, ...trailer.attributes }; - textBuffer.controller.close(); + if (trailer.reason) { + // A non-empty reason marks an abnormal close by the sender (e.g. an aborted send); + // surface it as an error rather than pretending the stream completed. + textBuffer.controller.error( + new DataStreamError( + `Data stream ${trailer.streamId} closed abnormally: ${trailer.reason}`, + DataStreamErrorReason.AbnormalEnd, + ), + ); + } else { + textBuffer.controller.close(); + } } this.textStreamControllers.delete(trailer.streamId); } @@ -414,7 +425,18 @@ export default class IncomingDataStreamManager { ); } else { fileBuffer.info.attributes = { ...fileBuffer.info.attributes, ...trailer.attributes }; - fileBuffer.controller.close(); + if (trailer.reason) { + // A non-empty reason marks an abnormal close by the sender (e.g. an aborted send); + // surface it as an error rather than pretending the stream completed. + fileBuffer.controller.error( + new DataStreamError( + `Data stream ${trailer.streamId} closed abnormally: ${trailer.reason}`, + DataStreamErrorReason.AbnormalEnd, + ), + ); + } else { + fileBuffer.controller.close(); + } } this.byteStreamControllers.delete(trailer.streamId); } From 902b960df005667922b2eb818615461d51893ab0 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 20 Jul 2026 13:12:16 -0400 Subject: [PATCH 63/68] feat: add new test cases for incoming / outgoing data stream handling --- .../IncomingDataStreamManager.test.ts | 565 +++++++++++++++++- .../OutgoingDataStreamManager.test.ts | 335 +++++++++++ 2 files changed, 896 insertions(+), 4 deletions(-) diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts index 73df5988d7..672bdaaab5 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.test.ts @@ -23,6 +23,90 @@ function randomText(length: number): string { return s.slice(0, length); } +/** Fills a buffer with uniform random bytes — genuinely incompressible. */ +function randomBytes(length: number): Uint8Array { + const out = new Uint8Array(length); + // crypto.getRandomValues rejects requests over 65536 bytes, so chunk it. + for (let offset = 0; offset < length; offset += 65536) { + crypto.getRandomValues(out.subarray(offset, offset + 65536)); + } + return out; +} + +type HeaderFields = NonNullable[0]>; + +/** Wraps a `DataStream_Header` (from `alice`, topic `my-topic`) in a `DataPacket`. */ +function headerPacket( + streamId: string, + contentCase: 'textHeader' | 'byteHeader', + fields: HeaderFields = {}, +): DataPacket { + return new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamHeader', + value: new DataStream_Header({ + streamId, + topic: 'my-topic', + mimeType: 'text/plain', + timestamp: 0n, + contentHeader: + contentCase === 'textHeader' + ? { case: 'textHeader', value: new DataStream_TextHeader({}) } + : { case: 'byteHeader', value: new DataStream_ByteHeader({}) }, + ...fields, + }), + }, + }); +} + +/** Wraps a `DataStream_Chunk` in a `DataPacket`. */ +function chunkPacket( + streamId: string, + chunkIndex: number, + content: Uint8Array, + version = 0, +): DataPacket { + return new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamChunk', + value: new DataStream_Chunk({ + streamId, + chunkIndex: BigInt(chunkIndex), + content, + version, + }), + }, + }); +} + +/** Wraps a `DataStream_Trailer` in a `DataPacket`. */ +function trailerPacket( + streamId: string, + attributes?: Record, + reason?: string, +): DataPacket { + return new DataPacket({ + participantIdentity: 'alice', + value: { + case: 'streamTrailer', + value: new DataStream_Trailer({ streamId, attributes, reason }), + }, + }); +} + +/** Concatenates the byte chunks a `ByteStreamReader.readAll()` resolves with. */ +function concatChunks(chunks: Array): Uint8Array { + const out = new Uint8Array(chunks.reduce((acc, chunk) => acc + chunk.byteLength, 0)); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.byteLength; + } + return out; +} + describe('IncomingDataStreamManager', () => { describe('Receiving v1 data streams', () => { it('should receive a v1 text data stream', async () => { @@ -913,8 +997,8 @@ describe('IncomingDataStreamManager', () => { const manager = new IncomingDataStreamManager(); manager.setConnected(true); - const readerPromise = new Promise((resolve) => { - manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + const readerPromise = new Promise((resolve) => { + manager.registerByteStreamHandler('my-topic', (reader) => resolve(reader)); }); const streamId = crypto.randomUUID(); @@ -927,11 +1011,11 @@ describe('IncomingDataStreamManager', () => { value: new DataStream_Header({ streamId, topic: 'my-topic', - mimeType: 'text/plain', + mimeType: 'application/octet-stream', timestamp: 0n, totalLength: BigInt(bytes.length), compression: DataStream_CompressionType.DEFLATE_RAW, - contentHeader: { case: 'textHeader', value: new DataStream_TextHeader({}) }, + contentHeader: { case: 'byteHeader', value: new DataStream_ByteHeader({}) }, }), }, }), @@ -1040,5 +1124,478 @@ describe('IncomingDataStreamManager', () => { expect(await reader.readAll()).toStrictEqual([new Uint8Array(0)]); expect(reader.info.attributes?.foo).toStrictEqual('bar'); }); + + it('should receive a v2 MULTI PACKET + COMPRESSED byte data stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerByteStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + // Incompressible, so the compressed form stays large enough to need two chunks (but fewer + // than three). + const bytes = randomBytes(20_000); + const compressed = await deflateRawCompress(bytes); + expect(compressed.length).toBeGreaterThan(STREAM_CHUNK_SIZE_BYTES); + expect(compressed.length).toBeLessThan(2 * STREAM_CHUNK_SIZE_BYTES); + + manager.handleDataStreamPacket( + headerPacket(streamId, 'byteHeader', { + totalLength: BigInt(bytes.length), + compression: DataStream_CompressionType.DEFLATE_RAW, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + chunkPacket(streamId, 0, compressed.slice(0, STREAM_CHUNK_SIZE_BYTES)), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + chunkPacket(streamId, 1, compressed.slice(STREAM_CHUNK_SIZE_BYTES)), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket(trailerPacket(streamId), Encryption_Type.NONE); + + const reader = await readerPromise; + expect(concatChunks(await reader.readAll())).toStrictEqual(bytes); + }); + + it('should drop a duplicate chunk index on a compressed stream and still decode', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = new Array(30) + .fill(null) + .map(() => `hello world${randomText(1_000)}`) + .join(''); + const compressed = await deflateRawCompress(new TextEncoder().encode(text)); + expect(compressed.length).toBeGreaterThan(STREAM_CHUNK_SIZE_BYTES); + expect(compressed.length).toBeLessThan(2 * STREAM_CHUNK_SIZE_BYTES); + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { + totalLength: BigInt(text.length), + compression: DataStream_CompressionType.DEFLATE_RAW, + }), + Encryption_Type.NONE, + ); + const chunk0 = chunkPacket(streamId, 0, compressed.slice(0, STREAM_CHUNK_SIZE_BYTES)); + manager.handleDataStreamPacket(chunk0, Encryption_Type.NONE); + // A replayed chunk (e.g. reconnect logic) must be dropped with a warning, not fed to the + // stateful decompressor a second time. + manager.handleDataStreamPacket(chunk0, Encryption_Type.NONE); + manager.handleDataStreamPacket( + chunkPacket(streamId, 1, compressed.slice(STREAM_CHUNK_SIZE_BYTES)), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket(trailerPacket(streamId), Encryption_Type.NONE); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual(text); + }); + + it('should error on a gap in chunk indices on a compressed stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = new Array(30) + .fill(null) + .map(() => `hello world${randomText(1_000)}`) + .join(''); + const compressed = await deflateRawCompress(new TextEncoder().encode(text)); + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { + totalLength: BigInt(text.length), + compression: DataStream_CompressionType.DEFLATE_RAW, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + chunkPacket(streamId, 0, compressed.slice(0, STREAM_CHUNK_SIZE_BYTES)), + Encryption_Type.NONE, + ); + // Skip chunk index 1 entirely — the stateful decompressor cannot continue past a gap. + manager.handleDataStreamPacket( + chunkPacket(streamId, 2, compressed.slice(STREAM_CHUNK_SIZE_BYTES)), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + await expect(reader.readAll()).rejects.toThrow('Missing chunk(s)'); + }); + + it('should reframe multibyte UTF-8 on chunk boundaries when decompressing a text stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = '😀你好世界 café — ¡ñandú! '.repeat(500); + const textBytes = new TextEncoder().encode(text); + const compressed = await deflateRawCompress(textBytes); + + // Split the compressed bytes at an arbitrary midpoint: the decompressor's output at the + // seam can land mid-codepoint, exercising the UTF-8 reframing stage. + const split = Math.floor(compressed.length / 2); + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { + totalLength: BigInt(textBytes.length), // NOTE: byte length, not character count + compression: DataStream_CompressionType.DEFLATE_RAW, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + chunkPacket(streamId, 0, compressed.slice(0, split)), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + chunkPacket(streamId, 1, compressed.slice(split)), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket(trailerPacket(streamId), Encryption_Type.NONE); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual(text); + }); + + it('should error when a compressed stream decompresses to fewer bytes than totalLength', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const compressed = await deflateRawCompress(new TextEncoder().encode('hello world')); // 11 bytes + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { + totalLength: 16n, // more than the decompressed payload + compression: DataStream_CompressionType.DEFLATE_RAW, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket(chunkPacket(streamId, 0, compressed), Encryption_Type.NONE); + manager.handleDataStreamPacket(trailerPacket(streamId), Encryption_Type.NONE); + + // The reader counts DECOMPRESSED bytes against totalLength. + const reader = await readerPromise; + await expect(reader.readAll()).rejects.toThrow('Not enough chunk(s)'); + }); + + it('should error when a compressed stream decompresses to more bytes than totalLength', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const compressed = await deflateRawCompress(new TextEncoder().encode('hello world')); // 11 bytes + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { + totalLength: 5n, // fewer than the decompressed payload + compression: DataStream_CompressionType.DEFLATE_RAW, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket(chunkPacket(streamId, 0, compressed), Encryption_Type.NONE); + manager.handleDataStreamPacket(trailerPacket(streamId), Encryption_Type.NONE); + + const reader = await readerPromise; + await expect(reader.readAll()).rejects.toThrow('Extra chunk(s)'); + }); + + it('should merge trailer attributes on a compressed stream', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = 'hello world'; + const compressed = await deflateRawCompress(new TextEncoder().encode(text)); + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { + totalLength: BigInt(text.length), + compression: DataStream_CompressionType.DEFLATE_RAW, + attributes: { foo: 'bar', baz: 'quux' }, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket(chunkPacket(streamId, 0, compressed), Encryption_Type.NONE); + manager.handleDataStreamPacket( + trailerPacket(streamId, { hello: 'world', foo: 'updated' }), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual(text); + expect(reader.info.attributes?.baz).toStrictEqual('quux'); + expect(reader.info.attributes?.hello).toStrictEqual('world'); + expect(reader.info.attributes?.foo).toStrictEqual('updated'); + }); + + it('should ignore a stream with an unknown compression type', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = 'hello world'; + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { + totalLength: BigInt(text.length), + compression: 99 as DataStream_CompressionType, + }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + chunkPacket(streamId, 0, new TextEncoder().encode(text)), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket(trailerPacket(streamId), Encryption_Type.NONE); + + // A compression type from a future protocol version can't be decoded; the stream is + // dropped and the handler is never invoked. + await expect( + Promise.race([readerPromise, Promise.resolve('still pending')]), + ).resolves.toStrictEqual('still pending'); + }); + }); + + describe('Receive-side protocol edge cases', () => { + it('should reject a duplicate TEXT streamId whose stream is already open', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readers: Array = []; + manager.registerTextStreamHandler('my-topic', (reader) => readers.push(reader)); + + const streamId = crypto.randomUUID(); + const header = headerPacket(streamId, 'textHeader', { totalLength: 11n }); + + manager.handleDataStreamPacket(header, Encryption_Type.NONE); + expect(readers).toHaveLength(1); + + // A second header re-using an open streamId must be rejected, not silently replace (or + // corrupt) the in-flight stream. + expect(() => manager.handleDataStreamPacket(header, Encryption_Type.NONE)).toThrow( + 'already in progress', + ); + expect(readers).toHaveLength(1); + }); + + it('should reject a duplicate BYTE streamId whose stream is already open', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readers: Array = []; + manager.registerByteStreamHandler('my-topic', (reader) => readers.push(reader)); + + const streamId = crypto.randomUUID(); + const header = headerPacket(streamId, 'byteHeader', { totalLength: 4n }); + + manager.handleDataStreamPacket(header, Encryption_Type.NONE); + expect(readers).toHaveLength(1); + + expect(() => manager.handleDataStreamPacket(header, Encryption_Type.NONE)).toThrow( + 'already in progress', + ); + expect(readers).toHaveLength(1); + }); + + it('should ignore empty chunks', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = 'hello world'; + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { totalLength: BigInt(text.length) }), + Encryption_Type.NONE, + ); + // An empty chunk must not count against totalLength or corrupt the stream. + manager.handleDataStreamPacket( + chunkPacket(streamId, 0, new Uint8Array(0)), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + chunkPacket(streamId, 1, new TextEncoder().encode(text)), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket(trailerPacket(streamId), Encryption_Type.NONE); + + const reader = await readerPromise; + expect(await reader.readAll()).toStrictEqual(text); + }); + + it('should silently drop packets for a topic with no registered handler', () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const streamId = crypto.randomUUID(); + + // No handler registered for the topic: header, chunks, and trailer are all dropped + // without raising. + expect(() => { + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { totalLength: 11n }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + chunkPacket(streamId, 0, new TextEncoder().encode('hello world')), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket(trailerPacket(streamId), Encryption_Type.NONE); + }).not.toThrow(); + }); + + it('should error the reader when the trailer reports an abnormal close reason', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = 'hello world'; + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { totalLength: BigInt(text.length) }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + chunkPacket(streamId, 0, new TextEncoder().encode(text)), + Encryption_Type.NONE, + ); + // A non-empty trailer reason means the sender aborted; the stream must not be reported as + // a successful close. + manager.handleDataStreamPacket( + trailerPacket(streamId, undefined, 'cancelled'), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + await expect(reader.readAll()).rejects.toThrow('closed abnormally: cancelled'); + }); + + it('should error an inline compressed payload that inflates past the max payload size', async () => { + // Decompression-bomb guard: a tiny compressed inline payload must not be allowed to + // expand without bound. + const manager = new IncomingDataStreamManager(1_000); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = randomText(50_000); + const textBytes = new TextEncoder().encode(text); + const compressed = await deflateRawCompress(textBytes); + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { + totalLength: BigInt(textBytes.length), + compression: DataStream_CompressionType.DEFLATE_RAW, + inlineContent: compressed, + }), + Encryption_Type.NONE, + ); + + const reader = await readerPromise; + await expect(reader.readAll()).rejects.toThrow('maximum payload size'); + }); + + it('should error a chunked compressed stream that inflates past the max payload size', async () => { + const manager = new IncomingDataStreamManager(50_000); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = randomText(60_000); // decompresses past the 50k cap + const textBytes = new TextEncoder().encode(text); + const compressed = await deflateRawCompress(textBytes); + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { + totalLength: BigInt(textBytes.length), + compression: DataStream_CompressionType.DEFLATE_RAW, + }), + Encryption_Type.NONE, + ); + for (let offset = 0, index = 0; offset < compressed.length; offset += 15_000, index += 1) { + manager.handleDataStreamPacket( + chunkPacket(streamId, index, compressed.slice(offset, offset + 15_000)), + Encryption_Type.NONE, + ); + } + manager.handleDataStreamPacket(trailerPacket(streamId), Encryption_Type.NONE); + + const reader = await readerPromise; + await expect(reader.readAll()).rejects.toThrow('maximum payload size'); + }); + + it('should error the reader when the trailer has a mismatched encryption type', async () => { + const manager = new IncomingDataStreamManager(); + manager.setConnected(true); + + const readerPromise = new Promise((resolve) => { + manager.registerTextStreamHandler('my-topic', (reader) => resolve(reader)); + }); + + const streamId = crypto.randomUUID(); + const text = 'hello world'; + + manager.handleDataStreamPacket( + headerPacket(streamId, 'textHeader', { totalLength: BigInt(text.length) }), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket( + chunkPacket(streamId, 0, new TextEncoder().encode(text)), + Encryption_Type.NONE, + ); + manager.handleDataStreamPacket(trailerPacket(streamId), Encryption_Type.GCM); + + const reader = await readerPromise; + await expect(reader.readAll()).rejects.toThrow('Encryption type mismatch'); + }); }); }); diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts index fd64cfed04..bb0425d3d2 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.test.ts @@ -1,7 +1,9 @@ import { ClientInfo_Capability, type DataPacket, + type DataStream_ByteHeader, DataStream_CompressionType, + type DataStream_TextHeader, } from '@livekit/protocol'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import log from '../../../logger'; @@ -954,4 +956,337 @@ describe('OutgoingDataStreamManager', () => { }); }); }); + + describe('header size limit (MTU)', () => { + // Attributes large enough that the serialized header packet alone exceeds + // STREAM_CHUNK_SIZE_BYTES (15 000). + const hugeAttributes = { big: 'x'.repeat(20_000) }; + + it('should throw HeaderTooLarge when sending a chunked TEXT stream with oversized attributes (pre-v2 room)', async () => { + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DEFAULT }); + + await expect( + manager.sendText('hello world', { topic: 'my-topic', attributes: hugeAttributes }), + ).rejects.toThrow('exceeds'); + + // The error must be raised before anything hits the wire. + expect(sentPackets).toHaveLength(0); + }); + + it('should throw HeaderTooLarge when oversized attributes force the inline path to fall back (v2 room)', async () => { + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DATA_STREAM_V2 }); + + // The payload is small, but the attributes alone blow the MTU budget: the inline attempt + // falls through gracefully (no throw), and then the chunked header send enforces the hard + // limit — per spec, large *attributes* throw while a large *payload* falls back fine. + await expect( + manager.sendText('hello hello compressible world', { + topic: 'my-topic', + destinationIdentities: ['alice'], + attributes: hugeAttributes, + }), + ).rejects.toThrow('exceeds'); + + expect(sentPackets).toHaveLength(0); + }); + + it('should throw HeaderTooLarge when opening a streamText writer with oversized attributes', async () => { + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DATA_STREAM_V2 }); + + await expect( + manager.streamText({ topic: 'my-topic', attributes: hugeAttributes }), + ).rejects.toThrow('exceeds'); + + expect(sentPackets).toHaveLength(0); + }); + + it('should throw HeaderTooLarge when opening a streamBytes writer with oversized attributes', async () => { + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DATA_STREAM_V2 }); + + await expect( + manager.streamBytes({ topic: 'my-topic', attributes: hugeAttributes }), + ).rejects.toThrow('exceeds'); + + expect(sentPackets).toHaveLength(0); + }); + }); + + describe('attachments', () => { + it('should send attachments as separate byte streams referenced by attachedStreamIds (never inline)', async () => { + const { manager, sentPackets } = createManager({ + alice: CLIENT_PROTOCOL_DATA_STREAM_V2, + bob: CLIENT_PROTOCOL_DATA_STREAM_V2, + }); + const file = new File([new Uint8Array(100).fill(0x07) as NonSharedUint8Array], 'att.bin'); + + const info = await manager.sendText('hello hello compressible world', { + topic: 'my-topic', + destinationIdentities: ['alice', 'bob'], + attachments: [file], + }); + + // The text part must NOT be inline — attachments disable the single-packet path — so both + // the text stream and the attachment stream open with their own headers. + const headers = sentPackets.filter((p) => p.value.case === 'streamHeader'); + expect(headers).toHaveLength(2); + + const textHeader = headerOf(headers[0]); + expect(textHeader.streamId).toStrictEqual(info.id); + expect(textHeader.contentHeader.case).toBe('textHeader'); + expect(textHeader.inlineContent).toBeUndefined(); + const attachedStreamIds = (textHeader.contentHeader.value as DataStream_TextHeader) + .attachedStreamIds; + expect(attachedStreamIds).toHaveLength(1); + + // The attachment is a byte stream whose streamId matches the reference in the text header. + const byteHeader = headerOf(headers[1]); + expect(byteHeader.contentHeader.case).toBe('byteHeader'); + expect(byteHeader.streamId).toStrictEqual(attachedStreamIds[0]); + expect((byteHeader.contentHeader.value as DataStream_ByteHeader).name).toStrictEqual( + 'att.bin', + ); + + // Both streams are terminated by trailers. + const trailers = sentPackets.filter((p) => p.value.case === 'streamTrailer'); + expect(trailers.map((p) => trailerOf(p).streamId).sort()).toStrictEqual( + [info.id, attachedStreamIds[0]].sort(), + ); + }); + + it('should forward destinationIdentities to attachment byte streams', async () => { + // Targeted send to the v2 subset of a mixed room: the attachment streams must be targeted + // the same way, or participants outside the destination list receive stray byte streams + // referenced by a text stream they never got. + const { manager, sentPackets } = createManager({ + alice: CLIENT_PROTOCOL_DEFAULT, + bob: CLIENT_PROTOCOL_DATA_STREAM_V2, + jim: CLIENT_PROTOCOL_DATA_STREAM_V2, + }); + const file = new File([new Uint8Array(100).fill(0x07) as NonSharedUint8Array], 'att.bin'); + + await manager.sendText('hello world', { + topic: 'my-topic', + destinationIdentities: ['bob', 'jim'], + attachments: [file], + }); + + for (const packet of sentPackets) { + expect(packet.destinationIdentities).toStrictEqual(['bob', 'jim']); + } + }); + }); + + describe('recipient eligibility edge cases', () => { + it('should send inline + compressed on a broadcast when every participant is v2 with the capability', async () => { + const { manager, sentPackets } = createManager({ + alice: CLIENT_PROTOCOL_DATA_STREAM_V2, + bob: CLIENT_PROTOCOL_DATA_STREAM_V2, + }); + + // No destinationIdentities: eligibility is evaluated over every remote participant. + await manager.sendText('hello hello compressible world', { topic: 'my-topic' }); + + expect(sentPackets).toHaveLength(1); + const header = headerOf(sentPackets[0]); + expect(header.compression).toBe(DataStream_CompressionType.DEFLATE_RAW); + expect(header.inlineContent).toBeInstanceOf(Uint8Array); + }); + + it('should send inline but uncompressed on a broadcast when one participant lacks the compression capability', async () => { + const { manager, sentPackets } = createManager({ + alice: CLIENT_PROTOCOL_DATA_STREAM_V2, + noCompression: [CLIENT_PROTOCOL_DATA_STREAM_V2, []], + }); + + await manager.sendText('hello hello compressible world', { topic: 'my-topic' }); + + expect(sentPackets).toHaveLength(1); + const header = headerOf(sentPackets[0]); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toStrictEqual( + new TextEncoder().encode('hello hello compressible world'), + ); + }); + + it('should treat an empty room as v2-eligible for a broadcast', async () => { + const { manager, sentPackets } = createManager({}); + + await manager.sendText('hello hello compressible world', { topic: 'my-topic' }); + + // Nobody to receive it, so nothing gates the v2 fast path: single inline compressed packet. + expect(sentPackets).toHaveLength(1); + expect(headerOf(sentPackets[0]).compression).toBe(DataStream_CompressionType.DEFLATE_RAW); + }); + + it('should treat an unknown destination identity as pre-v2 (legacy multi-packet)', async () => { + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DATA_STREAM_V2 }); + + // "stranger" has no ParticipantInfo, so its clientProtocol resolves to 0 — the send must + // fall back to the legacy format rather than assuming v2 support. + await manager.sendText('hello world', { + topic: 'my-topic', + destinationIdentities: ['stranger'], + }); + + expect(sentPackets).toHaveLength(3); + const header = headerOf(sentPackets[0]); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toBeUndefined(); + }); + + it('should send a large TEXT payload to a v2 recipient without the capability as uncompressed MTU-split chunks', async () => { + const { manager, sentPackets } = createManager({ + noCompression: [CLIENT_PROTOCOL_DATA_STREAM_V2, []], + }); + + const info = await manager.sendText('A'.repeat(40_000), { + topic: 'my-topic', + destinationIdentities: ['noCompression'], + }); + + // Inline is attempted (recipient is v2) but the raw payload overflows the MTU, and + // compression is gated off by the missing capability — so this falls back to an + // uncompressed multi-packet stream split at STREAM_CHUNK_SIZE_BYTES. + expect(sentPackets).toHaveLength(5); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toBeUndefined(); + expect(chunkOf(sentPackets[1]).content).toHaveLength(15_000); + expect(chunkOf(sentPackets[2]).content).toHaveLength(15_000); + expect(chunkOf(sentPackets[3]).content).toHaveLength(10_000); + expect(sentPackets[4].value.case).toBe('streamTrailer'); + }); + + it('should send a FILE uncompressed when compress: false even to capable recipients', async () => { + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DATA_STREAM_V2 }); + const bytes = new Uint8Array(10_000).fill(0x07); + + const info = await manager.sendFile(new File([bytes as NonSharedUint8Array], 'text.txt'), { + topic: 'my-topic', + destinationIdentities: ['alice'], + compress: false, + }); + + expect(sentPackets).toHaveLength(3); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + const chunk = chunkOf(sentPackets[1]); + expect(chunk.content).toHaveLength(10_000); + expect(chunk.content.every((byte) => byte === 0x07)).toBeTruthy(); + }); + }); + + describe('UTF-8 chunk splitting', () => { + it('should split streamText writes on UTF-8 character boundaries at the MTU', async () => { + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DEFAULT }); + + // 14 999 single-byte chars put the 4-byte emoji straddling the 15 000-byte MTU boundary. + const text = `${'a'.repeat(14_999)}😀${'b'.repeat(10)}`; + const writer = await manager.streamText({ topic: 'my-topic' }); + await writer.write(text); + await writer.close(); + + expect(sentPackets).toHaveLength(4); // header + 2 chunks + trailer + const first = chunkOf(sentPackets[1]); + const second = chunkOf(sentPackets[2]); + + // The split backs off below the MTU rather than bisecting the emoji, so each chunk decodes + // independently. + expect(first.content).toHaveLength(14_999); + const decode = (bytes: Uint8Array) => new TextDecoder('utf-8', { fatal: true }).decode(bytes); + expect(() => decode(first.content)).not.toThrow(); + expect(() => decode(second.content)).not.toThrow(); + expect(decode(first.content) + decode(second.content)).toStrictEqual(text); + }); + }); + + describe('empty payloads', () => { + it('should send an empty TEXT payload to a pre-v2 room as header + trailer', async () => { + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DEFAULT }); + + const info = await manager.sendText('', { topic: 'my-topic' }); + + // A well-formed (if contentless) stream: header declaring zero length, then the trailer. + expect(sentPackets).toHaveLength(2); + const header = headerOf(sentPackets[0]); + expect(header.streamId).toStrictEqual(info.id); + expect(header.totalLength).toStrictEqual(0n); + expect(sentPackets[1].value.case).toBe('streamTrailer'); + expect(trailerOf(sentPackets[1]).streamId).toStrictEqual(info.id); + }); + + it('should send an empty TEXT payload to a v2 room as a single raw inline packet', async () => { + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DATA_STREAM_V2 }); + + await manager.sendText('', { topic: 'my-topic', destinationIdentities: ['alice'] }); + + expect(sentPackets).toHaveLength(1); + const header = headerOf(sentPackets[0]); + // Deflate framing can't shrink an empty payload, so the raw (empty) bytes are kept. + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toStrictEqual(new Uint8Array(0)); + expect(header.totalLength).toStrictEqual(0n); + }); + + it('should send an empty BYTE payload to a pre-v2 room as header + trailer', async () => { + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DEFAULT }); + + const info = await manager.sendBytes(new Uint8Array(0), { topic: 'my-topic' }); + + expect(sentPackets).toHaveLength(2); + const header = headerOf(sentPackets[0]); + expect(header.totalLength).toStrictEqual(0n); + expect(sentPackets[1].value.case).toBe('streamTrailer'); + expect(trailerOf(sentPackets[1]).streamId).toStrictEqual(info.id); + }); + }); + + describe('runtime without CompressionStream', () => { + it('should send raw inline even to compression-capable recipients when the runtime cannot compress', async () => { + const originalCompressionStream = CompressionStream; + try { + (globalThis as any).CompressionStream = undefined; + + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DATA_STREAM_V2 }); + await manager.sendText('hello hello compressible world', { + topic: 'my-topic', + destinationIdentities: ['alice'], + }); + + // Compression eligibility also requires a local compressor; without one, inline still + // applies but the payload is sent raw. + expect(sentPackets).toHaveLength(1); + const header = headerOf(sentPackets[0]); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(header.inlineContent).toStrictEqual( + new TextEncoder().encode('hello hello compressible world'), + ); + } finally { + (globalThis as any).CompressionStream = originalCompressionStream; + } + }); + + it('should send a FILE uncompressed when the runtime cannot compress', async () => { + const originalCompressionStream = CompressionStream; + try { + (globalThis as any).CompressionStream = undefined; + + const { manager, sentPackets } = createManager({ alice: CLIENT_PROTOCOL_DATA_STREAM_V2 }); + const bytes = new Uint8Array(10_000).fill(0x07); + await manager.sendFile(new File([bytes as NonSharedUint8Array], 'text.txt'), { + topic: 'my-topic', + destinationIdentities: ['alice'], + }); + + expect(sentPackets).toHaveLength(3); + const header = headerOf(sentPackets[0]); + expect(header.compression).toBe(DataStream_CompressionType.NONE); + expect(chunkOf(sentPackets[1]).content).toHaveLength(10_000); + } finally { + (globalThis as any).CompressionStream = originalCompressionStream; + } + }); + }); }); From c8e079042bb6996a51fd251b55415908d060918b Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Mon, 20 Jul 2026 13:32:35 -0400 Subject: [PATCH 64/68] fix: adjust uint8array cast to get ci to pass --- src/room/data-stream/compression.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index 5b36558f5c..c876741183 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -49,7 +49,7 @@ export function inflateRawTransform(): ReadableWritablePair { const cs = new CompressionStream('deflate-raw'); const writer = cs.writable.getWriter(); - writer.write(data as NonSharedUint8Array); + writer.write(data as Uint8Array); writer.close(); return collect(cs.readable); } @@ -68,7 +68,7 @@ export async function deflateRawDecompress( // The writer promises are intentionally not awaited (output is consumed via `collect`), but // they reject when the byte cap cancels the readable mid-stream — swallow that so an enforced // cap doesn't surface as an unhandled rejection. - writer.write(data as NonSharedUint8Array).catch(() => {}); + writer.write(data as Uint8Array).catch(() => {}); writer.close().catch(() => {}); return collect(ds.readable, maxByteLength); } From a0e492190826139105308e49c1fe098c6587f4dc Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 22 Jul 2026 09:43:17 -0400 Subject: [PATCH 65/68] feat: lift NonSharedUint8Array casts further up and use NonSharedUint8Array in all data stream utility functions --- src/room/data-stream/compression.ts | 35 +++++++++++-------- .../incoming/IncomingDataStreamManager.ts | 5 +-- .../outgoing/OutgoingDataStreamManager.ts | 17 ++++----- src/room/utils.ts | 10 +++--- 4 files changed, 38 insertions(+), 29 deletions(-) diff --git a/src/room/data-stream/compression.ts b/src/room/data-stream/compression.ts index c876741183..b733b0fe86 100644 --- a/src/room/data-stream/compression.ts +++ b/src/room/data-stream/compression.ts @@ -16,6 +16,7 @@ * * @internal */ +import { type NonSharedUint8Array } from '../../type-polyfills/non-shared-typed-arrays'; import { DataStreamError, DataStreamErrorReason } from '../errors'; /** @@ -24,10 +25,13 @@ import { DataStreamError, DataStreamErrorReason } from '../errors'; * `sendText`/`sendBytes`/`sendFile` paths, where the full payload is known up front but is streamed * (e.g. from `file.stream()`) rather than held in memory. */ -export function deflateRawTransform(): ReadableWritablePair { +export function deflateRawTransform(): ReadableWritablePair< + NonSharedUint8Array, + NonSharedUint8Array +> { return new CompressionStream('deflate-raw') as unknown as ReadableWritablePair< - Uint8Array, - Uint8Array + NonSharedUint8Array, + NonSharedUint8Array >; } @@ -37,19 +41,22 @@ export function deflateRawTransform(): ReadableWritablePair { +export function inflateRawTransform(): ReadableWritablePair< + NonSharedUint8Array, + NonSharedUint8Array +> { return new DecompressionStream('deflate-raw') as unknown as ReadableWritablePair< - Uint8Array, - Uint8Array + NonSharedUint8Array, + NonSharedUint8Array >; } /** deflate-raw compresses a byte array in full. Use for inline payloads; prefer the streaming * path for the chunked case. */ -export async function deflateRawCompress(data: Uint8Array): Promise { +export async function deflateRawCompress(data: NonSharedUint8Array): Promise { const cs = new CompressionStream('deflate-raw'); const writer = cs.writable.getWriter(); - writer.write(data as Uint8Array); + writer.write(data as NonSharedUint8Array); writer.close(); return collect(cs.readable); } @@ -60,15 +67,15 @@ export async function deflateRawCompress(data: Uint8Array): Promise * with a `PayloadTooLarge` error. */ export async function deflateRawDecompress( - data: Uint8Array, + data: NonSharedUint8Array, maxByteLength?: number, -): Promise { +): Promise { const ds = new DecompressionStream('deflate-raw'); const writer = ds.writable.getWriter(); // The writer promises are intentionally not awaited (output is consumed via `collect`), but // they reject when the byte cap cancels the readable mid-stream — swallow that so an enforced // cap doesn't surface as an unhandled rejection. - writer.write(data as Uint8Array).catch(() => {}); + writer.write(data as NonSharedUint8Array).catch(() => {}); writer.close().catch(() => {}); return collect(ds.readable, maxByteLength); } @@ -79,11 +86,11 @@ export async function deflateRawDecompress( * accumulated output exceeds it. */ export async function collect( - stream: ReadableStream, + stream: ReadableStream, maxByteLength?: number, -): Promise { +): Promise { const reader = stream.getReader(); - const chunks: Uint8Array[] = []; + const chunks: Array = []; let total = 0; while (true) { const { done, value } = await reader.read(); diff --git a/src/room/data-stream/incoming/IncomingDataStreamManager.ts b/src/room/data-stream/incoming/IncomingDataStreamManager.ts index 572c1632f2..5e10b0aae6 100644 --- a/src/room/data-stream/incoming/IncomingDataStreamManager.ts +++ b/src/room/data-stream/incoming/IncomingDataStreamManager.ts @@ -7,6 +7,7 @@ import { Encryption_Type, } from '@livekit/protocol'; import log from '../../../logger'; +import { type NonSharedUint8Array } from '../../../type-polyfills/non-shared-typed-arrays'; import { DataStreamError, DataStreamErrorReason } from '../../errors'; import { type ByteStreamInfo, type StreamController, type TextStreamInfo } from '../../types'; import { bigIntToNumber, isCompressionStreamSupported, numberToBigInt } from '../../utils'; @@ -197,7 +198,7 @@ export default class IncomingDataStreamManager { // Single-packet stream: the entire payload was packaged into the header's `inlineContent`. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. - const inlineContent = streamHeader.inlineContent; + const inlineContent = streamHeader.inlineContent as NonSharedUint8Array; if (typeof inlineContent !== 'undefined') { // Inline bytes are the raw payload, optionally deflate-raw compressed. streamHandlerCallback( @@ -302,7 +303,7 @@ export default class IncomingDataStreamManager { // Single-packet stream: the entire payload was smuggled into the header's `inlineContent`. // Synthesize an already-complete stream and skip waiting for chunk/trailer packets. - const inlineContent = streamHeader.inlineContent; + const inlineContent = streamHeader.inlineContent as NonSharedUint8Array; if (typeof inlineContent !== 'undefined') { // Inline text is the raw UTF-8 payload, optionally deflate-raw compressed. const content = compressed diff --git a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts index a55b697f6e..6dbe3b66d2 100644 --- a/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts +++ b/src/room/data-stream/outgoing/OutgoingDataStreamManager.ts @@ -8,6 +8,7 @@ import { Encryption_Type, } from '@livekit/protocol'; import { type StructuredLogger } from '../../../logger'; +import { type NonSharedUint8Array } from '../../../type-polyfills/non-shared-typed-arrays'; import { CLIENT_PROTOCOL_DATA_STREAM_V2 } from '../../../version'; import type RTCEngine from '../../RTCEngine'; import { DataChannelKind } from '../../RTCEngine'; @@ -243,7 +244,7 @@ export default class OutgoingDataStreamManager { this.allRecipientsSupportCompression(destinationIdentities); let compressedStream = compressEligible ? MaybeCollectedStream.fromStream( - readableFromBytes(bytes) + readableFromBytes(bytes as NonSharedUint8Array) .pipeThrough(progressMonitorTap) .pipeThrough(deflateRawTransform()), ) @@ -284,7 +285,7 @@ export default class OutgoingDataStreamManager { const packet = createStreamHeaderPacket(header, destinationIdentities); const source = compressedStream ? compressedStream.stream() - : readableFromBytes(bytes).pipeThrough(progressMonitorTap); + : readableFromBytes(bytes as NonSharedUint8Array).pipeThrough(progressMonitorTap); await this.sendChunkedByteStream(packet, streamId, destinationIdentities, source); // Ensure there's always a 100% progress event fired, even if the buffer is zero bytes long @@ -341,7 +342,7 @@ export default class OutgoingDataStreamManager { headerPacket: DataPacket, streamId: string, destinationIdentities: Array | undefined, - source: ReadableStream, + source: ReadableStream, ): Promise { const engine = this.engine; await sendHeaderPacket(engine, headerPacket); @@ -599,14 +600,14 @@ export default class OutgoingDataStreamManager { */ class MaybeCollectedStream { private state: - | { type: 'stream'; stream: ReadableStream } - | { type: 'collected'; bytes: Uint8Array }; + | { type: 'stream'; stream: ReadableStream } + | { type: 'collected'; bytes: NonSharedUint8Array }; private constructor(state: typeof this.state) { this.state = state; } - static fromStream(stream: ReadableStream) { + static fromStream(stream: ReadableStream) { return new MaybeCollectedStream({ type: 'stream', stream }); } @@ -645,9 +646,9 @@ class MaybeCollectedStream { function progressReportingStream( totalPreCompressionLength: number | undefined, onProgress?: (progress: number) => void, -): ReadableWritablePair { +): ReadableWritablePair { let sent = 0; - return new TransformStream({ + return new TransformStream({ transform(chunk, controller) { sent += chunk.byteLength; if ( diff --git a/src/room/utils.ts b/src/room/utils.ts index ceb91a320d..4585360107 100644 --- a/src/room/utils.ts +++ b/src/room/utils.ts @@ -782,10 +782,10 @@ export function splitUtf8(s: string, n: number): NonSharedUint8Array[] { } /** Wraps a byte array in a `ReadableStream` that yields it as a single chunk and then closes. */ -export function readableFromBytes(bytes: Uint8Array): ReadableStream { - return new ReadableStream({ +export function readableFromBytes(bytes: NonSharedUint8Array): ReadableStream { + return new ReadableStream({ start(controller) { - controller.enqueue(bytes as NonSharedUint8Array); + controller.enqueue(bytes); controller.close(); }, }); @@ -798,9 +798,9 @@ export function readableFromBytes(bytes: Uint8Array): ReadableStream * `CompressionStream`/`file.stream()` output into MTU-sized data-stream chunks. */ export async function* readBytesInChunks( - source: ReadableStream, + source: ReadableStream, chunkSize: number, -): AsyncGenerator { +): AsyncGenerator { const reader = source.getReader(); let buffer = new Uint8Array(0); try { From 0d41037358c2a5d91f5b703bb7028703492dd872 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Wed, 22 Jul 2026 09:50:19 -0400 Subject: [PATCH 66/68] fix: upgrade @livekit/protocol to 1.50.4 --- package.json | 2 +- pnpm-lock.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 16172f300c..b11f4fa9e9 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ }, "dependencies": { "@livekit/mutex": "1.1.1", - "@livekit/protocol": "1.48.0", + "@livekit/protocol": "1.50.4", "events": "^3.3.0", "jose": "^6.1.0", "loglevel": "^1.9.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b74bfe3cb3..ab7fc902b0 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: 1.1.1 version: 1.1.1 '@livekit/protocol': - specifier: 1.48.0 - version: 1.48.0 + specifier: 1.50.4 + version: 1.50.4 '@types/dom-mediacapture-record': specifier: ^1 version: 1.0.22 @@ -1224,8 +1224,8 @@ packages: '@livekit/mutex@1.1.1': resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} - '@livekit/protocol@1.48.0': - resolution: {integrity: sha512-fYHYgltH6YavAsokl3qsHLkBdQeKCl4UORVTub5crS3t8JtKFZ0uinHDFQ+XXdNKS6Ub9gcOjV+UHcDiqnWXoQ==} + '@livekit/protocol@1.50.4': + resolution: {integrity: sha512-L1uggNQAqyY21smQY8AllyOYbcv9Me9TaxwuLytL1R8ck9nbYPmQLNwEDi3pOFGAMa5F8I2nUi2Jc59W5awxlA==} '@livekit/throws-transformer@0.1.3': resolution: {integrity: sha512-PBttE6W6g/2ALGu6kWOunZ5qdrXwP9Ge1An2/62OfE6Rhc0Abd4yp6ex2pWhwUfGxDsSZvFgoB1Ia/5mWAMuKQ==} @@ -5121,7 +5121,7 @@ snapshots: '@livekit/mutex@1.1.1': {} - '@livekit/protocol@1.48.0': + '@livekit/protocol@1.50.4': dependencies: '@bufbuild/protobuf': 1.10.1 From f291fd8d820675d039364a503d755f6dec8c39c5 Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 23 Jul 2026 09:45:13 -0400 Subject: [PATCH 67/68] fix: adjust changeset to minor --- .changeset/slick-geese-happen.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/slick-geese-happen.md b/.changeset/slick-geese-happen.md index bb06ec43ec..f759375a77 100644 --- a/.changeset/slick-geese-happen.md +++ b/.changeset/slick-geese-happen.md @@ -1,5 +1,5 @@ --- -'livekit-client': patch +'livekit-client': minor --- Add support for data streams v2 From c3fd1ab71472e0de6a365718e10bbedeab8e833a Mon Sep 17 00:00:00 2001 From: Ryan Gaus Date: Thu, 23 Jul 2026 09:46:47 -0400 Subject: [PATCH 68/68] fix: remove data streams spec and benchmark before merging --- DATA_STREAMS_SPEC.md | 761 ----- examples/data-stream-benchmark/README.md | 42 - examples/data-stream-benchmark/api.ts | 121 - examples/data-stream-benchmark/benchmark.ts | 527 ---- examples/data-stream-benchmark/index.html | 36 - examples/data-stream-benchmark/package.json | 27 - examples/data-stream-benchmark/payload.ts | 68 - examples/data-stream-benchmark/pnpm-lock.yaml | 2462 ----------------- .../data-stream-benchmark/pnpm-workspace.yaml | 8 - examples/data-stream-benchmark/styles.css | 148 - examples/data-stream-benchmark/tsconfig.json | 20 - examples/data-stream-benchmark/vite.config.js | 10 - 12 files changed, 4230 deletions(-) delete mode 100644 DATA_STREAMS_SPEC.md delete mode 100644 examples/data-stream-benchmark/README.md delete mode 100644 examples/data-stream-benchmark/api.ts delete mode 100644 examples/data-stream-benchmark/benchmark.ts delete mode 100644 examples/data-stream-benchmark/index.html delete mode 100644 examples/data-stream-benchmark/package.json delete mode 100644 examples/data-stream-benchmark/payload.ts delete mode 100644 examples/data-stream-benchmark/pnpm-lock.yaml delete mode 100644 examples/data-stream-benchmark/pnpm-workspace.yaml delete mode 100644 examples/data-stream-benchmark/styles.css delete mode 100644 examples/data-stream-benchmark/tsconfig.json delete mode 100644 examples/data-stream-benchmark/vite.config.js diff --git a/DATA_STREAMS_SPEC.md b/DATA_STREAMS_SPEC.md deleted file mode 100644 index e2498c5eb6..0000000000 --- a/DATA_STREAMS_SPEC.md +++ /dev/null @@ -1,761 +0,0 @@ -# Data Streams v2 Specification - -## Overview - -Data streams let a participant send a finite or open-ended sequence of bytes to one or more other -participants in a LiveKit room over the reliable data channel, routed by a **topic** to a registered -handler on the receiving side. There are two flavors: - -- **Text streams** — UTF-8 string content (chat, LLM/agent transcriptions, RPC v2 payloads). -- **Byte streams** — arbitrary binary content (files, blobs). - -A stream is always three logical parts on the wire: a **header** (opens the stream, carries -metadata), zero or more **chunks** (the content, split to fit the MTU), and a **trailer** (closes -the stream). All three are `DataPacket`s on the **reliable** channel. - -**Data streams v2** adds three optimizations/safeguards on top of the base stream protocol, -negotiated per-recipient via the participant's advertised `clientProtocol` and client capabilities: - -1. **Single-packet (inline) sends** — small finite payloads are smuggled entirely into the header - packet, skipping the chunk/trailer packets (1 packet instead of 3). Gated on `clientProtocol`. -2. **Compression** — finite, fully-known payloads (`sendText`/`sendBytes`/`sendFile`) are deflate-raw - compressed; incremental writers (`streamText`/`streamBytes`) are never compressed. Gated on the - `CAP_COMPRESSION_DEFLATE_RAW` capability (separately from `clientProtocol`). -3. **Header size limit** — the header packet must fit the MTU budget, bounding attribute size and - closing a DoS / oversized-packet vector. - -All v2 behavior is invisible to the user-facing API and falls back gracefully when a recipient does -not support it. A v2 sender must interoperate with pre-v2 receivers by sending uncompressed, -multi-packet streams. - ---- - -## Part 1: Client protocol and capabilities - -Two independent signals gate v2 features, and they are deliberately separate: - -- **`clientProtocol`** — a monotonic integer version. Crossing `>= 2` is a *baseline* commitment: - the client guarantees it understands inline single-packet streams. There is no opting out of - baseline v2 behavior. -- **client capabilities** — a set of *optional* feature flags, each advertised independently. They - cover features a client may or may not be able to do depending on platform/runtime, rather than - protocol-level invariants. Compression is one of these: a v2 client might still lack a deflate-raw - codec. - -### `clientProtocol` - -An integer each participant advertises in its `ParticipantInfo` over the signaling channel (the -same field used by RPC v2). Distinct from the signaling `protocol` version. - -| Value | Constant name | Meaning | -|-------|---------------|---------| -| `0` | `CLIENT_PROTOCOL_DEFAULT` | Legacy client. No v2 data-stream features. | -| `1` | `CLIENT_PROTOCOL_DATA_STREAM_RPC` | RPC v2 (see RPC spec). No v2 data-stream features. | -| `2` | `CLIENT_PROTOCOL_DATA_STREAM_V2` | Understands **inline single-packet** data streams. | - -### Client capabilities - -A set of optional feature flags a client advertises in its `ClientInfo.capabilities` (a repeated -enum) during the join handshake. - -| Value | Constant name | Meaning | -|-------|---------------|---------| -| `2` | `CAP_COMPRESSION_DEFLATE_RAW` | The client can **decompress** a deflate-raw compressed stream. | - -(Other capability values exist for unrelated features, e.g. `CAP_PACKET_TRAILER`.) - -### What SDKs need to do - -1. **Advertise `clientProtocol`**: set it to at least `2` (`CLIENT_PROTOCOL_DATA_STREAM_V2`) in the - join handshake. -2. **Advertise capabilities**: include `CAP_COMPRESSION_DEFLATE_RAW` in `ClientInfo.capabilities` - when the runtime can decompress deflate-raw (i.e. it has the codec the receive path needs). -3. **Read both, per remote**: store every remote participant's advertised `clientProtocol` (absent - ⇒ `0`) **and** its advertised capabilities (absent ⇒ empty). Expose a per-participant - capabilities accessor for the send path. -4. **Use**: before sending a finite stream, gate inline on recipients' `clientProtocol` and - compression on recipients' capabilities (see "recipient eligibility"). - -### Recipient eligibility - -Eligibility is evaluated over **every** recipient: the named destination identities for a targeted -send, or every remote participant in the room for a broadcast. An **empty room** (no recipients) is -considered eligible. The two v2 features gate independently: - -- **Inline single-packet** is eligible when every recipient advertises `clientProtocol >= 2`. -- **Compression** is eligible when every recipient advertises `clientProtocol >= 2` **and** - `CAP_COMPRESSION_DEFLATE_RAW`, **and** the local runtime can compress deflate-raw. - -So a v2 recipient that does not advertise the compression capability still receives **inline** -single-packet sends, but its chunked streams are sent **uncompressed**. If any recipient is pre-v2, -the send falls back to an uncompressed, multi-packet stream — which all clients understand. - ---- - -## Part 2: Wire protocol - -All packets are `DataPacket`s sent on the **reliable** data channel. A `DataPacket` carries optional -`destinationIdentities` (empty ⇒ broadcast) and a `value` oneof that, for data streams, is one of -`streamHeader` / `streamChunk` / `streamTrailer`. - -### `DataStream.Header` - -Opens a stream. Fields: - -| Field | Type | Meaning | -|-------|------|---------| -| `streamId` | string | UUID identifying this stream; used to correlate chunks and the trailer. | -| `timestamp` | int64 | Creation time (ms). | -| `topic` | string | Routes the stream to a handler registered for this topic. | -| `mimeType` | string | `text/plain` for text; the file/blob MIME type for bytes. | -| `totalLength` | optional int64 | Total **uncompressed** content byte length for finite streams; absent for unknown-length (incremental) streams. | -| `encryptionType` | enum | `NONE` or `GCM` (see E2EE below). | -| `attributes` | map | Caller-supplied metadata only (v2 carries its own signals in dedicated fields, not here). | -| `inlineContent` | optional bytes | The full payload smuggled into the header for single-packet (inline) sends; absent for chunked streams. Deflate-raw compressed iff `compression` is `DEFLATE_RAW`. | -| `compression` | enum (`DataStream.CompressionType`) | `NONE` or `DEFLATE_RAW`; applies to the inline or chunked payload. | -| `contentHeader` | oneof | `textHeader` (`DataStream.TextHeader`) or `byteHeader` (`DataStream.ByteHeader`). | - -`DataStream.TextHeader`: `operationType` (`CREATE`/`UPDATE`/`DELETE`/`REACTION`), `version` (int, -for supersede-by-version updates), `replyToStreamId`, `attachedStreamIds` (stream IDs of attached -byte streams, e.g. file attachments to a text message), `generated` (bool, marks -machine/agent-generated text such as transcriptions). - -`DataStream.ByteHeader`: `name` (file/blob name). - -### `DataStream.Chunk` - -One slice of content. Fields: - -| Field | Type | Meaning | -|-------|------|---------| -| `streamId` | string | The stream this chunk belongs to. | -| `chunkIndex` | int64 | 0-based, **contiguous** index for ordering and dedup. | -| `content` | bytes | The chunk payload (for text, a UTF-8 slice; for compressed streams, compressed bytes). | -| `version` | int | For text stream updates: supersedes a previously-received chunk at the same `chunkIndex` when higher. Otherwise `0`. | -| `iv` | optional bytes (deprecated) | Initialization vector when the chunk is E2EE-encrypted. | - -### `DataStream.Trailer` - -Closes a stream. Fields: `streamId`, `reason` (string; empty on normal close), `attributes` -(map; merged into the stream's attributes on the receiver at close — lets a sender -append metadata known only after the content, e.g. a final checksum). A conforming v2 sender sends a -trailer carrying only `streamId` on normal close; `reason`/`attributes` are optional extensions. - -### Stream lifecycle (multi-packet) - -``` -Sender Receiver - | | - |--- streamHeader (topic, attrs) ----->| (looks up handler by topic, creates a reader) - |--- streamChunk (index 0) ----------->| (delivers content as it arrives) - |--- streamChunk (index 1) ----------->| - | ... | - |--- streamTrailer -------------------->| (merges trailer attrs, closes the reader) - | | -``` - -- The receiver routes on `topic`: if no handler is registered for that topic, the stream is ignored - (chunks/trailer for an unhandled header are dropped). -- Content is delivered incrementally as chunks arrive — a receiver must not wait for the trailer to - begin yielding content. -- Chunk content larger than the MTU budget is split across multiple chunks with contiguous indices. - - Text data streams should ALWAYS split text at valid UTF-8 boundaries, which should already be - implemented for data streams v1. For an example of this behavior, see the relevant web sdk - code: https://github.com/livekit/client-sdk-js/blob/23326f9c9b85d6babb562d54b6a663f132189880/src/room/utils.ts#L758 - -### Topics and handlers - -The receiving SDK exposes registration of one handler per topic, separately for text and byte -streams. A handler is invoked once per incoming stream (when its header arrives) with a **reader** -object and the sending participant's identity. Handlers must be registered **before** connecting / -before the stream arrives, or the stream is dropped. - -### Readers - -A reader exposes the stream's `info` (id, topic, mimeType, size, attributes, name for bytes, -encryptionType, etc.) and lets the consumer either: - -- **read incrementally** — iterate chunks as they arrive (text yields decoded string pieces; bytes - yield byte arrays), or -- **read to completion** — await the full concatenated content (string for text, list/array of byte - arrays for bytes). - -The reader counts received content bytes against `totalLength` (when present) and surfaces an error -if the stream ends short, or if more bytes than declared arrive. - ---- - -## Part 3: Send APIs - -Five send operations, three finite (full payload known up front) and two incremental: - -| API | Content | Payload known up front? | Eligible for inline? | Eligible for compression? | -|-----|---------|-------------------------|----------------------|---------------------------| -| `sendText(text, opts)` | text | yes | yes | yes | -| `sendBytes(bytes, opts)` | bytes | yes | yes | yes | -| `sendFile(file, opts)` _(optional)_ | bytes | yes (streamed from disk) | **no** | yes | -| `streamText(opts) -> writer` | text | no (incremental writes) | no | **no** | -| `streamBytes(opts) -> writer` | bytes | no (incremental writes) | no | **no** | - -`sendBytes` is the byte-stream analogue of `sendText`: the whole payload is already in memory, so it -gets the same inline single-packet fast path and one-shot compression. `sendFile` is the special -case for files — the payload is streamed from disk (never buffered) and so does **not** get the -inline path. `sendBytes` does not infer a `name`/`mimeType` from its input; its byte-stream header -defaults to `name = "unknown"` and `mimeType = "application/octet-stream"`. - -**`sendBytes`, `sendText`, `streamText`, and `streamBytes` are the required send APIs; `sendFile` is -optional and platform-dependent.** Since `sendBytes` already covers sending an in-memory byte -payload, `sendFile` is only worth adding on platforms that have a native streamable file type whose -contents can be read without buffering the whole thing into memory (e.g. the `File`/`Blob` object in -browser JS, which exposes a `ReadableStream`). On such platforms `sendFile` is a thin convenience -wrapper that derives the `name`/`mimeType` from the file and streams its contents. On platforms with -no such type, omit `sendFile` entirely — callers read the bytes themselves and use `sendBytes`. The -two produce identical wire output for the same bytes (a `byteHeader` stream), so there is no -interop concern in omitting it. - -Common options: `topic`, `destinationIdentities` (omit ⇒ broadcast), `attributes` -(map), and for the finite APIs a `compress` boolean (default `true`, opt-out). -`sendText` additionally supports `attachments` (each becomes an attached byte stream referenced by -`attachedStreamIds` in the text header). `streamText` additionally supports `type` -(`create`/`update`) and `version` for streaming edits/updates of a prior stream. - -The v2 signals are carried in dedicated header fields, **not** attributes: `inlineContent` (the -single-packet payload, as raw bytes) and `compression` (`NONE` / `DEFLATE_RAW`). `attributes` carries -only caller-supplied metadata. - -### `sendText` send algorithm - -1. Compute the UTF-8 byte length as `totalLength`. -2. **Inline attempt** (only when there are no attachments and all recipients are v2): build a header - carrying the UTF-8 payload bytes in `inlineContent` with `compression = NONE`. If `compress` and - the runtime supports compression, deflate-raw the payload and, **only if the compressed form is - smaller**, put the compressed bytes in `inlineContent` and set `compression = DEFLATE_RAW`. If the - resulting serialized header packet is `<= STREAM_CHUNK_SIZE_BYTES`, send it as a single packet and - finish. Otherwise fall through. -3. **Chunked compressed** (when the send is compression-eligible — see Part 1 § Recipient - eligibility: `compress` set, runtime can compress, and every recipient is v2 **and** advertises - `CAP_COMPRESSION_DEFLATE_RAW`): send a header with `compression = DEFLATE_RAW` and `totalLength` - = uncompressed length, then the compressed content as chunks, then the trailer. -4. **Chunked uncompressed** (fallback): send the payload as a normal multi-packet text stream - (UTF-8-boundary-split chunks), header has `compression = NONE`. -5. Send each attachment as its own byte stream (`sendFile` semantics), referenced by - `attachedStreamIds` in the text header. - -### `sendBytes` send algorithm - -`sendBytes` mirrors `sendText` for an in-memory `Uint8Array` (byte-stream header, no attachments): - -1. `totalLength` is the payload's byte length. -2. **Inline attempt** (when all recipients are v2): build a byte-stream header carrying the raw - payload in `inlineContent` with `compression = NONE`. If `compress` and the runtime supports - compression, deflate-raw the payload and, **only if the compressed form is smaller**, put the - compressed bytes in `inlineContent` and set `compression = DEFLATE_RAW`. If the serialized header - packet is `<= STREAM_CHUNK_SIZE_BYTES`, send it as a single packet and finish. Otherwise fall - through. -3. **Chunked** (fallback): send a byte-stream header (`compression = DEFLATE_RAW` iff - compression-eligible, else `NONE`) then the payload — deflate-raw compressed when eligible — as - chunk packets, then the trailer. - -### `sendFile` send algorithm - -> `sendFile` is **optional** (see the note under the API table): implement it only where the -> platform has a native streamable file type. Where present, it behaves as below; where absent, -> callers use `sendBytes` instead. - -`sendFile` is fully streamed from the file's byte stream and is **never** sent inline (file uploads -are an edge case; the inline single-packet optimization is intentionally dropped for them): - -1. Compress iff the send is compression-eligible (Part 1 § Recipient eligibility): the `compress` - option is set (default true), the runtime can compress, and every recipient is v2 **and** - advertises `CAP_COMPRESSION_DEFLATE_RAW`. -2. Send a byte-stream header with `totalLength` = file size and `compression = DEFLATE_RAW` iff - compressing. -3. Stream the file's bytes → (deflate-raw if compressing) → chunk packets, then the trailer. The - whole file is never buffered in memory at once. - -Even a tiny file is sent as header + chunk(s) + trailer — there is no inline single-packet fast -path for `sendFile`, because deciding inline-eligibility would require buffering and compressing the -whole file up front. An empty file still produces a well-formed stream (`totalLength` 0 + trailer). - -### `streamText` / `streamBytes` (incremental) - -Open a header immediately (unknown `totalLength`), then the caller writes content over time; each -write is split into chunks and sent, and `close()` sends the trailer. **Incremental writers are -never compressed**: the platform stream compressor cannot flush mid-stream, and per-write flushing -costs more than it saves at typical write sizes (validated against agent-transcription workloads, -where it *expanded* the wire data). Text writes are split on UTF-8 character boundaries so each chunk -decodes independently. - ---- - -## Part 4: Single-packet (inline) optimization - -For small finite text payloads, the entire content is smuggled into the header's `inlineContent` -field (raw bytes) and sent as **one** packet (no chunks, no trailer). The decision is made by -**serializing the candidate header packet and checking its byte length against -`STREAM_CHUNK_SIZE_BYTES`** — if it fits, send inline; if not, fall back to the chunked path. This -naturally accounts for attributes, topic, framing, and (when used) the compressed payload all -together. - -- Inline applies to `sendText` and `sendBytes` (not `sendFile`, not incremental writers), and only - when all recipients are v2 (and, for `sendText`, there are no attachments). -- The receiver detects an inline stream by the presence of `inlineContent` on the header. It - synthesizes an already-complete stream from those bytes (decompressing first if `compression` is - `DEFLATE_RAW`) and never waits for chunk/trailer packets. - ---- - -## Part 5: Compression - -Compression is **deflate-raw** (raw DEFLATE, no zlib/gzip wrapper). It is applied only by the finite -send APIs (`sendText`/`sendBytes`/`sendFile`), where the full payload is known up front. Two forms: - -### Inline payload compression (single packet) - -One-shot deflate-raw of the full payload, written as raw bytes into `inlineContent`, flagged -`compression = DEFLATE_RAW`. **Kept only if it actually shrinks** the payload (deflate framing can -make tiny payloads larger). For uncompressed inline (text or byte), `inlineContent` holds the raw -payload bytes with `compression = NONE`; because `inlineContent` is a binary field there is no base64 -round-trip. - -### Chunked stream compression (multi packet) - -The full payload is compressed as a **single deflate-raw stream whose bytes are spread across the -chunk contents in `chunkIndex` order**, terminated by the DEFLATE final block before the trailer. -The header is flagged `compression = DEFLATE_RAW` and carries `totalLength` = the **pre-compression** -(decompressed) byte length. Chunk packets carry **no** compression metadata. - -### Receiver decompression - -The receiver detects `compression = DEFLATE_RAW` on the header and feeds all chunk contents (in -`chunkIndex` order) through **one** deflate-raw decompressor for the whole stream, emitting -decompressed content as it is produced. Because the decompressor is stateful and order-sensitive: - -- **Duplicate** chunk indices (index ≤ last processed) are dropped with a warning (reliable delivery - is expected, but reconnect logic may replay). -- A **gap** in chunk indices is a hard error (the stream cannot continue decompressing). - -The receiver counts **decompressed** bytes against the header's `totalLength`. For text, decompressed -bytes are re-framed on UTF-8 character boundaries so each delivered piece decodes independently. - -If the receiver has no deflate-raw decompressor (no platform `DecompressionStream`), an incoming -compressed stream is **ignored** — the topic handler is never invoked. A conforming receiver only -advertises `CAP_COMPRESSION_DEFLATE_RAW` when it can decompress, so a conforming sender never sends -it a compressed stream; the drop is a defensive backstop against a non-conforming peer. - -### Forward-compatibility note (context takeover) - -The receive path is deliberately more general than the current send path: a deflate-raw stream that -is sync-flushed at write boundaries (permessage-deflate "context takeover") also decodes -incrementally through the same single-decompressor path. This means a future incremental sender -(compressed `streamText`/`streamBytes`) could be introduced **without a `clientProtocol` bump** — -existing v2 receivers already decode that wire format. - -### Eligibility recap - -Compression is used iff `compress` is requested (default true) AND the local runtime provides a -deflate-raw compressor AND every recipient advertises `clientProtocol >= 2` AND every recipient -advertises the `CAP_COMPRESSION_DEFLATE_RAW` capability. Otherwise the stream is sent uncompressed. -A pre-v2 recipient — or a v2 recipient that does not advertise the capability — therefore always -receives uncompressed, multi-packet streams (it still receives inline single-packet sends, which -are gated on `clientProtocol` alone). - ---- - -## Part 6: Header size limit (MTU) - -A `DataStream.Header` is a single `DataPacket`; one larger than the MTU cannot be reliably sent, and -unbounded `attributes` are both a correctness hazard and a DoS vector. Therefore: - -**When sending any stream header on the chunked path, the SDK serializes the header packet and, if -its byte length exceeds `STREAM_CHUNK_SIZE_BYTES`, throws an error (`HeaderTooLarge`) instead of -emitting the packet.** This bounds attributes + topic + framing together against the MTU. - -- This is a **breaking change**: previously oversized attributes were accepted; they now error. -- The **inline** path keeps its existing graceful behavior — if the inline header exceeds the - budget it falls back to the chunked path (no throw); the chunked header send is what enforces the - hard limit. So a large *payload* with small attributes falls back and sends fine, while large - *attributes* (whose chunked header still exceeds the MTU) throw. -- Enforcement is **send-side only**. Receivers are not required to reject oversized incoming headers - (interop with other/older SDKs). - -### Constants - -| Constant | Value | Meaning | -|----------|-------|---------| -| `STREAM_CHUNK_SIZE_BYTES` | `15000` | Max chunk content size AND the header-packet MTU budget. Kept below the ~16 KB data-channel MTU for protocol/E2EE framing headroom. | - ---- - -## Part 7: Receive-side semantics - -1. **Header** → look up the topic handler (text or byte). If none, ignore the stream. Otherwise - build the stream `info` (stripping reserved attributes), detect inline / compression, create the - reader, register it by `streamId`, and invoke the handler with the reader + sender identity. - Reject a duplicate `streamId` whose stream is already open. -2. **Chunk** → route by `streamId` to the open stream; enforce consistent `encryptionType`; deliver - content (through the decompressor for compressed streams). Empty chunks are ignored. -3. **Trailer** → merge `trailer.attributes` into the stream `info`, then close the reader. Drop the - stream's registration. -4. **Length validation** → the reader compares received content bytes against the header's - `totalLength` (when present): short ⇒ "incomplete" error at close; over ⇒ "length exceeded" - error. -5. **Abnormal end** → if a sending participant disconnects while it has streams in flight to this - receiver, those open readers are errored ("abnormal end"). -6. **Text updates** → a later chunk at an existing `chunkIndex` with a higher `version` supersedes - the earlier one (used with `TextHeader.operationType = UPDATE`). -7. **Connection gating** → packets received before the receiver is marked connected are buffered and - replayed in order once it connects (streams can begin arriving during the join handshake). -8. **Unsupported compression** → a compressed stream is ignored (its handler is never invoked) when - the receiver has no deflate-raw decompressor (see Part 5). - ---- - -## Recommended naming - -In the reference implementation: - -- The entity that builds and sends streams is `OutgoingDataStreamManager`. -- The entity that receives, routes, and exposes streams to handlers is `IncomingDataStreamManager`. -- Client-protocol constants: `CLIENT_PROTOCOL_DEFAULT` (0), `CLIENT_PROTOCOL_DATA_STREAM_RPC` (1), - `CLIENT_PROTOCOL_DATA_STREAM_V2` (2). -- Capability constant: `CAP_COMPRESSION_DEFLATE_RAW` (2), stored on each remote participant and - exposed via a `getRemoteParticipantCapabilities(identity)` accessor the send path consults. -- V2 header fields: `inlineContent` (bytes), `compression` (`DataStream.CompressionType`: `NONE` / - `DEFLATE_RAW`). -- Chunk/header budget constant: `STREAM_CHUNK_SIZE_BYTES` (15000). - -Use these names unless prior SDK architecture makes it burdensome; if you diverge, explain the -rationale and confirm with the user before continuing. The header fields (`inlineContent`, -`compression`), topics, protobuf field names, `clientProtocol` values, the -`CAP_COMPRESSION_DEFLATE_RAW` capability value, and the `STREAM_CHUNK_SIZE_BYTES` budget are **wire -contract** and must match exactly for cross-SDK interop. - ---- - -## Minimum required test cases - -The two managers are independently testable and a conforming implementation must pass both sets: - -- The **`OutgoingDataStreamManager`** is exercised by calling the send APIs against a captured-packet - engine and a configurable set of remote participants (each with a `clientProtocol` and a capability - set), then asserting on the emitted packets (which case, header attributes, chunk contents/indices, - trailer). Recipient scenarios: a room of all pre-v2 participants, a room of all v2 participants - (one of which advertises **no** compression capability), and a mixed room. -- The **`IncomingDataStreamManager`** is exercised by registering a topic handler, marking the - manager connected, feeding hand-crafted packets, and asserting on what the reader yields or the - error it raises. - -### `OutgoingDataStreamManager` (send side) - -**Test harness.** Construct the manager with an engine that pushes every `sendDataPacket(packet)` -into a `sentPackets` array, plus a configurable map of remote participants → `(clientProtocol, -capabilities)`. Call a send API, then assert on `sentPackets`: each packet's `value.case` -(`streamHeader` / `streamChunk` / `streamTrailer`), the header's `contentHeader.case` -(`textHeader` / `byteHeader`) and `attributes`, each chunk's `chunkIndex` and `content`, and the -trailer's `streamId`. Three participant rooms are used: - -- **all pre-v2** — `alice`, `bob` at `clientProtocol 0`, `jim` at `1` (RPC). -- **all v2** — `alice`, `bob` at `clientProtocol 2` with `CAP_COMPRESSION_DEFLATE_RAW`; `noCompression` - at `clientProtocol 2` with **no** capabilities. -- **mixed** — `alice` (0), `bob`/`jim` (2 + cap), `mallory` (1), `noCompression` (2, no cap). - -The `sendFile` cases below apply only to SDKs that implement the optional `sendFile` API (see Part 3); -omit them on platforms without a native streamable file type. - -#### Sending to a room where every recipient is pre-v2 - -1. **Short text → legacy multi-packet, uncompressed** - - Call `sendText('hello world', { topic })` (broadcast; all recipients pre-v2). - - Expect exactly **3** packets. - - Packet 0 is a `streamHeader` with `contentHeader.case === 'textHeader'`, `streamId === info.id`, - and the given `topic`. - - Packet 1 is a `streamChunk` with `chunkIndex 0` and `content` equal to the raw UTF-8 of - `'hello world'`. - - Packet 2 is a `streamTrailer` with the matching `streamId` and empty `reason`. - - The header's `compression` is `NONE` and `inlineContent` is absent. - -2. **Short bytes → legacy multi-packet, uncompressed** - - Open `streamBytes({ topic })`, `write([0x00,0x01,0x02,0x03])`, `close()`. - - Expect **3** packets: a `byteHeader`, one `streamChunk` (`chunkIndex 0`, `content` equal to the - four raw bytes), and a trailer. - -3. **Long text → uncompressed multi-packet** - - Call `sendText('A'.repeat(40_000), { topic })`. - - Expect **5** packets: header + 3 chunks + trailer. Chunk indices are `0,1,2` (contiguous), each - `content` is all `'A'`, and chunks are split at `STREAM_CHUNK_SIZE_BYTES` (15000, 15000, 10000). - - The header's `compression` is `NONE`. - -4. **Long bytes → uncompressed multi-packet** - - Open `streamBytes({ topic })`, write a 20 000-byte buffer of `0x01` twice, `close()`. - - Expect **6** packets: header + 4 chunks + trailer. Each write produces a 15 000-byte chunk then - a 5 000-byte chunk; chunk indices are `0,1,2,3` (contiguous across writes); content is all `0x01`. - -5. **File → uncompressed multi-packet** - - Call `sendFile(new File([Uint8Array(20_000).fill(0x07)], 'text.txt'), { topic })`. - - Expect **4** packets: a `byteHeader` with `compression = NONE`, a 15 000-byte chunk - (`chunkIndex 0`), a 5 000-byte chunk (`chunkIndex 1`), and a trailer; chunk content is raw `0x07`. - -#### Sending to a room where every recipient is v2 - -6. **Short compressible text → single inline packet, compressed** - - Call `sendText('hello hello compressible world', { topic, destinationIdentities: ['alice','bob'] })`. - - Expect exactly **1** packet: a `streamHeader` (`textHeader`). - - `compression === DEFLATE_RAW`; `inlineContent` is a `Uint8Array` and is **not** the raw UTF-8 - of the text (it is the compressed bytes). - - No `streamChunk` or `streamTrailer` packets. - -7. **Short incompressible text → single inline packet, raw** - - Call `sendText('short', { ..., destinationIdentities: ['alice','bob'] })`. - - Expect **1** packet. `compression` is `NONE` and `inlineContent` equals the raw UTF-8 of - `'short'` — deflate didn't shrink it, so the raw bytes are kept (compression is only applied - when it actually reduces size). - -8. **Short text to a recipient lacking the compression capability → single inline packet, raw** - - Call `sendText('hello hello compressible world', { ..., destinationIdentities: ['noCompression'] })`. - - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the raw UTF-8 of the text. - Inline still happens (gated on `clientProtocol`); compression does not (gated on the capability). - -9. **Large highly-compressible text → single inline packet, compressed** - - Call `sendText('hello world'.repeat(20_000), { ..., destinationIdentities: ['alice','bob'] })`. - - Expect **1** packet. `compression === DEFLATE_RAW`; `inlineContent` is compressed bytes (does - **not** start with the UTF-8 of `'hello world'`). It compresses well under the MTU, so it still - goes inline. - -10. **Large somewhat-compressible text → compressed multi-packet** - - Build a ~50 KB payload of 50 × (`'hello world'` + 1 000 random chars) and `sendText` it to - `['alice','bob']`. - - Expect **5** packets: header (`compression = DEFLATE_RAW`) + **3** chunks + trailer — fewer - than the `ceil(50_000 / 15_000) = 4` chunks an uncompressed send would need. The first chunk's - `content` length is 15 000 (MTU). - -11. **Large incompressible file → compressed multi-packet** - - Call `sendFile(new File([50_000 random bytes], 'text.txt'), { ..., destinationIdentities: ['alice','bob'] })`. - - Expect **6** packets: a `byteHeader` (`compression = DEFLATE_RAW`) + **4** chunks (first is - 15 000 bytes) + trailer. - - The summed chunk content length is **greater** than 50 000 — deflate adds slight overhead on - incompressible data, the accepted trade-off for streaming the file instead of buffering it. - -12. **`compress: false`, short payload → single inline packet, raw** - - Call `sendText('hello hello compressible world', { ..., destinationIdentities: ['alice','bob'], compress: false })`. - - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the raw UTF-8 of the text - (inline still applies; the opt-out only disables compression). - -13. **`compress: false`, large payload → uncompressed multi-packet** - - `sendText` the same ~50 KB somewhat-compressible payload to `['alice','bob']` with `compress: false`. - - Expect **6** packets: header (`compression = NONE`) + **4** chunks (first is 15 000 bytes) + - trailer (uncompressed → `ceil(50_000 / 15_000) = 4` chunks). - -14. **`streamText` never compresses or inlines** - - Open `streamText({ topic, destinationIdentities: ['noCompression'] })`; after open, expect **1** - packet — a `textHeader` with `compression = NONE`. - - `write('hello world')` → expect **2** packets total; the new `streamChunk` content equals the - raw UTF-8 of `'hello world'`. - - `close()` → expect **3** packets total; the last is a `streamTrailer`. - -15. **`streamBytes` never compresses or inlines** - - Open `streamBytes({ topic, destinationIdentities: ['noCompression'] })`; expect **1** packet — a - `byteHeader` with `compression = NONE`. - - `write([0x00,0x01,0x02,0x03])` → expect **2** packets; the chunk content equals the raw bytes. - - `close()` → expect **3** packets; the last is a trailer. - -16. **`sendFile` never sends a single inline packet** - - Call `sendFile(new File([Uint8Array(10_000).fill(0x01)], 'text.txt'), { ..., destinationIdentities: ['alice','bob'] })` - (highly compressible). - - Expect **3** packets (header + 1 chunk + trailer), **not** 1. The header is a `byteHeader` with - `compression = DEFLATE_RAW`; the chunk `content` length is **less** than 10 000 (compressed). - `sendFile` never uses the inline path regardless of how small/compressible the file is. - -17. **File to a recipient lacking the compression capability → uncompressed multi-packet** - - Call `sendFile(new File([Uint8Array(10_000).fill(0x07)], 'text.txt'), { ..., destinationIdentities: ['noCompression'] })`. - - Expect **3** packets: a `byteHeader` with `compression = NONE` and **no** `inlineContent`, - one 10 000-byte chunk (`chunkIndex 0`, all `0x07`, uncompressed, under the MTU), and a trailer. - -18. **Empty file** - - Call `sendFile(new File([], 'empty.bin'), { ..., destinationIdentities: ['alice','bob'] })`. - - Packet 0 is a `byteHeader` with `totalLength === 0` and `compression = DEFLATE_RAW`. - - The last packet is a `streamTrailer` with the matching `streamId`. (A well-formed stream is - still produced — the deflate stream's final block plus the trailer.) - -#### Sending to a mixed room (some pre-v2, some v2) - -19. **Broadcast falls back to legacy** - - Call `sendText('hello world', { topic })` (broadcast) in the mixed room (contains pre-v2 - participants). - - Expect **3** packets: `textHeader` + chunk (`content` = raw `'hello world'`) + trailer. No - `inlineContent`; `compression = NONE`. - -20. **Targeted send to an all-v2, all-capable subset → single inline packet, compressed** - - Call `sendText('hello hello compressible world', { ..., destinationIdentities: ['bob','jim'] })` - (both v2 + capability). - - Expect **1** packet with `compression = DEFLATE_RAW` and `inlineContent` being compressed bytes - (not the raw text). Restricting the send to capable recipients re-enables inline + compression. - -21. **Targeted send to a subset where one lacks the capability → inline, uncompressed** - - Call `sendText('hello hello compressible world', { ..., destinationIdentities: ['bob','jim','noCompression'] })`. - - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the raw UTF-8 of the text — - inline still happens (all three are v2) but compression is gated off by `noCompression`. - -#### `sendBytes` (in-memory byte payloads) - -`sendBytes` mirrors `sendText`'s inline + compression behavior on a `byteHeader`. Every emitted -header has `contentHeader.case === 'byteHeader'`, and the returned `info` defaults `name` to -`'unknown'` and `mimeType` to `'application/octet-stream'` (and threads `topic`/`attributes`/`size`). - -22. **Short compressible bytes → single inline packet, compressed** (all-v2 room) - - Call `sendBytes(utf8('hello hello compressible world'), { topic, attributes: { foo: 'bar' }, destinationIdentities: ['alice','bob'] })`. - - Expect **1** packet: a `byteHeader` with `compression === DEFLATE_RAW`; `inlineContent` is a - `Uint8Array` that is **not** the raw payload. `info.name === 'unknown'`, - `info.mimeType === 'application/octet-stream'`, `info.size === payload.byteLength`, - `info.attributes === { foo: 'bar' }`. - -23. **Short incompressible bytes → single inline packet, raw** (all-v2 room) - - Call `sendBytes([0x00,0x01,0x02,0x03], { ..., destinationIdentities: ['alice','bob'] })`. - - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the four raw bytes. - -24. **Bytes to a recipient lacking the compression capability → single inline packet, raw** - - Call `sendBytes(utf8('hello hello compressible world'), { ..., destinationIdentities: ['noCompression'] })`. - - Expect **1** packet. `compression` is `NONE`; `inlineContent` equals the raw payload (inline is - gated on `clientProtocol`; compression on the capability). - -25. **Large highly-compressible bytes → single inline packet, compressed** (all-v2 room) - - Call `sendBytes(Uint8Array(50_000).fill(0x01), { ..., destinationIdentities: ['alice','bob'] })`. - - Expect **1** packet. `compression === DEFLATE_RAW`; `inlineContent` byte length is **less** than - 50 000 (compresses well under the MTU, so it still goes inline). - -26. **Large somewhat-compressible bytes → compressed multi-packet** (all-v2 room) - - `sendBytes` a ~50 KB payload of 50 × (`'hello world'` + 1 000 random chars) (UTF-8 encoded) to - `['alice','bob']`. - - Expect **5** packets: `byteHeader` (`compression = DEFLATE_RAW`, **no** `inlineContent`) + **3** - chunks (first is 15 000 bytes) + trailer. - -27. **`compress: false`, large bytes → uncompressed multi-packet** (all-v2 room) - - Call `sendBytes(Uint8Array(40_000).fill(0x07), { ..., destinationIdentities: ['alice','bob'], compress: false })`. - - Expect **5** packets: `byteHeader` (`compression = NONE`, no `inlineContent`) + **3** chunks - (15 000, 15 000, 10 000; all `0x07`) + trailer. - -28. **Bytes to a pre-v2 room → legacy uncompressed multi-packet** - - Call `sendBytes([0x00,0x01,0x02,0x03], { topic })` with only pre-v2 recipients. - - Expect **3** packets: `byteHeader` (`compression = NONE`, no `inlineContent`) + one chunk - (`chunkIndex 0`, the four raw bytes) + trailer. Never inline, never compressed. - -### `IncomingDataStreamManager` (receive side) - -**Test harness.** Construct the manager, register a text or byte stream handler for a topic that -resolves a promise with the delivered `reader`, mark the manager connected (`setConnected(true)`, -except where buffering is under test), then feed hand-crafted packets via -`handleDataStreamPacket(packet, encryptionType)`. Assertions are on the reader: `await -reader.readAll()` (the full string for text, the array of byte chunks for bytes), `reader.info` -(attributes, `attachedStreamIds`), or the error `readAll()` rejects with. The sending participant -identity is `'alice'` throughout. - -#### Receiving v1 (legacy multi-packet) streams - -1. **Text stream round-trips** - - Feed a `textHeader` (`totalLength` = the byte length, `attributes: { foo: 'bar' }`), a chunk - (`chunkIndex 0`, raw UTF-8), and a trailer (matching `streamId`). - - `await reader.readAll()` equals the text; `reader.info.attributes.foo === 'bar'`. - -2. **Byte stream round-trips** - - Feed a `byteHeader` (`totalLength 4`, `attributes: { foo: 'bar' }`), a chunk - (`content [0x01,0x02,0x03,0x04]`), and a trailer. - - `await reader.readAll()` equals `[Uint8Array([1,2,3,4])]`; `reader.info.attributes.foo === 'bar'`. - -3. **Text stream with attachments** - - Register both a text and a byte handler for the topic. - - Feed a `textHeader` whose `contentHeader.attachedStreamIds` references an attachment stream id - (with an inline payload for the text body), then a separate byte stream for that attachment id - (header + chunk + trailer). - - Both handlers fire: the text reader yields the body, the byte reader yields the attachment - bytes, and `textReader.info.attachedStreamIds` has length 1. - -4. **Buffers packets received while disconnected** - - `setConnected(false)`. Feed header + chunk + trailer. - - Assert the handler has **not** fired yet (the reader promise is still pending). - - `setConnected(true)`. The handler now fires and `await reader.readAll()` equals the text. - -5. **Merges trailer attributes** - - Feed a header with `attributes: { foo: 'bar', baz: 'quux' }`, a chunk, and a trailer with - `attributes: { hello: 'world', foo: 'updated' }`. - - After close, `reader.info.attributes` has `baz === 'quux'` (header), `hello === 'world'` - (trailer), and `foo === 'updated'` (trailer overrides header). - -6. **Drops packets with a mismatched `encryptionType`** - - Feed the header with `Encryption_Type.NONE`, then feed a chunk with `Encryption_Type.GCM`. - - `await reader.readAll()` rejects with an `EncryptionTypeMismatch` error ("Encryption type - mismatch"). - -7. **Errors when too few bytes arrive** - - Feed a header declaring `totalLength 5`, a single 1-byte chunk, and a trailer. - - `await reader.readAll()` rejects with "Not enough chunk(s)" (raised when the stream closes). - -8. **Errors when too many bytes arrive** - - Feed a header declaring `totalLength 3`, then a 5-byte chunk (and a trailer). - - `await reader.readAll()` rejects with "Extra chunk(s)" (raised as the over-budget chunk is - processed, before the trailer matters). - -9. **Errors on sender disconnect mid-stream** - - Feed a header declaring `totalLength 10` and a 5-byte chunk (no trailer). - - Call `manager.validateParticipantHasNoActiveDataStreams('alice')` (the room calls this on - disconnect). - - `await reader.readAll()` rejects with "Participant alice unexpectedly disconnected in the middle - of sending data". - -#### Receiving v2 streams - -10. **Inline uncompressed text** - - Feed a single `textHeader` with `inlineContent` = the raw UTF-8 of the text, `attributes: - { foo: 'bar' }`, and **no** chunk or trailer packets. - - `await reader.readAll()` equals the text; `reader.info.attributes.foo === 'bar'` (the v2 signals - were never in `attributes`, so nothing is stripped). - -11. **Inline uncompressed byte** - - Feed a single `byteHeader` (`totalLength 3`) with `inlineContent` = `[0x01,0x02,0x03]` and - `compression = NONE`. - - `await reader.readAll()` equals `[Uint8Array([1,2,3])]`. - -12. **Inline compressed text** - - Feed a single `textHeader` (`totalLength` = the *uncompressed* byte length) with `inlineContent` - = `deflateRaw(text)`, `compression = DEFLATE_RAW`, `attributes: { foo: 'bar' }`. - - `await reader.readAll()` equals the decompressed text; `reader.info.attributes.foo === 'bar'`. - -13. **Inline compressed byte** - - Feed a single `byteHeader` (`totalLength` = the uncompressed length) with `inlineContent` = - `deflateRaw(bytes)`, `compression = DEFLATE_RAW`. - - `await reader.readAll()` equals the original bytes (decompressed). - -14. **Multi-packet compressed text** - - Compress a ~30 KB somewhat-compressible text with one deflate-raw pass (verify the compressed - output is under `2 × STREAM_CHUNK_SIZE_BYTES`). - - Feed a `textHeader` (`totalLength` = the uncompressed length, `compression = DEFLATE_RAW`), - then split the compressed bytes into two chunks at `STREAM_CHUNK_SIZE_BYTES` (`chunkIndex 0` and - `1`), then a trailer. - - `await reader.readAll()` equals the original text. - -15. **Ignores a compressed stream when no decompressor is available** - - IMPORTANT NOTE: this test is only relevant if the client relies on platform support for compression / - decompression. If the client always supports encryption (ie, maybe via a library which is - always available) this test can be skipped. - - Temporarily make the platform `CompressionStream`/`DecompressionStream` unavailable. - - Feed a compressed (`compression = DEFLATE_RAW`) **text** stream (header + chunk + trailer); - assert the handler is **never** invoked (the reader promise stays pending) — the stream is - dropped. - - Repeat with a compressed **byte** stream; same result. Restore the globals afterward. - ---- - -## Benchmarking - -Implementing a benchmark is optional but useful for validating correctness and performance under -realistic conditions. Two reference shapes from `client-sdk-js`: - -- **Throughput grid** — connect a sender and receiver to one room; for a grid of payload sizes × - simulated network conditions, send finite streams for a fixed window and measure - received-stream throughput, one-way latency percentiles (p50/p95/p99), and integrity - (checksum) mismatches. -- **Agent-transcription scenario** — model a voice-agent transcript: stream short word-sized writes - at a realistic cadence (median ~350 ms between writes, occasional bursts) over a long-lived - `streamText` stream, optionally alongside competing "junk" reliable data packets on the same - channel, and measure **per-chunk staleness** (receiver-arrival minus sender-write time) - percentiles plus chunks delivered vs sent. This workload is what established that incremental - compression is not worthwhile at word granularity. diff --git a/examples/data-stream-benchmark/README.md b/examples/data-stream-benchmark/README.md deleted file mode 100644 index f4a4beafc1..0000000000 --- a/examples/data-stream-benchmark/README.md +++ /dev/null @@ -1,42 +0,0 @@ -# Data Stream Benchmark - -Measures the end-to-end latency of LiveKit **v2 data streams** across a grid of network conditions -(X axis) and payload sizes (Y axis). - -Two participants (`bench-sender` and `bench-receiver`) join a shared room in the same browser tab. -For each box in the grid the sender sends a fixed number of data streams (default 10) of random, -realistic JSON data to the receiver. Each stream header carries a `checksum` (XOR of the payload -bytes) and a `sendTs` timestamp; the receiver verifies the checksum and computes the end-to-end -latency. Each cell shows the average latency over the checksum-matching streams, or `N/A`. - -## Running - -1. Create `.env.local` with `LIVEKIT_API_KEY`, `LIVEKIT_API_SECRET`, and `LIVEKIT_URL`. -2. Install dependencies: `pnpm install` -3. Start the server: `pnpm dev` -4. Open the local URL (typically http://localhost:5173). -5. Click **Connect**, then **Run Benchmark**. Click **Disconnect** when done. - -## Network conditioning (macOS only) - -The **X axis** (Edge / 3G / LTE / Wi-Fi / None) is driven by the macOS **Network Link Conditioner**. -The server (`api.ts`) toggles it via `osascript` through `POST /api/network-condition`. - -Prerequisites: - -- The **Network Link Conditioner** preference pane must be installed - (`/Library/PreferencePanes/Network Link Conditioner.prefPane`, from Apple's "Additional Tools for - Xcode"). -- The process running `vite`/node needs macOS **Accessibility** permission (System Settings → - Privacy & Security → Accessibility) so it can drive System Settings. -- The preset names must match the conditioner's menu exactly: `Edge`, `3G`, `LTE`, `Wi-Fi`. - -The benchmark always resets the conditioner to **off** when it finishes, errors, or you disconnect. - -## Notes - -- The v2 single-packet (inline) optimization only changes behavior for payloads under the ~15 KB - header budget (10 B–10 KB); 100 KB and 1 MB are multi-packet regardless. -- `Edge`/`3G` × `1 MB` can be slow; each send has a 60 s timeout (`SEND_TIMEOUT_MS` in - `benchmark.ts`) and slow boxes show `N/A`. Repeat count and timeout are constants at the top of - `benchmark.ts`. diff --git a/examples/data-stream-benchmark/api.ts b/examples/data-stream-benchmark/api.ts deleted file mode 100644 index 850fdd7173..0000000000 --- a/examples/data-stream-benchmark/api.ts +++ /dev/null @@ -1,121 +0,0 @@ -import { exec } from 'child_process'; -import dotenv from 'dotenv'; -import express from 'express'; -import { AccessToken } from 'livekit-server-sdk'; -import { promisify } from 'util'; -import type { Express } from 'express'; - -dotenv.config({ path: '.env.local' }); - -const execAsync = promisify(exec); - -const LIVEKIT_API_KEY = process.env.LIVEKIT_API_KEY; -const LIVEKIT_API_SECRET = process.env.LIVEKIT_API_SECRET; -const LIVEKIT_URL = process.env.LIVEKIT_URL; - -/** Network Link Conditioner presets we allow the client to request. */ -const ALLOWED_PRESETS = ['Edge', '3G', 'LTE', 'Wi-Fi', 'Very Bad Network']; - -const app = express(); -app.use(express.json()); - -app.post('/api/get-token', async (req, res) => { - const { identity, roomName } = req.body; - - if (!LIVEKIT_API_KEY || !LIVEKIT_API_SECRET) { - res.status(500).json({ error: 'Server misconfigured' }); - return; - } - - const token = new AccessToken(LIVEKIT_API_KEY, LIVEKIT_API_SECRET, { - identity, - }); - token.addGrant({ - room: roomName, - roomJoin: true, - canPublish: true, - canSubscribe: true, - }); - - res.json({ - token: await token.toJwt(), - url: LIVEKIT_URL, - }); -}); - -/** - * Enables the macOS Network Link Conditioner at the given preset. The preset name is read from the - * `mode` environment variable via `system attribute "mode"`. - */ -const ENABLE_SCRIPT = `osascript <<'EOF' -set mode to system attribute "mode" -do shell script "open '/Library/PreferencePanes/Network Link Conditioner.prefPane'" -delay 2 -tell application "System Settings" to activate -tell application "System Events" - tell process "System Settings" - tell window "Network Link Conditioner" - tell scroll area 1 of group 3 of splitter group 1 of group 1 - tell group 1 - click pop up button 1 - delay 0.3 - click menu item mode of menu 1 of pop up button 1 - end tell - delay 0.3 - click button "ON" - end tell - end tell - end tell -end tell -EOF`; - -/** Disables the macOS Network Link Conditioner, returning to the host network speed. */ -const DISABLE_SCRIPT = `osascript <<'EOF' -do shell script "open '/Library/PreferencePanes/Network Link Conditioner.prefPane'" -delay 2 -tell application "System Settings" to activate -tell application "System Events" - tell process "System Settings" - tell window "Network Link Conditioner" - tell scroll area 1 of group 3 of splitter group 1 of group 1 - click button "OFF" - end tell - end tell - end tell -end tell -EOF`; - -app.post('/api/network-condition', async (req, res) => { - const { preset } = req.body as { preset?: string }; - - if (preset === 'off') { - try { - await execAsync(DISABLE_SCRIPT, { timeout: 30_000 }); - res.json({ ok: true, preset: 'off' }); - } catch (error) { - res.status(500).json({ error: `Failed to disable network conditioner: ${String(error)}` }); - } - return; - } - - if (!preset || !ALLOWED_PRESETS.includes(preset)) { - res - .status(400) - .json({ error: `Invalid preset "${preset}". Allowed: ${ALLOWED_PRESETS.join(', ')}, off` }); - return; - } - - try { - await execAsync(ENABLE_SCRIPT, { - timeout: 30_000, - env: { ...process.env, mode: preset }, - }); - res.json({ ok: true, preset }); - } catch (error) { - res - .status(500) - .json({ error: `Failed to set network conditioner to ${preset}: ${String(error)}` }); - } -}); - -export const handler: Express = app; diff --git a/examples/data-stream-benchmark/benchmark.ts b/examples/data-stream-benchmark/benchmark.ts deleted file mode 100644 index 86276122ac..0000000000 --- a/examples/data-stream-benchmark/benchmark.ts +++ /dev/null @@ -1,527 +0,0 @@ -import { Room, RoomEvent } from '../../src/index'; -import { checksum, generatePayload } from './payload'; - -// --------------------------------------------------------------------------- -// Tunables -// --------------------------------------------------------------------------- - -/** Number of concurrent caller "threads" (async send loops) per box. */ -const CONCURRENCY = 4; -/** How long each box sends for. */ -const BOX_DURATION_MS = 5_000; -/** Grace period after the send window to let in-flight streams finish arriving. */ -const DRAIN_MS = 2_000; -/** Received count that maps to a fully-unfilled (white) cell; 0 received is fully filled. */ -const MAX_FILL_COUNT = BOX_DURATION_MS; -/** Cell fill hue (R,G,B); opacity scales with throughput. */ -const FILL_RGB = '52,152,219'; - -/** Chunk size to split up the data stream payload into. If `0`, send all at once with `sendText`. */ -const STREAM_CHUNK_SIZE_BYTES = 0; - -const TOPIC = 'benchmark'; -const SENDER_IDENTITY = 'bench-sender'; -const RECEIVER_IDENTITY = 'bench-receiver'; - -const SIZES: Array<{ label: string; bytes: number }> = [ - { label: '10 B', bytes: 10 }, - { label: '100 B', bytes: 100 }, - { label: '512 B', bytes: 512 }, - { label: '1 KB', bytes: 1_000 }, - { label: '15 KB', bytes: 15_000 }, - { label: '100 KB', bytes: 100_000 }, - { label: '500 KB', bytes: 500_000 }, - { label: '1 MB', bytes: 1_000_000 }, -]; - -// `value` is the preset passed to /api/network-condition ('off' disables the conditioner). -const PRESETS: Array<{ label: string; value: string }> = [ - { label: 'None', value: 'off' }, - { label: 'Wi-Fi', value: 'Wi-Fi' }, - { label: 'LTE', value: 'LTE' }, - { label: '3G', value: '3G' }, - { label: 'Edge', value: 'Edge' }, - { label: 'Very Bad Net.', value: 'Very Bad Network' }, -]; - -// --------------------------------------------------------------------------- -// State -// --------------------------------------------------------------------------- - -/** Per-box metrics, modeled on the old RPC benchmark's BenchmarkStats but adapted to one-directional - * data streams (latency is measured one-way: receiver clock − sender's `sendTs` attribute). A fresh - * instance is created per box and captured by that box's receiver handler, so late (post-snapshot) - * arrivals can't leak into the next box. */ -class BoxStats { - sent = 0; - - sendErrors = 0; - - received = 0; - - mismatch = 0; - - latencies: number[] = []; - - errors: Record = {}; - - recordSent() { - this.sent += 1; - } - - recordSendError(kind: string) { - this.sendErrors += 1; - this.errors[kind] = (this.errors[kind] ?? 0) + 1; - } - - recordReceived(latencyMs: number, checksumOk: boolean) { - this.received += 1; - this.latencies.push(latencyMs); - if (!checksumOk) { - this.mismatch += 1; - } - } - - /** Arrived and checksum-valid — the analog of the old benchmark's `successfulCalls`. */ - get valid() { - return this.received - this.mismatch; - } - - /** Sent but never received within the drain window. */ - get lost() { - return Math.max(0, this.sent - this.received); - } - - get successRate() { - return this.sent > 0 ? (100 * this.valid) / this.sent : 0; - } - - private sortedLatencies(): number[] { - return [...this.latencies].sort((a, b) => a - b); - } - - get avgLatency(): number { - if (this.latencies.length === 0) { - return 0; - } - return this.latencies.reduce((a, b) => a + b, 0) / this.latencies.length; - } - - percentile(p: number): number { - const s = this.sortedLatencies(); - if (s.length === 0) { - return 0; - } - const idx = Math.min(Math.floor((p / 100) * s.length), s.length - 1); - return s[idx]; - } - - /** Received streams per second over the send window. */ - throughput(elapsedSec: number): number { - return elapsedSec > 0 ? this.received / elapsedSec : 0; - } - - /** Received bytes per second over the send window. */ - bytesPerSec(sizeBytes: number, elapsedSec: number): number { - return elapsedSec > 0 ? (this.received * sizeBytes) / elapsedSec : 0; - } -} - -let senderRoom: Room | null = null; -let receiverRoom: Room | null = null; -let running = false; - -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - -// --------------------------------------------------------------------------- -// UI helpers -// --------------------------------------------------------------------------- - -const $ = (id: string) => document.getElementById(id) as T; - -function log(message: string) { - const area = $('log'); - const ts = new Date().toLocaleTimeString(); - area.value += `[${ts}] ${message}\n`; - area.scrollTop = area.scrollHeight; - // eslint-disable-next-line no-console - console.log(message); -} - -function setStatus(message: string) { - $('status').textContent = message; -} - -function cell(rowIdx: number, colIdx: number) { - return $(`cell-${rowIdx}-${colIdx}`); -} - -function buildGrid() { - const table = $('grid'); - const header = ['Payload \\ Network', ...PRESETS.map((p) => p.label)] - .map((label) => `${label}`) - .join(''); - - const rows = SIZES.map((size, rowIdx) => { - const cells = PRESETS.map( - (_, colIdx) => - `
—
`, - ).join(''); - return `${size.label}${cells}`; - }).join(''); - - table.innerHTML = `${header}${rows}`; -} - -/** Renders a box: big throughput number, a compact multi-metric status line below, and a fill - * opacity from the received count. `statusHtml` may contain markup (e.g. a red mismatch token). */ -function renderCell( - rowIdx: number, - colIdx: number, - opts: { recv: string; statusHtml?: string; fill?: number; title?: string; running?: boolean }, -) { - const td = cell(rowIdx, colIdx); - const recvEl = td.querySelector('.recv') as HTMLElement; - const statusEl = td.querySelector('.status') as HTMLElement; - - recvEl.textContent = opts.recv; - statusEl.innerHTML = opts.statusHtml ?? ''; - td.className = opts.running ? 'cell running' : 'cell'; - td.title = opts.title ?? ''; - - if (opts.fill === undefined) { - td.style.backgroundColor = ''; - } else { - const alpha = Math.max(0, Math.min(1, opts.fill / MAX_FILL_COUNT)); - td.style.backgroundColor = `rgba(${FILL_RGB}, ${alpha.toFixed(3)})`; - } -} - -function setButtons(opts: { run: boolean; stop: boolean }) { - $('run').disabled = !opts.run; - $('stop').disabled = !opts.stop; -} - -// --------------------------------------------------------------------------- -// Networking helpers -// --------------------------------------------------------------------------- - -async function fetchToken( - identity: string, - roomName: string, -): Promise<{ token: string; url: string }> { - const response = await fetch('/api/get-token', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ identity, roomName }), - }); - if (!response.ok) { - throw new Error('Failed to fetch token'); - } - const data = await response.json(); - return { token: data.token, url: data.url }; -} - -async function setNetwork(preset: string): Promise { - const response = await fetch('/api/network-condition', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ preset }), - }); - if (!response.ok) { - const body = await response.text(); - throw new Error(`Failed to set network condition "${preset}": ${body}`); - } -} - -/** Waits until `room` sees a remote participant with the given identity (so sends can target it). */ -function waitForParticipant(room: Room, identity: string, timeoutMs = 15_000): Promise { - return new Promise((resolve, reject) => { - if (room.remoteParticipants.has(identity)) { - resolve(); - return; - } - const timer = setTimeout(() => { - room.off(RoomEvent.ParticipantConnected, onConnected); - reject(new Error(`timed out waiting for participant "${identity}"`)); - }, timeoutMs); - const onConnected = () => { - if (room.remoteParticipants.has(identity)) { - clearTimeout(timer); - room.off(RoomEvent.ParticipantConnected, onConnected); - resolve(); - } - }; - room.on(RoomEvent.ParticipantConnected, onConnected); - }); -} - -// --------------------------------------------------------------------------- -// Per-box connection lifecycle -// --------------------------------------------------------------------------- - -/** Connects a fresh sender/receiver pair into a new room and wires up the receiver metrics. */ -async function connectPair(stats: BoxStats): Promise { - const roomName = `ds-bench-${Math.random().toString(36).substring(7)}`; - receiverRoom = new Room(); - senderRoom = new Room(); - - // Register the receive handler before connecting so no stream is missed. The handler closes over - // this box's `stats` object, so post-snapshot stragglers can't bleed into the next box. - receiverRoom.registerTextStreamHandler(TOPIC, async (reader) => { - const attrs = reader.info.attributes ?? {}; - try { - const text = await reader.readAll(); - // One-way end-to-end latency — sender and receiver share this tab's clock. - const latency = Date.now() - Number(attrs.sendTs); - stats.recordReceived(latency, `${checksum(text)}` === attrs.checksum); - } catch { - // dropped/aborted mid-stream — not counted as received - } - }); - - const [senderToken, receiverToken] = await Promise.all([ - fetchToken(SENDER_IDENTITY, roomName), - fetchToken(RECEIVER_IDENTITY, roomName), - ]); - - await Promise.all([ - senderRoom.connect(senderToken.url, senderToken.token), - receiverRoom.connect(receiverToken.url, receiverToken.token), - ]); - - // The sender must see the receiver (with its advertised protocol) before sending, otherwise sends - // would fall back to the chunked path. - await waitForParticipant(senderRoom, RECEIVER_IDENTITY); -} - -async function disconnectPair(): Promise { - await Promise.allSettled([senderRoom?.disconnect(), receiverRoom?.disconnect()]); - senderRoom = null; - receiverRoom = null; -} - -// --------------------------------------------------------------------------- -// One box: fresh connect -> N concurrent senders for a fixed window -> disconnect -// --------------------------------------------------------------------------- - -async function runBox(rowIdx: number, colIdx: number, sizeBytes: number, label: string) { - const stats = new BoxStats(); - renderCell(rowIdx, colIdx, { recv: '…', running: true }); - - for (let i = 1; i <= 3; i += 1) { - try { - await connectPair(stats); - break; - } catch (err) { - log(`${label}: connect failed (try ${i}/3): ${String(err)}`); - await disconnectPair(); - if (i >= 3) { - renderCell(rowIdx, colIdx, { recv: 'conn err', fill: 0, title: String(err) }); - return; - } - } - } - - const callerLoop = async () => { - let resolve: (() => void) | null = null; - let timeoutHit = false; - setTimeout(() => { - timeoutHit = true; - resolve?.(); - }, BOX_DURATION_MS); - - const iteration = async () => { - const room = senderRoom; - if (!room) { - return; - } - - const payload = generatePayload(sizeBytes); - let promise; - if (STREAM_CHUNK_SIZE_BYTES > 0) { - // Stream payload data in STREAM_CHUNK_SIZE_BYTES chunks - const writer = await room.localParticipant.streamText({ - topic: TOPIC, - destinationIdentities: [RECEIVER_IDENTITY], - attributes: { sendTs: `${Date.now()}`, checksum: `${checksum(payload)}` }, - }); - for (let i = 0; i < Math.ceil(sizeBytes / STREAM_CHUNK_SIZE_BYTES); i += 1) { - await writer.write( - payload.slice(i * STREAM_CHUNK_SIZE_BYTES, (i + 1) * STREAM_CHUNK_SIZE_BYTES), - ); - } - promise = writer.close(); - } else { - // Send payload all in one go - promise = room.localParticipant.sendText(payload, { - topic: TOPIC, - destinationIdentities: [RECEIVER_IDENTITY], - attributes: { sendTs: `${Date.now()}`, checksum: `${checksum(payload)}` }, - }); - } - - promise - .then(async () => { - if (timeoutHit) { - return; - } - stats.recordSent(); - await iteration(); - }) - .catch(async (err) => { - if (timeoutHit) { - return; - } - - // Under a throttled link sends can fail transiently; back off briefly and keep going. - stats.recordSendError(errorKind(err)); - await sleep(50); - await iteration(); - }); - }; - iteration(); - - return new Promise((r) => { - resolve = r; - }); - }; - - await Promise.all(Array.from({ length: CONCURRENCY }, () => callerLoop())); - - // Let already-sent streams finish arriving before snapshotting and tearing down. - await sleep(DRAIN_MS); - await disconnectPair(); - - renderBoxStats(rowIdx, colIdx, label, stats, sizeBytes); -} - -/** Shortens an error into a stable bucket key for the error summary. */ -function errorKind(err: unknown): string { - if (err instanceof Error) { - return err.name && err.name !== 'Error' ? err.name : err.message.split('\n')[0].slice(0, 60); - } - return String(err).split('\n')[0].slice(0, 60); -} - -/** Renders the box cell (throughput + status line) and logs the full per-box summary. */ -function renderBoxStats( - rowIdx: number, - colIdx: number, - label: string, - stats: BoxStats, - sizeBytes: number, -) { - const elapsedSec = BOX_DURATION_MS / 1000; - const tput = stats.throughput(elapsedSec); - const kbPerSec = stats.bytesPerSec(sizeBytes, elapsedSec) / 1000; - const { sent, received, valid, mismatch, lost, sendErrors } = stats; - const p50 = stats.percentile(50); - const p95 = stats.percentile(95); - const p99 = stats.percentile(99); - const avg = stats.avgLatency; - const rate = stats.successRate; - - const mismatchTok = mismatch ? ` ✗${mismatch}` : ''; - const statusHtml = - `↑${sent} ✓${valid}${mismatchTok} ⊘${lost} · ${rate.toFixed(0)}%
` + - `p50 ${p50.toFixed(0)} p95 ${p95.toFixed(0)} p99 ${p99.toFixed(0)} ms`; - - renderCell(rowIdx, colIdx, { - recv: `${tput.toFixed(1)} ds/s`, - statusHtml, - fill: received, - title: - `sent ${sent}, received ${received}, valid ${valid}, mismatch ${mismatch}, ` + - `lost ${lost}, sendErrors ${sendErrors}`, - }); - - const errs = Object.keys(stats.errors).length ? ` · errors ${JSON.stringify(stats.errors)}` : ''; - log( - `${label}: sent ${sent} valid ${valid} (${rate.toFixed(1)}%) recv ${received} ✗${mismatch} ` + - `lost ${lost} sendErr ${sendErrors}`, - ); - log( - `${label}: lat avg ${avg.toFixed(1)} p50 ${p50.toFixed(1)} p95 ${p95.toFixed(1)} ` + - `p99 ${p99.toFixed(1)} ms · ${tput.toFixed(1)} ds/s · ${kbPerSec.toFixed(1)} KB/s${errs}`, - ); -} - -// --------------------------------------------------------------------------- -// Run / stop -// --------------------------------------------------------------------------- - -async function runBenchmark() { - if (running) { - return; - } - running = true; - setButtons({ run: false, stop: true }); - buildGrid(); - log( - `Starting benchmark: ${CONCURRENCY} concurrent senders, ${BOX_DURATION_MS / 1000}s per box, ` + - `fresh connection per box.`, - ); - - try { - for (let colIdx = 0; colIdx < PRESETS.length && running; colIdx += 1) { - const preset = PRESETS[colIdx]; - setStatus(`Setting network condition: ${preset.label}…`); - log(`Setting network condition: ${preset.label}`); - await setNetwork(preset.value); - - for (let rowIdx = 0; rowIdx < SIZES.length && running; rowIdx += 1) { - const size = SIZES[rowIdx]; - const label = `${preset.label} · ${size.label}`; - setStatus(`${label} — sending for ${BOX_DURATION_MS / 1000}s…`); - await runBox(rowIdx, colIdx, size.bytes, label); - } - } - log(running ? 'Benchmark complete.' : 'Benchmark stopped.'); - setStatus(running ? 'Benchmark complete' : 'Stopped'); - } catch (err) { - log(`Benchmark error: ${String(err)}`); - setStatus('Benchmark error'); - } finally { - // Always reset the conditioner and tear down any lingering connection. - try { - await setNetwork('off'); - } catch (err) { - log(`Failed to reset network: ${String(err)}`); - } - await disconnectPair(); - running = false; - setButtons({ run: true, stop: false }); - } -} - -async function stop() { - if (!running) { - return; - } - log('Stopping…'); - setStatus('Stopping…'); - running = false; - setButtons({ run: false, stop: false }); - - // Always reset the conditioner and tear down any lingering connection. - try { - await setNetwork('off'); - } catch (err) { - log(`Failed to reset network: ${String(err)}`); - } - - // Abort any in-flight box immediately; runBenchmark's finally handles the rest. - await disconnectPair(); -} - -// --------------------------------------------------------------------------- -// Wire up -// --------------------------------------------------------------------------- - -document.addEventListener('DOMContentLoaded', () => { - buildGrid(); - setButtons({ run: true, stop: false }); - $('run').addEventListener('click', runBenchmark); - $('stop').addEventListener('click', stop); -}); diff --git a/examples/data-stream-benchmark/index.html b/examples/data-stream-benchmark/index.html deleted file mode 100644 index 7706afe463..0000000000 --- a/examples/data-stream-benchmark/index.html +++ /dev/null @@ -1,36 +0,0 @@ - - - - - - LiveKit Data Stream Benchmark - - - -
-

LiveKit v2 Data Stream Benchmark

- -
- - -
- -
Idle
- -
- -

- Each box does a fresh connect → send for a fixed window with several concurrent senders → - disconnect. The large number is throughput (data streams received per - second). The status line below shows ↑sent ✓valid ✗mismatch ⊘lost · success % and - one-way latency percentiles (p50/p95/p99). Box fill scales with throughput; full per-box - stats are printed to the log. -

- -
- -
-
- - - diff --git a/examples/data-stream-benchmark/package.json b/examples/data-stream-benchmark/package.json deleted file mode 100644 index 1d2f1599b5..0000000000 --- a/examples/data-stream-benchmark/package.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "name": "livekit-data-stream-benchmark", - "version": "1.0.0", - "description": "Benchmark of LiveKit v2 data streams across network conditions and payload sizes", - "type": "module", - "scripts": { - "dev": "vite", - "build": "tsc && vite build", - "preview": "vite preview" - }, - "dependencies": { - "cors": "^2.8.5", - "dotenv": "^16.4.5", - "express": "^5.2.1", - "livekit-server-sdk": "^2.7.0", - "vite": "^5.4.21", - "vite-plugin-mix": "^0.4.0" - }, - "devDependencies": { - "@types/cors": "^2.8.17", - "@types/express": "^5.0.0", - "@types/node": "^20.0.0", - "concurrently": "^8.2.0", - "tsx": "^4.7.0", - "typescript": "^5.4.5" - } -} diff --git a/examples/data-stream-benchmark/payload.ts b/examples/data-stream-benchmark/payload.ts deleted file mode 100644 index 0408efb795..0000000000 --- a/examples/data-stream-benchmark/payload.ts +++ /dev/null @@ -1,68 +0,0 @@ -/** - * Realistic structured JSON test data for the data-stream benchmark. Each line is a self-contained - * JSON object resembling production payloads (user profiles, events, metrics, logs, etc.). All lines - * are ASCII so that one character equals one byte, which lets us slice payloads to an exact byte - * length without worrying about UTF-8 boundaries. - */ -const TEST_DATA_LINES: string[] = [ - '{"id":"usr_a1b2c3","name":"Alice Chen","email":"alice.chen@example.com","role":"engineer","department":"platform","projects":["livekit-core","media-pipeline","signaling"],"metrics":{"commits":342,"reviews":128,"deployments":57},"location":"San Francisco, CA","joined":"2022-03-15T08:30:00Z"}', - '{"event":"room.participant_joined","timestamp":"2025-01-15T14:22:33.456Z","room_sid":"RM_xK9mPq2nR4","participant_sid":"PA_j7hLw3vYm1","identity":"speaker-042","metadata":{"display_name":"Dr. Sarah Mitchell","avatar_url":"https://cdn.example.com/avatars/sm042.jpg","hand_raised":false}}', - '{"sensor_id":"temp-rack-07b","readings":[{"ts":1705312800,"value":23.4,"unit":"celsius"},{"ts":1705312860,"value":23.6,"unit":"celsius"},{"ts":1705312920,"value":24.1,"unit":"celsius"},{"ts":1705312980,"value":23.8,"unit":"celsius"}],"status":"nominal","location":"datacenter-west-3"}', - '{"order_id":"ORD-2025-00847","customer":{"id":"cust_9f8e7d","name":"Bob Williams","tier":"premium"},"items":[{"sku":"WDG-1042","name":"Wireless Adapter Pro","qty":2,"price":49.99},{"sku":"CBL-3001","name":"USB-C Cable 2m","qty":5,"price":12.99}],"total":164.93,"currency":"USD","status":"processing"}', - '{"trace_id":"abc123def456","spans":[{"name":"http.request","duration_ms":245,"status":"ok","attributes":{"http.method":"POST","http.url":"/api/v2/rooms","http.status_code":201}},{"name":"db.query","duration_ms":12,"status":"ok","attributes":{"db.system":"postgresql","db.statement":"INSERT INTO rooms"}}]}', - '{"log_level":"warn","service":"media-router","instance":"mr-us-east-07","message":"Track subscription delayed due to network congestion","context":{"room_sid":"RM_pQ8nL2mK5x","track_sid":"TR_w4jR7vN9y3","participant_sid":"PA_k2mX5bH8r1","delay_ms":1847,"retry_count":3,"bandwidth_estimate_bps":2450000}}', - '{"config":{"video":{"codecs":["VP8","H264","AV1"],"simulcast":{"enabled":true,"layers":[{"rid":"f","maxBitrate":2500000,"maxFramerate":30},{"rid":"h","maxBitrate":800000,"maxFramerate":15},{"rid":"q","maxBitrate":200000,"maxFramerate":7}]},"dynacast":true},"audio":{"codecs":["opus"],"dtx":true,"red":true,"stereo":false}}}', - '{"benchmark":{"test":"data-stream-throughput","iteration":1547,"payload_bytes":15360,"latency_ms":23.7,"path":"inline","timestamp":"2025-06-20T10:15:33.891Z","sender":"bench-sender","receiver":"bench-receiver","room":"benchmark-room-8f3a"}}', - '{"user_id":"u_7k3m9p","session":{"id":"sess_abc123","started":"2025-01-15T09:00:00Z","duration_minutes":47,"pages_viewed":12,"actions":[{"type":"click","target":"#start-call","ts":1705308120},{"type":"input","target":"#chat-message","ts":1705308245},{"type":"click","target":"#share-screen","ts":1705308390}],"device":{"browser":"Chrome 121","os":"macOS 14.2","screen":"2560x1440"}}}', - '{"pipeline_id":"pipe_rtc_042","stages":[{"name":"capture","codec":"VP8","resolution":"1920x1080","fps":30,"bitrate_kbps":2500},{"name":"encode","profile":"constrained-baseline","hardware_accel":true,"latency_ms":4.2},{"name":"packetize","mtu":1200,"fec_enabled":true,"nack_enabled":true},{"name":"transport","protocol":"UDP","ice_candidates":3,"dtls_setup":"actpass"}]}', - '{"cluster":{"id":"lk-us-east-1","region":"us-east-1","nodes":[{"id":"node-01","type":"media","status":"healthy","load":0.67,"rooms":42,"participants":318,"cpu_pct":54.2,"mem_pct":71.8},{"id":"node-02","type":"media","status":"healthy","load":0.43,"rooms":31,"participants":201,"cpu_pct":38.1,"mem_pct":55.4}],"total_rooms":73,"total_participants":519}}', - '{"deployment":{"id":"deploy_20250115_003","service":"livekit-server","version":"1.8.2","environment":"production","region":"eu-west-1","status":"completed","started_at":"2025-01-15T03:00:00Z","completed_at":"2025-01-15T03:12:47Z","changes":["fix: ice restart race condition","feat: improved simulcast layer selection","perf: reduce memory allocation in media forwarding"],"rollback_available":true,"health_check":"passing"}}', - '{"analytics":{"room_id":"RM_daily_standup_042","period":"2025-01-15T09:00:00Z/2025-01-15T09:30:00Z","participants":{"total":8,"max_concurrent":7,"avg_duration_minutes":22.4},"media":{"audio":{"total_minutes":156.8,"avg_bitrate_kbps":32,"packet_loss_pct":0.02},"video":{"total_minutes":134.2,"avg_bitrate_kbps":1850,"packet_loss_pct":0.08,"avg_fps":28.7}},"quality_score":4.7}}', - '{"ticket":{"id":"TICKET-8472","title":"Intermittent audio dropout in large rooms","priority":"high","status":"in_progress","assignee":"eng-media-team","reporter":"support-agent-12","created":"2025-01-14T16:30:00Z","updated":"2025-01-15T11:22:00Z","labels":["audio","production","p1"],"comments_count":7,"related_incidents":["INC-2025-0042","INC-2025-0039"]}}', -]; - -let cachedBase: string | null = null; - -/** - * Builds (once) a large ASCII string by repeatedly joining the test data lines until it comfortably - * exceeds the largest payload we benchmark, so payloads can be sliced out of it. - */ -function getBase(): string { - if (cachedBase !== null) { - return cachedBase; - } - const minLength = 1_100_000; - const parts: string[] = []; - let length = 0; - let idx = 0; - while (length < minLength) { - const line = TEST_DATA_LINES[idx % TEST_DATA_LINES.length]; - parts.push(line); - length += line.length + 1; // +1 for the '\n' join separator - idx += 1; - } - cachedBase = parts.join('\n'); - return cachedBase; -} - -/** - * Returns a JSON-ish payload of exactly `targetBytes` bytes, sliced from a random offset of the - * shared base string (ASCII, so byte length === character length). - */ -export function generatePayload(targetBytes: number): string { - const base = getBase(); - if (targetBytes >= base.length) { - throw new Error(`requested payload (${targetBytes}) larger than base data (${base.length})`); - } - const maxOffset = base.length - targetBytes; - const offset = Math.floor(Math.random() * maxOffset); - return base.slice(offset, offset + targetBytes); -} - -export function checksum(str: string) { - let sum = 0; - for (let i = 0; i < str.length; i += 1) { - sum += str.charCodeAt(i); - } - return sum; -} diff --git a/examples/data-stream-benchmark/pnpm-lock.yaml b/examples/data-stream-benchmark/pnpm-lock.yaml deleted file mode 100644 index 9c96ebc90e..0000000000 --- a/examples/data-stream-benchmark/pnpm-lock.yaml +++ /dev/null @@ -1,2462 +0,0 @@ -lockfileVersion: '9.0' - -settings: - autoInstallPeers: true - excludeLinksFromLockfile: false - -importers: - - .: - dependencies: - cors: - specifier: ^2.8.5 - version: 2.8.6 - dotenv: - specifier: ^16.4.5 - version: 16.6.1 - express: - specifier: ^5.2.1 - version: 5.2.1 - livekit-server-sdk: - specifier: ^2.7.0 - version: 2.15.0 - vite: - specifier: ^5.4.21 - version: 5.4.21(@types/node@20.19.41) - vite-plugin-mix: - specifier: ^0.4.0 - version: 0.4.0(vite@5.4.21(@types/node@20.19.41)) - devDependencies: - '@types/cors': - specifier: ^2.8.17 - version: 2.8.19 - '@types/express': - specifier: ^5.0.0 - version: 5.0.6 - '@types/node': - specifier: ^20.0.0 - version: 20.19.41 - concurrently: - specifier: ^8.2.0 - version: 8.2.2 - tsx: - specifier: ^4.7.0 - version: 4.21.0 - typescript: - specifier: ^5.4.5 - version: 5.9.3 - -packages: - - '@babel/runtime@7.28.6': - resolution: {integrity: sha512-05WQkdpL9COIMz4LjTxGpPNCdlpyimKppYNoJ5Di5EUObifl8t4tuLuUBBZEpoLYOmfvIWrsp9fCl0HoPRVTdA==} - engines: {node: '>=6.9.0'} - - '@bufbuild/protobuf@1.10.1': - resolution: {integrity: sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==} - - '@esbuild/aix-ppc64@0.21.5': - resolution: {integrity: sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [aix] - - '@esbuild/aix-ppc64@0.27.3': - resolution: {integrity: sha512-9fJMTNFTWZMh5qwrBItuziu834eOCUcEqymSH7pY+zoMVEZg3gcPuBNxH1EvfVYe9h0x/Ptw8KBzv7qxb7l8dg==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [aix] - - '@esbuild/android-arm64@0.21.5': - resolution: {integrity: sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==} - engines: {node: '>=12'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm64@0.27.3': - resolution: {integrity: sha512-YdghPYUmj/FX2SYKJ0OZxf+iaKgMsKHVPF1MAq/P8WirnSpCStzKJFjOjzsW0QQ7oIAiccHdcqjbHmJxRb/dmg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [android] - - '@esbuild/android-arm@0.21.5': - resolution: {integrity: sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==} - engines: {node: '>=12'} - cpu: [arm] - os: [android] - - '@esbuild/android-arm@0.27.3': - resolution: {integrity: sha512-i5D1hPY7GIQmXlXhs2w8AWHhenb00+GxjxRncS2ZM7YNVGNfaMxgzSGuO8o8SJzRc/oZwU2bcScvVERk03QhzA==} - engines: {node: '>=18'} - cpu: [arm] - os: [android] - - '@esbuild/android-x64@0.21.5': - resolution: {integrity: sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==} - engines: {node: '>=12'} - cpu: [x64] - os: [android] - - '@esbuild/android-x64@0.27.3': - resolution: {integrity: sha512-IN/0BNTkHtk8lkOM8JWAYFg4ORxBkZQf9zXiEOfERX/CzxW3Vg1ewAhU7QSWQpVIzTW+b8Xy+lGzdYXV6UZObQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [android] - - '@esbuild/darwin-arm64@0.21.5': - resolution: {integrity: sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==} - engines: {node: '>=12'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-arm64@0.27.3': - resolution: {integrity: sha512-Re491k7ByTVRy0t3EKWajdLIr0gz2kKKfzafkth4Q8A5n1xTHrkqZgLLjFEHVD+AXdUGgQMq+Godfq45mGpCKg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [darwin] - - '@esbuild/darwin-x64@0.21.5': - resolution: {integrity: sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==} - engines: {node: '>=12'} - cpu: [x64] - os: [darwin] - - '@esbuild/darwin-x64@0.27.3': - resolution: {integrity: sha512-vHk/hA7/1AckjGzRqi6wbo+jaShzRowYip6rt6q7VYEDX4LEy1pZfDpdxCBnGtl+A5zq8iXDcyuxwtv3hNtHFg==} - engines: {node: '>=18'} - cpu: [x64] - os: [darwin] - - '@esbuild/freebsd-arm64@0.21.5': - resolution: {integrity: sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==} - engines: {node: '>=12'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-arm64@0.27.3': - resolution: {integrity: sha512-ipTYM2fjt3kQAYOvo6vcxJx3nBYAzPjgTCk7QEgZG8AUO3ydUhvelmhrbOheMnGOlaSFUoHXB6un+A7q4ygY9w==} - engines: {node: '>=18'} - cpu: [arm64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.21.5': - resolution: {integrity: sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [freebsd] - - '@esbuild/freebsd-x64@0.27.3': - resolution: {integrity: sha512-dDk0X87T7mI6U3K9VjWtHOXqwAMJBNN2r7bejDsc+j03SEjtD9HrOl8gVFByeM0aJksoUuUVU9TBaZa2rgj0oA==} - engines: {node: '>=18'} - cpu: [x64] - os: [freebsd] - - '@esbuild/linux-arm64@0.21.5': - resolution: {integrity: sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==} - engines: {node: '>=12'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm64@0.27.3': - resolution: {integrity: sha512-sZOuFz/xWnZ4KH3YfFrKCf1WyPZHakVzTiqji3WDc0BCl2kBwiJLCXpzLzUBLgmp4veFZdvN5ChW4Eq/8Fc2Fg==} - engines: {node: '>=18'} - cpu: [arm64] - os: [linux] - - '@esbuild/linux-arm@0.21.5': - resolution: {integrity: sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==} - engines: {node: '>=12'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-arm@0.27.3': - resolution: {integrity: sha512-s6nPv2QkSupJwLYyfS+gwdirm0ukyTFNl3KTgZEAiJDd+iHZcbTPPcWCcRYH+WlNbwChgH2QkE9NSlNrMT8Gfw==} - engines: {node: '>=18'} - cpu: [arm] - os: [linux] - - '@esbuild/linux-ia32@0.21.5': - resolution: {integrity: sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==} - engines: {node: '>=12'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-ia32@0.27.3': - resolution: {integrity: sha512-yGlQYjdxtLdh0a3jHjuwOrxQjOZYD/C9PfdbgJJF3TIZWnm/tMd/RcNiLngiu4iwcBAOezdnSLAwQDPqTmtTYg==} - engines: {node: '>=18'} - cpu: [ia32] - os: [linux] - - '@esbuild/linux-loong64@0.21.5': - resolution: {integrity: sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==} - engines: {node: '>=12'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-loong64@0.27.3': - resolution: {integrity: sha512-WO60Sn8ly3gtzhyjATDgieJNet/KqsDlX5nRC5Y3oTFcS1l0KWba+SEa9Ja1GfDqSF1z6hif/SkpQJbL63cgOA==} - engines: {node: '>=18'} - cpu: [loong64] - os: [linux] - - '@esbuild/linux-mips64el@0.21.5': - resolution: {integrity: sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==} - engines: {node: '>=12'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-mips64el@0.27.3': - resolution: {integrity: sha512-APsymYA6sGcZ4pD6k+UxbDjOFSvPWyZhjaiPyl/f79xKxwTnrn5QUnXR5prvetuaSMsb4jgeHewIDCIWljrSxw==} - engines: {node: '>=18'} - cpu: [mips64el] - os: [linux] - - '@esbuild/linux-ppc64@0.21.5': - resolution: {integrity: sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==} - engines: {node: '>=12'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-ppc64@0.27.3': - resolution: {integrity: sha512-eizBnTeBefojtDb9nSh4vvVQ3V9Qf9Df01PfawPcRzJH4gFSgrObw+LveUyDoKU3kxi5+9RJTCWlj4FjYXVPEA==} - engines: {node: '>=18'} - cpu: [ppc64] - os: [linux] - - '@esbuild/linux-riscv64@0.21.5': - resolution: {integrity: sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==} - engines: {node: '>=12'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-riscv64@0.27.3': - resolution: {integrity: sha512-3Emwh0r5wmfm3ssTWRQSyVhbOHvqegUDRd0WhmXKX2mkHJe1SFCMJhagUleMq+Uci34wLSipf8Lagt4LlpRFWQ==} - engines: {node: '>=18'} - cpu: [riscv64] - os: [linux] - - '@esbuild/linux-s390x@0.21.5': - resolution: {integrity: sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==} - engines: {node: '>=12'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-s390x@0.27.3': - resolution: {integrity: sha512-pBHUx9LzXWBc7MFIEEL0yD/ZVtNgLytvx60gES28GcWMqil8ElCYR4kvbV2BDqsHOvVDRrOxGySBM9Fcv744hw==} - engines: {node: '>=18'} - cpu: [s390x] - os: [linux] - - '@esbuild/linux-x64@0.21.5': - resolution: {integrity: sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==} - engines: {node: '>=12'} - cpu: [x64] - os: [linux] - - '@esbuild/linux-x64@0.27.3': - resolution: {integrity: sha512-Czi8yzXUWIQYAtL/2y6vogER8pvcsOsk5cpwL4Gk5nJqH5UZiVByIY8Eorm5R13gq+DQKYg0+JyQoytLQas4dA==} - engines: {node: '>=18'} - cpu: [x64] - os: [linux] - - '@esbuild/netbsd-arm64@0.27.3': - resolution: {integrity: sha512-sDpk0RgmTCR/5HguIZa9n9u+HVKf40fbEUt+iTzSnCaGvY9kFP0YKBWZtJaraonFnqef5SlJ8/TiPAxzyS+UoA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.21.5': - resolution: {integrity: sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==} - engines: {node: '>=12'} - cpu: [x64] - os: [netbsd] - - '@esbuild/netbsd-x64@0.27.3': - resolution: {integrity: sha512-P14lFKJl/DdaE00LItAukUdZO5iqNH7+PjoBm+fLQjtxfcfFE20Xf5CrLsmZdq5LFFZzb5JMZ9grUwvtVYzjiA==} - engines: {node: '>=18'} - cpu: [x64] - os: [netbsd] - - '@esbuild/openbsd-arm64@0.27.3': - resolution: {integrity: sha512-AIcMP77AvirGbRl/UZFTq5hjXK+2wC7qFRGoHSDrZ5v5b8DK/GYpXW3CPRL53NkvDqb9D+alBiC/dV0Fb7eJcw==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.21.5': - resolution: {integrity: sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==} - engines: {node: '>=12'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openbsd-x64@0.27.3': - resolution: {integrity: sha512-DnW2sRrBzA+YnE70LKqnM3P+z8vehfJWHXECbwBmH/CU51z6FiqTQTHFenPlHmo3a8UgpLyH3PT+87OViOh1AQ==} - engines: {node: '>=18'} - cpu: [x64] - os: [openbsd] - - '@esbuild/openharmony-arm64@0.27.3': - resolution: {integrity: sha512-NinAEgr/etERPTsZJ7aEZQvvg/A6IsZG/LgZy+81wON2huV7SrK3e63dU0XhyZP4RKGyTm7aOgmQk0bGp0fy2g==} - engines: {node: '>=18'} - cpu: [arm64] - os: [openharmony] - - '@esbuild/sunos-x64@0.21.5': - resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} - engines: {node: '>=12'} - cpu: [x64] - os: [sunos] - - '@esbuild/sunos-x64@0.27.3': - resolution: {integrity: sha512-PanZ+nEz+eWoBJ8/f8HKxTTD172SKwdXebZ0ndd953gt1HRBbhMsaNqjTyYLGLPdoWHy4zLU7bDVJztF5f3BHA==} - engines: {node: '>=18'} - cpu: [x64] - os: [sunos] - - '@esbuild/win32-arm64@0.21.5': - resolution: {integrity: sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==} - engines: {node: '>=12'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-arm64@0.27.3': - resolution: {integrity: sha512-B2t59lWWYrbRDw/tjiWOuzSsFh1Y/E95ofKz7rIVYSQkUYBjfSgf6oeYPNWHToFRr2zx52JKApIcAS/D5TUBnA==} - engines: {node: '>=18'} - cpu: [arm64] - os: [win32] - - '@esbuild/win32-ia32@0.21.5': - resolution: {integrity: sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==} - engines: {node: '>=12'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-ia32@0.27.3': - resolution: {integrity: sha512-QLKSFeXNS8+tHW7tZpMtjlNb7HKau0QDpwm49u0vUp9y1WOF+PEzkU84y9GqYaAVW8aH8f3GcBck26jh54cX4Q==} - engines: {node: '>=18'} - cpu: [ia32] - os: [win32] - - '@esbuild/win32-x64@0.21.5': - resolution: {integrity: sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==} - engines: {node: '>=12'} - cpu: [x64] - os: [win32] - - '@esbuild/win32-x64@0.27.3': - resolution: {integrity: sha512-4uJGhsxuptu3OcpVAzli+/gWusVGwZZHTlS63hh++ehExkVT8SgiEf7/uC/PclrPPkLhZqGgCTjd0VWLo6xMqA==} - engines: {node: '>=18'} - cpu: [x64] - os: [win32] - - '@livekit/protocol@1.44.0': - resolution: {integrity: sha512-/vfhDUGcUKO8Q43r6i+5FrDhl5oZjm/X3U4x2Iciqvgn5C8qbj+57YPcWSJ1kyIZm5Cm6AV2nAPjMm3ETD/iyg==} - - '@rollup/rollup-android-arm-eabi@4.59.0': - resolution: {integrity: sha512-upnNBkA6ZH2VKGcBj9Fyl9IGNPULcjXRlg0LLeaioQWueH30p6IXtJEbKAgvyv+mJaMxSm1l6xwDXYjpEMiLMg==} - cpu: [arm] - os: [android] - - '@rollup/rollup-android-arm64@4.59.0': - resolution: {integrity: sha512-hZ+Zxj3SySm4A/DylsDKZAeVg0mvi++0PYVceVyX7hemkw7OreKdCvW2oQ3T1FMZvCaQXqOTHb8qmBShoqk69Q==} - cpu: [arm64] - os: [android] - - '@rollup/rollup-darwin-arm64@4.59.0': - resolution: {integrity: sha512-W2Psnbh1J8ZJw0xKAd8zdNgF9HRLkdWwwdWqubSVk0pUuQkoHnv7rx4GiF9rT4t5DIZGAsConRE3AxCdJ4m8rg==} - cpu: [arm64] - os: [darwin] - - '@rollup/rollup-darwin-x64@4.59.0': - resolution: {integrity: sha512-ZW2KkwlS4lwTv7ZVsYDiARfFCnSGhzYPdiOU4IM2fDbL+QGlyAbjgSFuqNRbSthybLbIJ915UtZBtmuLrQAT/w==} - cpu: [x64] - os: [darwin] - - '@rollup/rollup-freebsd-arm64@4.59.0': - resolution: {integrity: sha512-EsKaJ5ytAu9jI3lonzn3BgG8iRBjV4LxZexygcQbpiU0wU0ATxhNVEpXKfUa0pS05gTcSDMKpn3Sx+QB9RlTTA==} - cpu: [arm64] - os: [freebsd] - - '@rollup/rollup-freebsd-x64@4.59.0': - resolution: {integrity: sha512-d3DuZi2KzTMjImrxoHIAODUZYoUUMsuUiY4SRRcJy6NJoZ6iIqWnJu9IScV9jXysyGMVuW+KNzZvBLOcpdl3Vg==} - cpu: [x64] - os: [freebsd] - - '@rollup/rollup-linux-arm-gnueabihf@4.59.0': - resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} - cpu: [arm] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-arm-musleabihf@4.59.0': - resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} - cpu: [arm] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-arm64-gnu@4.59.0': - resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-arm64-musl@4.59.0': - resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-loong64-gnu@4.59.0': - resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} - cpu: [loong64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-loong64-musl@4.59.0': - resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} - cpu: [loong64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-ppc64-gnu@4.59.0': - resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} - cpu: [ppc64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-ppc64-musl@4.59.0': - resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} - cpu: [ppc64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-riscv64-gnu@4.59.0': - resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} - cpu: [riscv64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-riscv64-musl@4.59.0': - resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} - cpu: [riscv64] - os: [linux] - libc: [musl] - - '@rollup/rollup-linux-s390x-gnu@4.59.0': - resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} - cpu: [s390x] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-x64-gnu@4.59.0': - resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@rollup/rollup-linux-x64-musl@4.59.0': - resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} - cpu: [x64] - os: [linux] - libc: [musl] - - '@rollup/rollup-openbsd-x64@4.59.0': - resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} - cpu: [x64] - os: [openbsd] - - '@rollup/rollup-openharmony-arm64@4.59.0': - resolution: {integrity: sha512-tt9KBJqaqp5i5HUZzoafHZX8b5Q2Fe7UjYERADll83O4fGqJ49O1FsL6LpdzVFQcpwvnyd0i+K/VSwu/o/nWlA==} - cpu: [arm64] - os: [openharmony] - - '@rollup/rollup-win32-arm64-msvc@4.59.0': - resolution: {integrity: sha512-V5B6mG7OrGTwnxaNUzZTDTjDS7F75PO1ae6MJYdiMu60sq0CqN5CVeVsbhPxalupvTX8gXVSU9gq+Rx1/hvu6A==} - cpu: [arm64] - os: [win32] - - '@rollup/rollup-win32-ia32-msvc@4.59.0': - resolution: {integrity: sha512-UKFMHPuM9R0iBegwzKF4y0C4J9u8C6MEJgFuXTBerMk7EJ92GFVFYBfOZaSGLu6COf7FxpQNqhNS4c4icUPqxA==} - cpu: [ia32] - os: [win32] - - '@rollup/rollup-win32-x64-gnu@4.59.0': - resolution: {integrity: sha512-laBkYlSS1n2L8fSo1thDNGrCTQMmxjYY5G0WFWjFFYZkKPjsMBsgJfGf4TLxXrF6RyhI60L8TMOjBMvXiTcxeA==} - cpu: [x64] - os: [win32] - - '@rollup/rollup-win32-x64-msvc@4.59.0': - resolution: {integrity: sha512-2HRCml6OztYXyJXAvdDXPKcawukWY2GpR5/nxKp4iBgiO3wcoEGkAaqctIbZcNB6KlUQBIqt8VYkNSj2397EfA==} - cpu: [x64] - os: [win32] - - '@types/body-parser@1.19.6': - resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==} - - '@types/connect@3.4.38': - resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} - - '@types/cors@2.8.19': - resolution: {integrity: sha512-mFNylyeyqN93lfe/9CSxOGREz8cpzAhH+E93xJ4xWQf62V8sQ/24reV2nyzUWM6H6Xji+GGHpkbLe7pVoUEskg==} - - '@types/estree@1.0.8': - resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} - - '@types/express-serve-static-core@5.1.1': - resolution: {integrity: sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A==} - - '@types/express@5.0.6': - resolution: {integrity: sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==} - - '@types/http-errors@2.0.5': - resolution: {integrity: sha512-r8Tayk8HJnX0FztbZN7oVqGccWgw98T/0neJphO91KkmOzug1KkofZURD4UaD5uH8AqcFLfdPErnBod0u71/qg==} - - '@types/node@20.19.41': - resolution: {integrity: sha512-ECymXOukMnOoVkC2bb1Vc/w/836DXncOg5m8Xj1RH7xSHZJWNYY6Zh7EH477vcnD5egKNNfy2RpNOmuChhFPgQ==} - - '@types/qs@6.14.0': - resolution: {integrity: sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ==} - - '@types/range-parser@1.2.7': - resolution: {integrity: sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==} - - '@types/send@1.2.1': - resolution: {integrity: sha512-arsCikDvlU99zl1g69TcAB3mzZPpxgw0UQnaHeC1Nwb015xp8bknZv5rIfri9xTOcMuaVgvabfIRA7PSZVuZIQ==} - - '@types/serve-static@2.2.0': - resolution: {integrity: sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==} - - '@vercel/nft@0.10.1': - resolution: {integrity: sha512-xhINCdohfeWg/70QLs3De/rfNFcO2+Sw4tL9oqgFl4zQzhogT3q0MjH6Hda5uM2KuFGndRPs6VkKJphAhWmymg==} - hasBin: true - - abbrev@1.1.1: - resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} - - accepts@2.0.0: - resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} - engines: {node: '>= 0.6'} - - acorn-class-fields@1.0.0: - resolution: {integrity: sha512-l+1FokF34AeCXGBHkrXFmml9nOIRI+2yBnBpO5MaVAaTIJ96irWLtcCxX+7hAp6USHFCe+iyyBB4ZhxV807wmA==} - engines: {node: '>=4.8.2'} - peerDependencies: - acorn: ^6 || ^7 || ^8 - - acorn-private-class-elements@1.0.0: - resolution: {integrity: sha512-zYNcZtxKgVCg1brS39BEou86mIao1EV7eeREG+6WMwKbuYTeivRRs6S2XdWnboRde6G9wKh2w+WBydEyJsJ6mg==} - engines: {node: '>=4.8.2'} - peerDependencies: - acorn: ^6.1.0 || ^7 || ^8 - - acorn-static-class-features@1.0.0: - resolution: {integrity: sha512-XZJECjbmMOKvMHiNzbiPXuXpLAJfN3dAKtfIYbk1eHiWdsutlek+gS7ND4B8yJ3oqvHo1NxfafnezVmq7NXK0A==} - engines: {node: '>=4.8.2'} - peerDependencies: - acorn: ^6.1.0 || ^7 || ^8 - - acorn@8.16.0: - resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} - engines: {node: '>=0.4.0'} - hasBin: true - - ansi-regex@2.1.1: - resolution: {integrity: sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==} - engines: {node: '>=0.10.0'} - - ansi-regex@5.0.1: - resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} - engines: {node: '>=8'} - - ansi-styles@4.3.0: - resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} - engines: {node: '>=8'} - - aproba@1.2.0: - resolution: {integrity: sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==} - - are-we-there-yet@1.1.7: - resolution: {integrity: sha512-nxwy40TuMiUGqMyRHgCSWZ9FM4VAoRP4xUYSTv5ImRog+h9yISPbVH7H8fASCIzYn9wlEv4zvFL7uKDMCFQm3g==} - deprecated: This package is no longer supported. - - balanced-match@1.0.2: - resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} - - bindings@1.5.0: - resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} - - body-parser@2.2.2: - resolution: {integrity: sha512-oP5VkATKlNwcgvxi0vM0p/D3n2C3EReYVX+DNYs5TjZFn/oQt2j+4sVJtSMr18pdRr8wjTcBl6LoV+FUwzPmNA==} - engines: {node: '>=18'} - - brace-expansion@1.1.12: - resolution: {integrity: sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==} - - braces@3.0.3: - resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} - engines: {node: '>=8'} - - bytes@3.1.2: - resolution: {integrity: sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==} - engines: {node: '>= 0.8'} - - call-bind-apply-helpers@1.0.2: - resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} - engines: {node: '>= 0.4'} - - call-bound@1.0.4: - resolution: {integrity: sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==} - engines: {node: '>= 0.4'} - - camelcase-keys@9.1.3: - resolution: {integrity: sha512-Rircqi9ch8AnZscQcsA1C47NFdaO3wukpmIRzYcDOrmvgt78hM/sj5pZhZNec2NM12uk5vTwRHZ4anGcrC4ZTg==} - engines: {node: '>=16'} - - camelcase@8.0.0: - resolution: {integrity: sha512-8WB3Jcas3swSvjIeA2yvCJ+Miyz5l1ZmB6HFb9R1317dt9LCQoswg/BGrmAmkWVEszSrrg4RwmO46qIm2OEnSA==} - engines: {node: '>=16'} - - chalk@4.1.2: - resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} - engines: {node: '>=10'} - - chownr@1.1.4: - resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} - - cliui@8.0.1: - resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} - engines: {node: '>=12'} - - code-point-at@1.1.0: - resolution: {integrity: sha512-RpAVKQA5T63xEj6/giIbUEtZwJ4UFIc3ZtvEkiaUERylqe8xb5IvqcgOurZLahv93CLKfxcw5YI+DZcUBRyLXA==} - engines: {node: '>=0.10.0'} - - color-convert@2.0.1: - resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} - engines: {node: '>=7.0.0'} - - color-name@1.1.4: - resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} - - concat-map@0.0.1: - resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} - - concurrently@8.2.2: - resolution: {integrity: sha512-1dP4gpXFhei8IOtlXRE/T/4H88ElHgTiUzh71YUmtjTEHMSRS2Z/fgOxHSxxusGHogsRfxNq1vyAwxSC+EVyDg==} - engines: {node: ^14.13.0 || >=16.0.0} - hasBin: true - - console-control-strings@1.1.0: - resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} - - content-disposition@1.0.1: - resolution: {integrity: sha512-oIXISMynqSqm241k6kcQ5UwttDILMK4BiurCfGEREw6+X9jkkpEe5T9FZaApyLGGOnFuyMWZpdolTXMtvEJ08Q==} - engines: {node: '>=18'} - - content-type@1.0.5: - resolution: {integrity: sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==} - engines: {node: '>= 0.6'} - - cookie-signature@1.2.2: - resolution: {integrity: sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==} - engines: {node: '>=6.6.0'} - - cookie@0.7.2: - resolution: {integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==} - engines: {node: '>= 0.6'} - - core-util-is@1.0.3: - resolution: {integrity: sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==} - - cors@2.8.6: - resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==} - engines: {node: '>= 0.10'} - - date-fns@2.30.0: - resolution: {integrity: sha512-fnULvOpxnC5/Vg3NCiWelDsLiUc9bRwAPs/+LfTLNvetFCtCTN+yQz15C/fs4AwX1R9K5GLtLfn8QW+dWisaAw==} - engines: {node: '>=0.11'} - - debug@3.2.7: - resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - - debug@4.4.3: - resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} - engines: {node: '>=6.0'} - peerDependencies: - supports-color: '*' - peerDependenciesMeta: - supports-color: - optional: true - - deep-extend@0.6.0: - resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} - engines: {node: '>=4.0.0'} - - delegates@1.0.0: - resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} - - depd@2.0.0: - resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} - engines: {node: '>= 0.8'} - - detect-libc@1.0.3: - resolution: {integrity: sha512-pGjwhsmsp4kL2RTz08wcOlGN83otlqHeD/Z5T8GXZB+/YcpQ/dgo+lbU8ZsGxV0HIvqqxo9l7mqYwyYMD9bKDg==} - engines: {node: '>=0.10'} - hasBin: true - - dotenv@16.6.1: - resolution: {integrity: sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==} - engines: {node: '>=12'} - - dunder-proto@1.0.1: - resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} - engines: {node: '>= 0.4'} - - ee-first@1.1.1: - resolution: {integrity: sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==} - - emoji-regex@8.0.0: - resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} - - encodeurl@2.0.0: - resolution: {integrity: sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==} - engines: {node: '>= 0.8'} - - es-define-property@1.0.1: - resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} - engines: {node: '>= 0.4'} - - es-errors@1.3.0: - resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} - engines: {node: '>= 0.4'} - - es-object-atoms@1.1.1: - resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} - engines: {node: '>= 0.4'} - - esbuild@0.21.5: - resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} - engines: {node: '>=12'} - hasBin: true - - esbuild@0.27.3: - resolution: {integrity: sha512-8VwMnyGCONIs6cWue2IdpHxHnAjzxnw2Zr7MkVxB2vjmQ2ivqGFb4LEG3SMnv0Gb2F/G/2yA8zUaiL1gywDCCg==} - engines: {node: '>=18'} - hasBin: true - - escalade@3.2.0: - resolution: {integrity: sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==} - engines: {node: '>=6'} - - escape-html@1.0.3: - resolution: {integrity: sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==} - - estree-walker@0.6.1: - resolution: {integrity: sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==} - - etag@1.8.1: - resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} - engines: {node: '>= 0.6'} - - express@5.2.1: - resolution: {integrity: sha512-hIS4idWWai69NezIdRt2xFVofaF4j+6INOpJlVOLDO8zXGpUVEVzIYk12UUi2JzjEzWL3IOAxcTubgz9Po0yXw==} - engines: {node: '>= 18'} - - file-uri-to-path@1.0.0: - resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} - - fill-range@7.1.1: - resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} - engines: {node: '>=8'} - - finalhandler@2.1.1: - resolution: {integrity: sha512-S8KoZgRZN+a5rNwqTxlZZePjT/4cnm0ROV70LedRHZ0p8u9fRID0hJUZQpkKLzro8LfmC8sx23bY6tVNxv8pQA==} - engines: {node: '>= 18.0.0'} - - forwarded@0.2.0: - resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} - engines: {node: '>= 0.6'} - - fresh@2.0.0: - resolution: {integrity: sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==} - engines: {node: '>= 0.8'} - - fs-minipass@1.2.7: - resolution: {integrity: sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==} - - fs.realpath@1.0.0: - resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} - - fsevents@2.3.3: - resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} - engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} - os: [darwin] - - function-bind@1.1.2: - resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} - - gauge@2.7.4: - resolution: {integrity: sha512-14x4kjc6lkD3ltw589k0NrPD6cCNTD6CWoVUNpB85+DrtONoZn+Rug6xZU5RvSC4+TZPxA5AnBibQYAvZn41Hg==} - deprecated: This package is no longer supported. - - get-caller-file@2.0.5: - resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} - engines: {node: 6.* || 8.* || >= 10.*} - - get-intrinsic@1.3.0: - resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} - engines: {node: '>= 0.4'} - - get-proto@1.0.1: - resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} - engines: {node: '>= 0.4'} - - get-tsconfig@4.13.6: - resolution: {integrity: sha512-shZT/QMiSHc/YBLxxOkMtgSid5HFoauqCE3/exfsEcwg1WkeqjG+V40yBbBrsD+jW2HDXcs28xOfcbm2jI8Ddw==} - - glob@7.2.3: - resolution: {integrity: sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==} - deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me - - gopd@1.2.0: - resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} - engines: {node: '>= 0.4'} - - graceful-fs@4.2.11: - resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} - - has-flag@4.0.0: - resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} - engines: {node: '>=8'} - - has-symbols@1.1.0: - resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} - engines: {node: '>= 0.4'} - - has-unicode@2.0.1: - resolution: {integrity: sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==} - - hasown@2.0.2: - resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} - engines: {node: '>= 0.4'} - - http-errors@2.0.1: - resolution: {integrity: sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==} - engines: {node: '>= 0.8'} - - iconv-lite@0.4.24: - resolution: {integrity: sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==} - engines: {node: '>=0.10.0'} - - iconv-lite@0.7.2: - resolution: {integrity: sha512-im9DjEDQ55s9fL4EYzOAv0yMqmMBSZp6G0VvFyTMPKWxiSBHUj9NW/qqLmXUwXrrM7AvqSlTCfvqRb0cM8yYqw==} - engines: {node: '>=0.10.0'} - - ignore-walk@3.0.4: - resolution: {integrity: sha512-PY6Ii8o1jMRA1z4F2hRkH/xN59ox43DavKvD3oDpfurRlOJyAHpifIwpbdv1n4jt4ov0jSpw3kQ4GhJnpBL6WQ==} - - inflight@1.0.6: - resolution: {integrity: sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==} - deprecated: This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful. - - inherits@2.0.4: - resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} - - ini@1.3.8: - resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} - - ipaddr.js@1.9.1: - resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} - engines: {node: '>= 0.10'} - - is-fullwidth-code-point@1.0.0: - resolution: {integrity: sha512-1pqUqRjkhPJ9miNq9SwMfdvi6lBJcd6eFxvfaivQhaH3SgisfiuudvFntdKOmxuee/77l+FPjKrQjWvmPjWrRw==} - engines: {node: '>=0.10.0'} - - is-fullwidth-code-point@3.0.0: - resolution: {integrity: sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==} - engines: {node: '>=8'} - - is-number@7.0.0: - resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} - engines: {node: '>=0.12.0'} - - is-promise@4.0.0: - resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==} - - isarray@1.0.0: - resolution: {integrity: sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==} - - jose@5.10.0: - resolution: {integrity: sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==} - - livekit-server-sdk@2.15.0: - resolution: {integrity: sha512-HmzjWnwEwwShu8yUf7VGFXdc+BuMJR5pnIY4qsdlhqI9d9wDgq+4cdTEHg0NEBaiGnc6PCOBiaTYgmIyVJ0S9w==} - engines: {node: '>=18'} - - lodash@4.17.23: - resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} - - map-obj@5.0.0: - resolution: {integrity: sha512-2L3MIgJynYrZ3TYMriLDLWocz15okFakV6J12HXvMXDHui2x/zgChzg1u9mFFGbbGWE+GsLpQByt4POb9Or+uA==} - engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - - math-intrinsics@1.1.0: - resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} - engines: {node: '>= 0.4'} - - media-typer@1.1.0: - resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} - engines: {node: '>= 0.8'} - - merge-descriptors@2.0.0: - resolution: {integrity: sha512-Snk314V5ayFLhp3fkUREub6WtjBfPdCPY1Ln8/8munuLuiYhsABgBVWsozAG+MWMbVEvcdcpbi9R7ww22l9Q3g==} - engines: {node: '>=18'} - - micromatch@4.0.8: - resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} - engines: {node: '>=8.6'} - - mime-db@1.54.0: - resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} - engines: {node: '>= 0.6'} - - mime-types@3.0.2: - resolution: {integrity: sha512-Lbgzdk0h4juoQ9fCKXW4by0UJqj+nOOrI9MJ1sSj4nI8aI2eo1qmvQEie4VD1glsS250n15LsWsYtCugiStS5A==} - engines: {node: '>=18'} - - minimatch@3.1.5: - resolution: {integrity: sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==} - - minimist@1.2.8: - resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} - - minipass@2.9.0: - resolution: {integrity: sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==} - - minizlib@1.3.3: - resolution: {integrity: sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==} - - mkdirp@0.5.6: - resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} - hasBin: true - - ms@2.1.3: - resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} - - nanoid@3.3.11: - resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} - engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} - hasBin: true - - needle@2.9.1: - resolution: {integrity: sha512-6R9fqJ5Zcmf+uYaFgdIHmLwNldn5HbK8L5ybn7Uz+ylX/rnOsSp1AHcvQSrCaFN+qNM1wpymHqD7mVasEOlHGQ==} - engines: {node: '>= 4.4.x'} - hasBin: true - - negotiator@1.0.0: - resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} - engines: {node: '>= 0.6'} - - node-gyp-build@4.8.4: - resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} - hasBin: true - - node-pre-gyp@0.13.0: - resolution: {integrity: sha512-Md1D3xnEne8b/HGVQkZZwV27WUi1ZRuZBij24TNaZwUPU3ZAFtvT6xxJGaUVillfmMKnn5oD1HoGsp2Ftik7SQ==} - deprecated: 'Please upgrade to @mapbox/node-pre-gyp: the non-scoped node-pre-gyp package is deprecated and only the @mapbox scoped package will recieve updates in the future' - hasBin: true - - nopt@4.0.3: - resolution: {integrity: sha512-CvaGwVMztSMJLOeXPrez7fyfObdZqNUK1cPAEzLHrTybIua9pMdmmPR5YwtfNftIOMv3DPUhFaxsZMNTQO20Kg==} - hasBin: true - - npm-bundled@1.1.2: - resolution: {integrity: sha512-x5DHup0SuyQcmL3s7Rx/YQ8sbw/Hzg0rj48eN0dV7hf5cmQq5PXIeioroH3raV1QC1yh3uTYuMThvEQF3iKgGQ==} - - npm-normalize-package-bin@1.0.1: - resolution: {integrity: sha512-EPfafl6JL5/rU+ot6P3gRSCpPDW5VmIzX959Ob1+ySFUuuYHWHekXpwdUZcKP5C+DS4GEtdJluwBjnsNDl+fSA==} - - npm-packlist@1.4.8: - resolution: {integrity: sha512-5+AZgwru5IevF5ZdnFglB5wNlHG1AOOuw28WhUq8/8emhBmLv6jX5by4WJCh7lW0uSYZYS6DXqIsyZVIXRZU9A==} - - npmlog@4.1.2: - resolution: {integrity: sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==} - deprecated: This package is no longer supported. - - number-is-nan@1.0.1: - resolution: {integrity: sha512-4jbtZXNAsfZbAHiiqjLPBiCl16dES1zI4Hpzzxw61Tk+loF+sBDBKx1ICKKKwIqQ7M0mFn1TmkN7euSncWgHiQ==} - engines: {node: '>=0.10.0'} - - object-assign@4.1.1: - resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} - engines: {node: '>=0.10.0'} - - object-inspect@1.13.4: - resolution: {integrity: sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==} - engines: {node: '>= 0.4'} - - on-finished@2.4.1: - resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} - engines: {node: '>= 0.8'} - - once@1.4.0: - resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} - - os-homedir@1.0.2: - resolution: {integrity: sha512-B5JU3cabzk8c67mRRd3ECmROafjYMXbuzlwtqdM8IbS8ktlTix8aFGb2bAGKrSRIlnfKwovGUUr72JUPyOb6kQ==} - engines: {node: '>=0.10.0'} - - os-tmpdir@1.0.2: - resolution: {integrity: sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==} - engines: {node: '>=0.10.0'} - - osenv@0.1.5: - resolution: {integrity: sha512-0CWcCECdMVc2Rw3U5w9ZjqX6ga6ubk1xDVKxtBQPK7wis/0F2r9T6k4ydGYhecl7YUBxBVxhL5oisPsNxAPe2g==} - deprecated: This package is no longer supported. - - parseurl@1.3.3: - resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==} - engines: {node: '>= 0.8'} - - path-is-absolute@1.0.1: - resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} - engines: {node: '>=0.10.0'} - - path-to-regexp@8.3.0: - resolution: {integrity: sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==} - - picocolors@1.1.1: - resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} - - picomatch@2.3.1: - resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} - engines: {node: '>=8.6'} - - postcss@8.5.6: - resolution: {integrity: sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg==} - engines: {node: ^10 || ^12 || >=14} - - process-nextick-args@2.0.1: - resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} - - proxy-addr@2.0.7: - resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} - engines: {node: '>= 0.10'} - - qs@6.15.0: - resolution: {integrity: sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==} - engines: {node: '>=0.6'} - - quick-lru@6.1.2: - resolution: {integrity: sha512-AAFUA5O1d83pIHEhJwWCq/RQcRukCkn/NSm2QsTEMle5f2hP0ChI2+3Xb051PZCkLryI/Ir1MVKviT2FIloaTQ==} - engines: {node: '>=12'} - - range-parser@1.2.1: - resolution: {integrity: sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==} - engines: {node: '>= 0.6'} - - raw-body@3.0.2: - resolution: {integrity: sha512-K5zQjDllxWkf7Z5xJdV0/B0WTNqx6vxG70zJE4N0kBs4LovmEYWJzQGxC9bS9RAKu3bgM40lrd5zoLJ12MQ5BA==} - engines: {node: '>= 0.10'} - - rc@1.2.8: - resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} - hasBin: true - - readable-stream@2.3.8: - resolution: {integrity: sha512-8p0AUk4XODgIewSi0l8Epjs+EVnWiK7NoDIEGU0HhE7+ZyY8D1IMY7odu5lRrFXGg71L15KG8QrPmum45RTtdA==} - - require-directory@2.1.1: - resolution: {integrity: sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==} - engines: {node: '>=0.10.0'} - - resolve-from@5.0.0: - resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} - engines: {node: '>=8'} - - resolve-pkg-maps@1.0.0: - resolution: {integrity: sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==} - - rimraf@2.7.1: - resolution: {integrity: sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==} - deprecated: Rimraf versions prior to v4 are no longer supported - hasBin: true - - rollup-pluginutils@2.8.2: - resolution: {integrity: sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==} - - rollup@4.59.0: - resolution: {integrity: sha512-2oMpl67a3zCH9H79LeMcbDhXW/UmWG/y2zuqnF2jQq5uq9TbM9TVyXvA4+t+ne2IIkBdrLpAaRQAvo7YI/Yyeg==} - engines: {node: '>=18.0.0', npm: '>=8.0.0'} - hasBin: true - - router@2.2.0: - resolution: {integrity: sha512-nLTrUKm2UyiL7rlhapu/Zl45FwNgkZGaCpZbIHajDYgwlJCOzLSk+cIPAnsEqV955GjILJnKbdQC1nVPz+gAYQ==} - engines: {node: '>= 18'} - - rxjs@7.8.2: - resolution: {integrity: sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==} - - safe-buffer@5.1.2: - resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} - - safe-buffer@5.2.1: - resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} - - safer-buffer@2.1.2: - resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} - - sax@1.4.4: - resolution: {integrity: sha512-1n3r/tGXO6b6VXMdFT54SHzT9ytu9yr7TaELowdYpMqY/Ao7EnlQGmAQ1+RatX7Tkkdm6hONI2owqNx2aZj5Sw==} - engines: {node: '>=11.0.0'} - - semver@5.7.2: - resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==} - hasBin: true - - send@1.2.1: - resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} - engines: {node: '>= 18'} - - serve-static@2.2.1: - resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} - engines: {node: '>= 18'} - - set-blocking@2.0.0: - resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==} - - setprototypeof@1.2.0: - resolution: {integrity: sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==} - - shell-quote@1.8.3: - resolution: {integrity: sha512-ObmnIF4hXNg1BqhnHmgbDETF8dLPCggZWBjkQfhZpbszZnYur5DUljTcCHii5LC3J5E0yeO/1LIMyH+UvHQgyw==} - engines: {node: '>= 0.4'} - - side-channel-list@1.0.0: - resolution: {integrity: sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==} - engines: {node: '>= 0.4'} - - side-channel-map@1.0.1: - resolution: {integrity: sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==} - engines: {node: '>= 0.4'} - - side-channel-weakmap@1.0.2: - resolution: {integrity: sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==} - engines: {node: '>= 0.4'} - - side-channel@1.1.0: - resolution: {integrity: sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==} - engines: {node: '>= 0.4'} - - signal-exit@3.0.7: - resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} - - source-map-js@1.2.1: - resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} - engines: {node: '>=0.10.0'} - - spawn-command@0.0.2: - resolution: {integrity: sha512-zC8zGoGkmc8J9ndvml8Xksr1Amk9qBujgbF0JAIWO7kXr43w0h/0GJNM/Vustixu+YE8N/MTrQ7N31FvHUACxQ==} - - statuses@2.0.2: - resolution: {integrity: sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==} - engines: {node: '>= 0.8'} - - string-width@1.0.2: - resolution: {integrity: sha512-0XsVpQLnVCXHJfyEs8tC0zpTVIr5PKKsQtkT29IwupnPTjtPmQ3xT/4yCREF9hYkV/3M3kzcUTSAZT6a6h81tw==} - engines: {node: '>=0.10.0'} - - string-width@4.2.3: - resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} - engines: {node: '>=8'} - - string_decoder@1.1.1: - resolution: {integrity: sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==} - - strip-ansi@3.0.1: - resolution: {integrity: sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==} - engines: {node: '>=0.10.0'} - - strip-ansi@6.0.1: - resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} - engines: {node: '>=8'} - - strip-json-comments@2.0.1: - resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} - engines: {node: '>=0.10.0'} - - supports-color@7.2.0: - resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} - engines: {node: '>=8'} - - supports-color@8.1.1: - resolution: {integrity: sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==} - engines: {node: '>=10'} - - tar@4.4.19: - resolution: {integrity: sha512-a20gEsvHnWe0ygBY8JbxoM4w3SJdhc7ZAuxkLqh+nvNQN2IOt0B5lLgM490X5Hl8FF0dl0tOf2ewFYAlIFgzVA==} - engines: {node: '>=4.5'} - deprecated: Old versions of tar are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me - - to-regex-range@5.0.1: - resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} - engines: {node: '>=8.0'} - - toidentifier@1.0.1: - resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} - engines: {node: '>=0.6'} - - tree-kill@1.2.2: - resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} - hasBin: true - - tslib@2.8.1: - resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} - - tsx@4.21.0: - resolution: {integrity: sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==} - engines: {node: '>=18.0.0'} - hasBin: true - - type-fest@4.41.0: - resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} - engines: {node: '>=16'} - - type-is@2.0.1: - resolution: {integrity: sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==} - engines: {node: '>= 0.6'} - - typescript@5.9.3: - resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} - engines: {node: '>=14.17'} - hasBin: true - - undici-types@6.21.0: - resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} - - unpipe@1.0.0: - resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} - engines: {node: '>= 0.8'} - - util-deprecate@1.0.2: - resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} - - vary@1.1.2: - resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} - engines: {node: '>= 0.8'} - - vite-plugin-mix@0.4.0: - resolution: {integrity: sha512-9X8hiwhl0RbtEXBB0XqnQ5suheAtP3VHn794WcWwjU5ziYYWdlqpMh/2J8APpx/YdpvQ2CZT7dlcGGd/31ya3w==} - peerDependencies: - vite: ^3 - - vite@5.4.21: - resolution: {integrity: sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==} - engines: {node: ^18.0.0 || >=20.0.0} - hasBin: true - peerDependencies: - '@types/node': ^18.0.0 || >=20.0.0 - less: '*' - lightningcss: ^1.21.0 - sass: '*' - sass-embedded: '*' - stylus: '*' - sugarss: '*' - terser: ^5.4.0 - peerDependenciesMeta: - '@types/node': - optional: true - less: - optional: true - lightningcss: - optional: true - sass: - optional: true - sass-embedded: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - - wide-align@1.1.5: - resolution: {integrity: sha512-eDMORYaPNZ4sQIuuYPDHdQvf4gyCF9rEEV/yPxGfwPkRodwEgiMUUXTx/dex+Me0wxx53S+NgUHaP7y3MGlDmg==} - - wrap-ansi@7.0.0: - resolution: {integrity: sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==} - engines: {node: '>=10'} - - wrappy@1.0.2: - resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} - - y18n@5.0.8: - resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} - engines: {node: '>=10'} - - yallist@3.1.1: - resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} - - yargs-parser@21.1.1: - resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} - engines: {node: '>=12'} - - yargs@17.7.2: - resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} - engines: {node: '>=12'} - -snapshots: - - '@babel/runtime@7.28.6': {} - - '@bufbuild/protobuf@1.10.1': {} - - '@esbuild/aix-ppc64@0.21.5': - optional: true - - '@esbuild/aix-ppc64@0.27.3': - optional: true - - '@esbuild/android-arm64@0.21.5': - optional: true - - '@esbuild/android-arm64@0.27.3': - optional: true - - '@esbuild/android-arm@0.21.5': - optional: true - - '@esbuild/android-arm@0.27.3': - optional: true - - '@esbuild/android-x64@0.21.5': - optional: true - - '@esbuild/android-x64@0.27.3': - optional: true - - '@esbuild/darwin-arm64@0.21.5': - optional: true - - '@esbuild/darwin-arm64@0.27.3': - optional: true - - '@esbuild/darwin-x64@0.21.5': - optional: true - - '@esbuild/darwin-x64@0.27.3': - optional: true - - '@esbuild/freebsd-arm64@0.21.5': - optional: true - - '@esbuild/freebsd-arm64@0.27.3': - optional: true - - '@esbuild/freebsd-x64@0.21.5': - optional: true - - '@esbuild/freebsd-x64@0.27.3': - optional: true - - '@esbuild/linux-arm64@0.21.5': - optional: true - - '@esbuild/linux-arm64@0.27.3': - optional: true - - '@esbuild/linux-arm@0.21.5': - optional: true - - '@esbuild/linux-arm@0.27.3': - optional: true - - '@esbuild/linux-ia32@0.21.5': - optional: true - - '@esbuild/linux-ia32@0.27.3': - optional: true - - '@esbuild/linux-loong64@0.21.5': - optional: true - - '@esbuild/linux-loong64@0.27.3': - optional: true - - '@esbuild/linux-mips64el@0.21.5': - optional: true - - '@esbuild/linux-mips64el@0.27.3': - optional: true - - '@esbuild/linux-ppc64@0.21.5': - optional: true - - '@esbuild/linux-ppc64@0.27.3': - optional: true - - '@esbuild/linux-riscv64@0.21.5': - optional: true - - '@esbuild/linux-riscv64@0.27.3': - optional: true - - '@esbuild/linux-s390x@0.21.5': - optional: true - - '@esbuild/linux-s390x@0.27.3': - optional: true - - '@esbuild/linux-x64@0.21.5': - optional: true - - '@esbuild/linux-x64@0.27.3': - optional: true - - '@esbuild/netbsd-arm64@0.27.3': - optional: true - - '@esbuild/netbsd-x64@0.21.5': - optional: true - - '@esbuild/netbsd-x64@0.27.3': - optional: true - - '@esbuild/openbsd-arm64@0.27.3': - optional: true - - '@esbuild/openbsd-x64@0.21.5': - optional: true - - '@esbuild/openbsd-x64@0.27.3': - optional: true - - '@esbuild/openharmony-arm64@0.27.3': - optional: true - - '@esbuild/sunos-x64@0.21.5': - optional: true - - '@esbuild/sunos-x64@0.27.3': - optional: true - - '@esbuild/win32-arm64@0.21.5': - optional: true - - '@esbuild/win32-arm64@0.27.3': - optional: true - - '@esbuild/win32-ia32@0.21.5': - optional: true - - '@esbuild/win32-ia32@0.27.3': - optional: true - - '@esbuild/win32-x64@0.21.5': - optional: true - - '@esbuild/win32-x64@0.27.3': - optional: true - - '@livekit/protocol@1.44.0': - dependencies: - '@bufbuild/protobuf': 1.10.1 - - '@rollup/rollup-android-arm-eabi@4.59.0': - optional: true - - '@rollup/rollup-android-arm64@4.59.0': - optional: true - - '@rollup/rollup-darwin-arm64@4.59.0': - optional: true - - '@rollup/rollup-darwin-x64@4.59.0': - optional: true - - '@rollup/rollup-freebsd-arm64@4.59.0': - optional: true - - '@rollup/rollup-freebsd-x64@4.59.0': - optional: true - - '@rollup/rollup-linux-arm-gnueabihf@4.59.0': - optional: true - - '@rollup/rollup-linux-arm-musleabihf@4.59.0': - optional: true - - '@rollup/rollup-linux-arm64-gnu@4.59.0': - optional: true - - '@rollup/rollup-linux-arm64-musl@4.59.0': - optional: true - - '@rollup/rollup-linux-loong64-gnu@4.59.0': - optional: true - - '@rollup/rollup-linux-loong64-musl@4.59.0': - optional: true - - '@rollup/rollup-linux-ppc64-gnu@4.59.0': - optional: true - - '@rollup/rollup-linux-ppc64-musl@4.59.0': - optional: true - - '@rollup/rollup-linux-riscv64-gnu@4.59.0': - optional: true - - '@rollup/rollup-linux-riscv64-musl@4.59.0': - optional: true - - '@rollup/rollup-linux-s390x-gnu@4.59.0': - optional: true - - '@rollup/rollup-linux-x64-gnu@4.59.0': - optional: true - - '@rollup/rollup-linux-x64-musl@4.59.0': - optional: true - - '@rollup/rollup-openbsd-x64@4.59.0': - optional: true - - '@rollup/rollup-openharmony-arm64@4.59.0': - optional: true - - '@rollup/rollup-win32-arm64-msvc@4.59.0': - optional: true - - '@rollup/rollup-win32-ia32-msvc@4.59.0': - optional: true - - '@rollup/rollup-win32-x64-gnu@4.59.0': - optional: true - - '@rollup/rollup-win32-x64-msvc@4.59.0': - optional: true - - '@types/body-parser@1.19.6': - dependencies: - '@types/connect': 3.4.38 - '@types/node': 20.19.41 - - '@types/connect@3.4.38': - dependencies: - '@types/node': 20.19.41 - - '@types/cors@2.8.19': - dependencies: - '@types/node': 20.19.41 - - '@types/estree@1.0.8': {} - - '@types/express-serve-static-core@5.1.1': - dependencies: - '@types/node': 20.19.41 - '@types/qs': 6.14.0 - '@types/range-parser': 1.2.7 - '@types/send': 1.2.1 - - '@types/express@5.0.6': - dependencies: - '@types/body-parser': 1.19.6 - '@types/express-serve-static-core': 5.1.1 - '@types/serve-static': 2.2.0 - - '@types/http-errors@2.0.5': {} - - '@types/node@20.19.41': - dependencies: - undici-types: 6.21.0 - - '@types/qs@6.14.0': {} - - '@types/range-parser@1.2.7': {} - - '@types/send@1.2.1': - dependencies: - '@types/node': 20.19.41 - - '@types/serve-static@2.2.0': - dependencies: - '@types/http-errors': 2.0.5 - '@types/node': 20.19.41 - - '@vercel/nft@0.10.1': - dependencies: - acorn: 8.16.0 - acorn-class-fields: 1.0.0(acorn@8.16.0) - acorn-static-class-features: 1.0.0(acorn@8.16.0) - bindings: 1.5.0 - estree-walker: 0.6.1 - glob: 7.2.3 - graceful-fs: 4.2.11 - micromatch: 4.0.8 - mkdirp: 0.5.6 - node-gyp-build: 4.8.4 - node-pre-gyp: 0.13.0 - resolve-from: 5.0.0 - rollup-pluginutils: 2.8.2 - transitivePeerDependencies: - - supports-color - - abbrev@1.1.1: {} - - accepts@2.0.0: - dependencies: - mime-types: 3.0.2 - negotiator: 1.0.0 - - acorn-class-fields@1.0.0(acorn@8.16.0): - dependencies: - acorn: 8.16.0 - acorn-private-class-elements: 1.0.0(acorn@8.16.0) - - acorn-private-class-elements@1.0.0(acorn@8.16.0): - dependencies: - acorn: 8.16.0 - - acorn-static-class-features@1.0.0(acorn@8.16.0): - dependencies: - acorn: 8.16.0 - acorn-private-class-elements: 1.0.0(acorn@8.16.0) - - acorn@8.16.0: {} - - ansi-regex@2.1.1: {} - - ansi-regex@5.0.1: {} - - ansi-styles@4.3.0: - dependencies: - color-convert: 2.0.1 - - aproba@1.2.0: {} - - are-we-there-yet@1.1.7: - dependencies: - delegates: 1.0.0 - readable-stream: 2.3.8 - - balanced-match@1.0.2: {} - - bindings@1.5.0: - dependencies: - file-uri-to-path: 1.0.0 - - body-parser@2.2.2: - dependencies: - bytes: 3.1.2 - content-type: 1.0.5 - debug: 4.4.3 - http-errors: 2.0.1 - iconv-lite: 0.7.2 - on-finished: 2.4.1 - qs: 6.15.0 - raw-body: 3.0.2 - type-is: 2.0.1 - transitivePeerDependencies: - - supports-color - - brace-expansion@1.1.12: - dependencies: - balanced-match: 1.0.2 - concat-map: 0.0.1 - - braces@3.0.3: - dependencies: - fill-range: 7.1.1 - - bytes@3.1.2: {} - - call-bind-apply-helpers@1.0.2: - dependencies: - es-errors: 1.3.0 - function-bind: 1.1.2 - - call-bound@1.0.4: - dependencies: - call-bind-apply-helpers: 1.0.2 - get-intrinsic: 1.3.0 - - camelcase-keys@9.1.3: - dependencies: - camelcase: 8.0.0 - map-obj: 5.0.0 - quick-lru: 6.1.2 - type-fest: 4.41.0 - - camelcase@8.0.0: {} - - chalk@4.1.2: - dependencies: - ansi-styles: 4.3.0 - supports-color: 7.2.0 - - chownr@1.1.4: {} - - cliui@8.0.1: - dependencies: - string-width: 4.2.3 - strip-ansi: 6.0.1 - wrap-ansi: 7.0.0 - - code-point-at@1.1.0: {} - - color-convert@2.0.1: - dependencies: - color-name: 1.1.4 - - color-name@1.1.4: {} - - concat-map@0.0.1: {} - - concurrently@8.2.2: - dependencies: - chalk: 4.1.2 - date-fns: 2.30.0 - lodash: 4.17.23 - rxjs: 7.8.2 - shell-quote: 1.8.3 - spawn-command: 0.0.2 - supports-color: 8.1.1 - tree-kill: 1.2.2 - yargs: 17.7.2 - - console-control-strings@1.1.0: {} - - content-disposition@1.0.1: {} - - content-type@1.0.5: {} - - cookie-signature@1.2.2: {} - - cookie@0.7.2: {} - - core-util-is@1.0.3: {} - - cors@2.8.6: - dependencies: - object-assign: 4.1.1 - vary: 1.1.2 - - date-fns@2.30.0: - dependencies: - '@babel/runtime': 7.28.6 - - debug@3.2.7: - dependencies: - ms: 2.1.3 - - debug@4.4.3: - dependencies: - ms: 2.1.3 - - deep-extend@0.6.0: {} - - delegates@1.0.0: {} - - depd@2.0.0: {} - - detect-libc@1.0.3: {} - - dotenv@16.6.1: {} - - dunder-proto@1.0.1: - dependencies: - call-bind-apply-helpers: 1.0.2 - es-errors: 1.3.0 - gopd: 1.2.0 - - ee-first@1.1.1: {} - - emoji-regex@8.0.0: {} - - encodeurl@2.0.0: {} - - es-define-property@1.0.1: {} - - es-errors@1.3.0: {} - - es-object-atoms@1.1.1: - dependencies: - es-errors: 1.3.0 - - esbuild@0.21.5: - optionalDependencies: - '@esbuild/aix-ppc64': 0.21.5 - '@esbuild/android-arm': 0.21.5 - '@esbuild/android-arm64': 0.21.5 - '@esbuild/android-x64': 0.21.5 - '@esbuild/darwin-arm64': 0.21.5 - '@esbuild/darwin-x64': 0.21.5 - '@esbuild/freebsd-arm64': 0.21.5 - '@esbuild/freebsd-x64': 0.21.5 - '@esbuild/linux-arm': 0.21.5 - '@esbuild/linux-arm64': 0.21.5 - '@esbuild/linux-ia32': 0.21.5 - '@esbuild/linux-loong64': 0.21.5 - '@esbuild/linux-mips64el': 0.21.5 - '@esbuild/linux-ppc64': 0.21.5 - '@esbuild/linux-riscv64': 0.21.5 - '@esbuild/linux-s390x': 0.21.5 - '@esbuild/linux-x64': 0.21.5 - '@esbuild/netbsd-x64': 0.21.5 - '@esbuild/openbsd-x64': 0.21.5 - '@esbuild/sunos-x64': 0.21.5 - '@esbuild/win32-arm64': 0.21.5 - '@esbuild/win32-ia32': 0.21.5 - '@esbuild/win32-x64': 0.21.5 - - esbuild@0.27.3: - optionalDependencies: - '@esbuild/aix-ppc64': 0.27.3 - '@esbuild/android-arm': 0.27.3 - '@esbuild/android-arm64': 0.27.3 - '@esbuild/android-x64': 0.27.3 - '@esbuild/darwin-arm64': 0.27.3 - '@esbuild/darwin-x64': 0.27.3 - '@esbuild/freebsd-arm64': 0.27.3 - '@esbuild/freebsd-x64': 0.27.3 - '@esbuild/linux-arm': 0.27.3 - '@esbuild/linux-arm64': 0.27.3 - '@esbuild/linux-ia32': 0.27.3 - '@esbuild/linux-loong64': 0.27.3 - '@esbuild/linux-mips64el': 0.27.3 - '@esbuild/linux-ppc64': 0.27.3 - '@esbuild/linux-riscv64': 0.27.3 - '@esbuild/linux-s390x': 0.27.3 - '@esbuild/linux-x64': 0.27.3 - '@esbuild/netbsd-arm64': 0.27.3 - '@esbuild/netbsd-x64': 0.27.3 - '@esbuild/openbsd-arm64': 0.27.3 - '@esbuild/openbsd-x64': 0.27.3 - '@esbuild/openharmony-arm64': 0.27.3 - '@esbuild/sunos-x64': 0.27.3 - '@esbuild/win32-arm64': 0.27.3 - '@esbuild/win32-ia32': 0.27.3 - '@esbuild/win32-x64': 0.27.3 - - escalade@3.2.0: {} - - escape-html@1.0.3: {} - - estree-walker@0.6.1: {} - - etag@1.8.1: {} - - express@5.2.1: - dependencies: - accepts: 2.0.0 - body-parser: 2.2.2 - content-disposition: 1.0.1 - content-type: 1.0.5 - cookie: 0.7.2 - cookie-signature: 1.2.2 - debug: 4.4.3 - depd: 2.0.0 - encodeurl: 2.0.0 - escape-html: 1.0.3 - etag: 1.8.1 - finalhandler: 2.1.1 - fresh: 2.0.0 - http-errors: 2.0.1 - merge-descriptors: 2.0.0 - mime-types: 3.0.2 - on-finished: 2.4.1 - once: 1.4.0 - parseurl: 1.3.3 - proxy-addr: 2.0.7 - qs: 6.15.0 - range-parser: 1.2.1 - router: 2.2.0 - send: 1.2.1 - serve-static: 2.2.1 - statuses: 2.0.2 - type-is: 2.0.1 - vary: 1.1.2 - transitivePeerDependencies: - - supports-color - - file-uri-to-path@1.0.0: {} - - fill-range@7.1.1: - dependencies: - to-regex-range: 5.0.1 - - finalhandler@2.1.1: - dependencies: - debug: 4.4.3 - encodeurl: 2.0.0 - escape-html: 1.0.3 - on-finished: 2.4.1 - parseurl: 1.3.3 - statuses: 2.0.2 - transitivePeerDependencies: - - supports-color - - forwarded@0.2.0: {} - - fresh@2.0.0: {} - - fs-minipass@1.2.7: - dependencies: - minipass: 2.9.0 - - fs.realpath@1.0.0: {} - - fsevents@2.3.3: - optional: true - - function-bind@1.1.2: {} - - gauge@2.7.4: - dependencies: - aproba: 1.2.0 - console-control-strings: 1.1.0 - has-unicode: 2.0.1 - object-assign: 4.1.1 - signal-exit: 3.0.7 - string-width: 1.0.2 - strip-ansi: 3.0.1 - wide-align: 1.1.5 - - get-caller-file@2.0.5: {} - - get-intrinsic@1.3.0: - dependencies: - call-bind-apply-helpers: 1.0.2 - es-define-property: 1.0.1 - es-errors: 1.3.0 - es-object-atoms: 1.1.1 - function-bind: 1.1.2 - get-proto: 1.0.1 - gopd: 1.2.0 - has-symbols: 1.1.0 - hasown: 2.0.2 - math-intrinsics: 1.1.0 - - get-proto@1.0.1: - dependencies: - dunder-proto: 1.0.1 - es-object-atoms: 1.1.1 - - get-tsconfig@4.13.6: - dependencies: - resolve-pkg-maps: 1.0.0 - - glob@7.2.3: - dependencies: - fs.realpath: 1.0.0 - inflight: 1.0.6 - inherits: 2.0.4 - minimatch: 3.1.5 - once: 1.4.0 - path-is-absolute: 1.0.1 - - gopd@1.2.0: {} - - graceful-fs@4.2.11: {} - - has-flag@4.0.0: {} - - has-symbols@1.1.0: {} - - has-unicode@2.0.1: {} - - hasown@2.0.2: - dependencies: - function-bind: 1.1.2 - - http-errors@2.0.1: - dependencies: - depd: 2.0.0 - inherits: 2.0.4 - setprototypeof: 1.2.0 - statuses: 2.0.2 - toidentifier: 1.0.1 - - iconv-lite@0.4.24: - dependencies: - safer-buffer: 2.1.2 - - iconv-lite@0.7.2: - dependencies: - safer-buffer: 2.1.2 - - ignore-walk@3.0.4: - dependencies: - minimatch: 3.1.5 - - inflight@1.0.6: - dependencies: - once: 1.4.0 - wrappy: 1.0.2 - - inherits@2.0.4: {} - - ini@1.3.8: {} - - ipaddr.js@1.9.1: {} - - is-fullwidth-code-point@1.0.0: - dependencies: - number-is-nan: 1.0.1 - - is-fullwidth-code-point@3.0.0: {} - - is-number@7.0.0: {} - - is-promise@4.0.0: {} - - isarray@1.0.0: {} - - jose@5.10.0: {} - - livekit-server-sdk@2.15.0: - dependencies: - '@bufbuild/protobuf': 1.10.1 - '@livekit/protocol': 1.44.0 - camelcase-keys: 9.1.3 - jose: 5.10.0 - - lodash@4.17.23: {} - - map-obj@5.0.0: {} - - math-intrinsics@1.1.0: {} - - media-typer@1.1.0: {} - - merge-descriptors@2.0.0: {} - - micromatch@4.0.8: - dependencies: - braces: 3.0.3 - picomatch: 2.3.1 - - mime-db@1.54.0: {} - - mime-types@3.0.2: - dependencies: - mime-db: 1.54.0 - - minimatch@3.1.5: - dependencies: - brace-expansion: 1.1.12 - - minimist@1.2.8: {} - - minipass@2.9.0: - dependencies: - safe-buffer: 5.2.1 - yallist: 3.1.1 - - minizlib@1.3.3: - dependencies: - minipass: 2.9.0 - - mkdirp@0.5.6: - dependencies: - minimist: 1.2.8 - - ms@2.1.3: {} - - nanoid@3.3.11: {} - - needle@2.9.1: - dependencies: - debug: 3.2.7 - iconv-lite: 0.4.24 - sax: 1.4.4 - transitivePeerDependencies: - - supports-color - - negotiator@1.0.0: {} - - node-gyp-build@4.8.4: {} - - node-pre-gyp@0.13.0: - dependencies: - detect-libc: 1.0.3 - mkdirp: 0.5.6 - needle: 2.9.1 - nopt: 4.0.3 - npm-packlist: 1.4.8 - npmlog: 4.1.2 - rc: 1.2.8 - rimraf: 2.7.1 - semver: 5.7.2 - tar: 4.4.19 - transitivePeerDependencies: - - supports-color - - nopt@4.0.3: - dependencies: - abbrev: 1.1.1 - osenv: 0.1.5 - - npm-bundled@1.1.2: - dependencies: - npm-normalize-package-bin: 1.0.1 - - npm-normalize-package-bin@1.0.1: {} - - npm-packlist@1.4.8: - dependencies: - ignore-walk: 3.0.4 - npm-bundled: 1.1.2 - npm-normalize-package-bin: 1.0.1 - - npmlog@4.1.2: - dependencies: - are-we-there-yet: 1.1.7 - console-control-strings: 1.1.0 - gauge: 2.7.4 - set-blocking: 2.0.0 - - number-is-nan@1.0.1: {} - - object-assign@4.1.1: {} - - object-inspect@1.13.4: {} - - on-finished@2.4.1: - dependencies: - ee-first: 1.1.1 - - once@1.4.0: - dependencies: - wrappy: 1.0.2 - - os-homedir@1.0.2: {} - - os-tmpdir@1.0.2: {} - - osenv@0.1.5: - dependencies: - os-homedir: 1.0.2 - os-tmpdir: 1.0.2 - - parseurl@1.3.3: {} - - path-is-absolute@1.0.1: {} - - path-to-regexp@8.3.0: {} - - picocolors@1.1.1: {} - - picomatch@2.3.1: {} - - postcss@8.5.6: - dependencies: - nanoid: 3.3.11 - picocolors: 1.1.1 - source-map-js: 1.2.1 - - process-nextick-args@2.0.1: {} - - proxy-addr@2.0.7: - dependencies: - forwarded: 0.2.0 - ipaddr.js: 1.9.1 - - qs@6.15.0: - dependencies: - side-channel: 1.1.0 - - quick-lru@6.1.2: {} - - range-parser@1.2.1: {} - - raw-body@3.0.2: - dependencies: - bytes: 3.1.2 - http-errors: 2.0.1 - iconv-lite: 0.7.2 - unpipe: 1.0.0 - - rc@1.2.8: - dependencies: - deep-extend: 0.6.0 - ini: 1.3.8 - minimist: 1.2.8 - strip-json-comments: 2.0.1 - - readable-stream@2.3.8: - dependencies: - core-util-is: 1.0.3 - inherits: 2.0.4 - isarray: 1.0.0 - process-nextick-args: 2.0.1 - safe-buffer: 5.1.2 - string_decoder: 1.1.1 - util-deprecate: 1.0.2 - - require-directory@2.1.1: {} - - resolve-from@5.0.0: {} - - resolve-pkg-maps@1.0.0: {} - - rimraf@2.7.1: - dependencies: - glob: 7.2.3 - - rollup-pluginutils@2.8.2: - dependencies: - estree-walker: 0.6.1 - - rollup@4.59.0: - dependencies: - '@types/estree': 1.0.8 - optionalDependencies: - '@rollup/rollup-android-arm-eabi': 4.59.0 - '@rollup/rollup-android-arm64': 4.59.0 - '@rollup/rollup-darwin-arm64': 4.59.0 - '@rollup/rollup-darwin-x64': 4.59.0 - '@rollup/rollup-freebsd-arm64': 4.59.0 - '@rollup/rollup-freebsd-x64': 4.59.0 - '@rollup/rollup-linux-arm-gnueabihf': 4.59.0 - '@rollup/rollup-linux-arm-musleabihf': 4.59.0 - '@rollup/rollup-linux-arm64-gnu': 4.59.0 - '@rollup/rollup-linux-arm64-musl': 4.59.0 - '@rollup/rollup-linux-loong64-gnu': 4.59.0 - '@rollup/rollup-linux-loong64-musl': 4.59.0 - '@rollup/rollup-linux-ppc64-gnu': 4.59.0 - '@rollup/rollup-linux-ppc64-musl': 4.59.0 - '@rollup/rollup-linux-riscv64-gnu': 4.59.0 - '@rollup/rollup-linux-riscv64-musl': 4.59.0 - '@rollup/rollup-linux-s390x-gnu': 4.59.0 - '@rollup/rollup-linux-x64-gnu': 4.59.0 - '@rollup/rollup-linux-x64-musl': 4.59.0 - '@rollup/rollup-openbsd-x64': 4.59.0 - '@rollup/rollup-openharmony-arm64': 4.59.0 - '@rollup/rollup-win32-arm64-msvc': 4.59.0 - '@rollup/rollup-win32-ia32-msvc': 4.59.0 - '@rollup/rollup-win32-x64-gnu': 4.59.0 - '@rollup/rollup-win32-x64-msvc': 4.59.0 - fsevents: 2.3.3 - - router@2.2.0: - dependencies: - debug: 4.4.3 - depd: 2.0.0 - is-promise: 4.0.0 - parseurl: 1.3.3 - path-to-regexp: 8.3.0 - transitivePeerDependencies: - - supports-color - - rxjs@7.8.2: - dependencies: - tslib: 2.8.1 - - safe-buffer@5.1.2: {} - - safe-buffer@5.2.1: {} - - safer-buffer@2.1.2: {} - - sax@1.4.4: {} - - semver@5.7.2: {} - - send@1.2.1: - dependencies: - debug: 4.4.3 - encodeurl: 2.0.0 - escape-html: 1.0.3 - etag: 1.8.1 - fresh: 2.0.0 - http-errors: 2.0.1 - mime-types: 3.0.2 - ms: 2.1.3 - on-finished: 2.4.1 - range-parser: 1.2.1 - statuses: 2.0.2 - transitivePeerDependencies: - - supports-color - - serve-static@2.2.1: - dependencies: - encodeurl: 2.0.0 - escape-html: 1.0.3 - parseurl: 1.3.3 - send: 1.2.1 - transitivePeerDependencies: - - supports-color - - set-blocking@2.0.0: {} - - setprototypeof@1.2.0: {} - - shell-quote@1.8.3: {} - - side-channel-list@1.0.0: - dependencies: - es-errors: 1.3.0 - object-inspect: 1.13.4 - - side-channel-map@1.0.1: - dependencies: - call-bound: 1.0.4 - es-errors: 1.3.0 - get-intrinsic: 1.3.0 - object-inspect: 1.13.4 - - side-channel-weakmap@1.0.2: - dependencies: - call-bound: 1.0.4 - es-errors: 1.3.0 - get-intrinsic: 1.3.0 - object-inspect: 1.13.4 - side-channel-map: 1.0.1 - - side-channel@1.1.0: - dependencies: - es-errors: 1.3.0 - object-inspect: 1.13.4 - side-channel-list: 1.0.0 - side-channel-map: 1.0.1 - side-channel-weakmap: 1.0.2 - - signal-exit@3.0.7: {} - - source-map-js@1.2.1: {} - - spawn-command@0.0.2: {} - - statuses@2.0.2: {} - - string-width@1.0.2: - dependencies: - code-point-at: 1.1.0 - is-fullwidth-code-point: 1.0.0 - strip-ansi: 3.0.1 - - string-width@4.2.3: - dependencies: - emoji-regex: 8.0.0 - is-fullwidth-code-point: 3.0.0 - strip-ansi: 6.0.1 - - string_decoder@1.1.1: - dependencies: - safe-buffer: 5.1.2 - - strip-ansi@3.0.1: - dependencies: - ansi-regex: 2.1.1 - - strip-ansi@6.0.1: - dependencies: - ansi-regex: 5.0.1 - - strip-json-comments@2.0.1: {} - - supports-color@7.2.0: - dependencies: - has-flag: 4.0.0 - - supports-color@8.1.1: - dependencies: - has-flag: 4.0.0 - - tar@4.4.19: - dependencies: - chownr: 1.1.4 - fs-minipass: 1.2.7 - minipass: 2.9.0 - minizlib: 1.3.3 - mkdirp: 0.5.6 - safe-buffer: 5.2.1 - yallist: 3.1.1 - - to-regex-range@5.0.1: - dependencies: - is-number: 7.0.0 - - toidentifier@1.0.1: {} - - tree-kill@1.2.2: {} - - tslib@2.8.1: {} - - tsx@4.21.0: - dependencies: - esbuild: 0.27.3 - get-tsconfig: 4.13.6 - optionalDependencies: - fsevents: 2.3.3 - - type-fest@4.41.0: {} - - type-is@2.0.1: - dependencies: - content-type: 1.0.5 - media-typer: 1.1.0 - mime-types: 3.0.2 - - typescript@5.9.3: {} - - undici-types@6.21.0: {} - - unpipe@1.0.0: {} - - util-deprecate@1.0.2: {} - - vary@1.1.2: {} - - vite-plugin-mix@0.4.0(vite@5.4.21(@types/node@20.19.41)): - dependencies: - '@vercel/nft': 0.10.1 - vite: 5.4.21(@types/node@20.19.41) - transitivePeerDependencies: - - supports-color - - vite@5.4.21(@types/node@20.19.41): - dependencies: - esbuild: 0.21.5 - postcss: 8.5.6 - rollup: 4.59.0 - optionalDependencies: - '@types/node': 20.19.41 - fsevents: 2.3.3 - - wide-align@1.1.5: - dependencies: - string-width: 1.0.2 - - wrap-ansi@7.0.0: - dependencies: - ansi-styles: 4.3.0 - string-width: 4.2.3 - strip-ansi: 6.0.1 - - wrappy@1.0.2: {} - - y18n@5.0.8: {} - - yallist@3.1.1: {} - - yargs-parser@21.1.1: {} - - yargs@17.7.2: - dependencies: - cliui: 8.0.1 - escalade: 3.2.0 - get-caller-file: 2.0.5 - require-directory: 2.1.1 - string-width: 4.2.3 - y18n: 5.0.8 - yargs-parser: 21.1.1 diff --git a/examples/data-stream-benchmark/pnpm-workspace.yaml b/examples/data-stream-benchmark/pnpm-workspace.yaml deleted file mode 100644 index 7057c6516c..0000000000 --- a/examples/data-stream-benchmark/pnpm-workspace.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# This example installs standalone (its own pnpm workspace root), separate from the repo-root -# workspace. Some pinned (dev) deps trip pnpm's supply-chain trust-downgrade check -# (ERR_PNPM_TRUST_DOWNGRADE) under a strict global `trustPolicy: no-downgrade`. Exclude those -# specific packages here so `pnpm install` / `pnpm dev` work in this directory without changing -# global or repo-root config, while keeping the trust check active for everything else. -trustPolicyExclude: - - 'vite@5.4.21' - - 'undici-types@6.21.0' diff --git a/examples/data-stream-benchmark/styles.css b/examples/data-stream-benchmark/styles.css deleted file mode 100644 index d25e9056d2..0000000000 --- a/examples/data-stream-benchmark/styles.css +++ /dev/null @@ -1,148 +0,0 @@ -body { - font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif, - 'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol'; - background-color: #f0f2f5; - color: #333; - line-height: 1.6; - margin: 0; - padding: 0; -} - -.container { - max-width: 1600px; - margin: 40px auto; - padding: 30px; - background-color: #ffffff; - box-shadow: 0 0 20px rgba(0, 0, 0, 0.1); - border-radius: 12px; -} - -h1 { - text-align: center; - color: #2c3e50; - margin-bottom: 24px; - font-weight: 600; -} - -.controls { - display: flex; - gap: 12px; - justify-content: center; - margin-bottom: 16px; -} - -.status { - text-align: center; - font-weight: 500; - color: #555; - margin-bottom: 20px; - min-height: 1.5em; -} - -.grid { - width: 100%; - border-collapse: collapse; - font-variant-numeric: tabular-nums; -} - -.grid th, -.grid td { - border: 1px solid #dce1e6; - padding: 10px 8px; - text-align: center; - font-size: 14px; -} - -.grid thead th { - background-color: #2c3e50; - color: #fff; - font-weight: 600; -} - -.grid th.row-head { - background-color: #f1f4f7; - font-weight: 600; - white-space: nowrap; -} - -.cell { - color: #2c3e50; -} - -.cell .recv { - font-size: 16px; - font-weight: 600; - font-variant-numeric: tabular-nums; -} - -.cell .status { - margin-top: 2px; - font-size: 12px; - line-height: 1.35; - color: #34495e; - font-variant-numeric: tabular-nums; - white-space: nowrap; -} - -.cell .status .bad { - color: #c0392b; - font-weight: 600; -} - -.cell.running .recv { - color: #2980b9; - font-style: italic; - font-weight: 400; -} - -.hint { - font-size: 13px; - color: #7f8c8d; - margin: 16px 0; -} - -#log-area { - margin-top: 10px; -} - -#log { - box-sizing: border-box; - width: 100%; - height: 240px; - padding: 10px; - border: 1px solid #ddd; - border-radius: 4px; - font-family: monospace; - font-size: 13px; - resize: vertical; -} - -.btn { - padding: 10px 20px; - background-color: #3498db; - color: white; - border: none; - border-radius: 5px; - font-size: 15px; - cursor: pointer; - transition: - background-color 0.3s, - transform 0.1s; - font-weight: 500; -} - -.btn:hover { - background-color: #2980b9; - transform: translateY(-2px); -} - -.btn:active { - transform: translateY(0); -} - -.btn:disabled { - background-color: #bdc3c7; - color: #7f8c8d; - cursor: not-allowed; - transform: none; -} diff --git a/examples/data-stream-benchmark/tsconfig.json b/examples/data-stream-benchmark/tsconfig.json deleted file mode 100644 index ad5b077b7e..0000000000 --- a/examples/data-stream-benchmark/tsconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "target": "ESNext" /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017', 'ES2018', 'ES2019', 'ES2020', or 'ESNEXT'. */, - "module": "ESNext" /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', 'es2020', or 'ESNext'. */, - "outDir": "build", - "declaration": true, - "declarationMap": true, - "sourceMap": true, - "strict": true /* Enable all strict type-checking options. */, - "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */, - "skipLibCheck": true /* Skip type checking of declaration files. */, - "noUnusedLocals": true, - "forceConsistentCasingInFileNames": true /* Disallow inconsistently-cased references to the same file. */, - "moduleResolution": "node", - "resolveJsonModule": true - }, - "include": ["../../src/**/*", "benchmark.ts", "payload.ts", "api.ts"], - "exclude": ["**/*.test.ts", "build/**/*"] -} diff --git a/examples/data-stream-benchmark/vite.config.js b/examples/data-stream-benchmark/vite.config.js deleted file mode 100644 index 8f82d19f31..0000000000 --- a/examples/data-stream-benchmark/vite.config.js +++ /dev/null @@ -1,10 +0,0 @@ -import { defineConfig } from 'vite'; -import mix from 'vite-plugin-mix'; - -export default defineConfig({ - plugins: [ - mix.default({ - handler: './api.ts', - }), - ], -});