From be783f20eacd96417f784ab9504ab711deabd81e Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:03:56 +0000 Subject: [PATCH 1/3] fix(test): avoid false PCM failures in TTS tests --- plugins/test/src/tts.ts | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/plugins/test/src/tts.ts b/plugins/test/src/tts.ts index 3692b1202..b2d1aae3a 100644 --- a/plugins/test/src/tts.ts +++ b/plugins/test/src/tts.ts @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 import type { stt } from '@livekit/agents'; import { type AudioBuffer, initializeLogger, tokenize, tts as ttslib } from '@livekit/agents'; -import type { AudioFrame } from '@livekit/rtc-node'; +import { type AudioFrame, combineAudioFrames } from '@livekit/rtc-node'; import { distance } from 'fastest-levenshtein'; import { ReadableStream } from 'stream/web'; import { describe, expect, it } from 'vitest'; @@ -11,6 +11,42 @@ import { describe, expect, it } from 'vitest'; const TEXT = 'The people who are crazy enough to think they can change the world are the ones who do.'; +const detectCompressedContainer = (data: Uint8Array): string | undefined => { + const startsWith = (signature: string, offset = 0) => + data.length >= offset + signature.length && + signature.split('').every((char, i) => data[offset + i] === char.charCodeAt(0)); + + if (data[0] === 0xff && data.length > 1 && (data[1]! & 0xf6) === 0xf0) { + return 'aac'; + } + if ( + startsWith('ID3') || + (data[0] === 0xff && data.length > 1 && (data[1]! & 0xe0) === 0xe0 && (data[1]! & 0x06) !== 0) + ) { + return 'mp3'; + } + if (startsWith('OggS')) return 'ogg'; + if (startsWith('fLaC')) return 'flac'; + if (startsWith('RIFF') && startsWith('WAVE', 8)) return 'wav'; + if (startsWith('ftyp', 4)) return 'mov,mp4,m4a,3gp,3g2,mj2'; + if (data[0] === 0x1a && data[1] === 0x45 && data[2] === 0xdf && data[3] === 0xa3) { + return 'matroska,webm'; + } + if (data[0] === 0x00 && data[1] === 0x00 && data[2] === 0x01 && data[3] === 0xba) { + return 'mpeg'; + } +}; + +const assertPCM = (frames: AudioFrame[]) => { + const frame = combineAudioFrames(frames); + const data = new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength); + const container = detectCompressedContainer(data); + + if (container) { + throw new Error(`Audio data isn't PCM (detected ${container})`); + } +}; + const validate = async (frames: AudioBuffer, stt: stt.STT, text: string, threshold: number) => { const event = await stt.recognize(frames); const eventText = event.alternatives![0].text.toLowerCase().replace(/\s/g, ' ').trim(); @@ -64,6 +100,7 @@ export const tts = async ( frames.push(event.frame); } + assertPCM(frames); await validate(frames, stt, TEXT, 0.2); stream.close(); }); From 3eaf6b5aa3992a2fdbe9dc6671630aa2c1aba172 Mon Sep 17 00:00:00 2001 From: "rosetta-livekit-bot[bot]" <282703043+rosetta-livekit-bot[bot]@users.noreply.github.com> Date: Mon, 6 Jul 2026 17:18:45 +0000 Subject: [PATCH 2/3] fix(test): probe TTS audio with FFmpeg --- plugins/test/package.json | 1 + plugins/test/src/tts.ts | 69 +++++++++++++++++++++++---------------- pnpm-lock.yaml | 3 ++ 3 files changed, 44 insertions(+), 29 deletions(-) diff --git a/plugins/test/package.json b/plugins/test/package.json index 81d4ba098..d0daa2e2e 100644 --- a/plugins/test/package.json +++ b/plugins/test/package.json @@ -39,6 +39,7 @@ "zod": "^3.25.76" }, "dependencies": { + "@ffmpeg-installer/ffmpeg": "^1.1.0", "fastest-levenshtein": "^1.0.16", "vitest": "^4.0.17" }, diff --git a/plugins/test/src/tts.ts b/plugins/test/src/tts.ts index b2d1aae3a..6c7caf141 100644 --- a/plugins/test/src/tts.ts +++ b/plugins/test/src/tts.ts @@ -1,48 +1,59 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import ffmpegInstaller from '@ffmpeg-installer/ffmpeg'; import type { stt } from '@livekit/agents'; import { type AudioBuffer, initializeLogger, tokenize, tts as ttslib } from '@livekit/agents'; import { type AudioFrame, combineAudioFrames } from '@livekit/rtc-node'; import { distance } from 'fastest-levenshtein'; +import { spawn } from 'node:child_process'; import { ReadableStream } from 'stream/web'; import { describe, expect, it } from 'vitest'; const TEXT = 'The people who are crazy enough to think they can change the world are the ones who do.'; -const detectCompressedContainer = (data: Uint8Array): string | undefined => { - const startsWith = (signature: string, offset = 0) => - data.length >= offset + signature.length && - signature.split('').every((char, i) => data[offset + i] === char.charCodeAt(0)); +const compressedFormats = new Set([ + 'mp3', + 'aac', + 'ogg', + 'flac', + 'wav', + 'mov,mp4,m4a,3gp,3g2,mj2', + 'matroska,webm', + 'mpeg', +]); - if (data[0] === 0xff && data.length > 1 && (data[1]! & 0xf6) === 0xf0) { - return 'aac'; - } - if ( - startsWith('ID3') || - (data[0] === 0xff && data.length > 1 && (data[1]! & 0xe0) === 0xe0 && (data[1]! & 0x06) !== 0) - ) { - return 'mp3'; - } - if (startsWith('OggS')) return 'ogg'; - if (startsWith('fLaC')) return 'flac'; - if (startsWith('RIFF') && startsWith('WAVE', 8)) return 'wav'; - if (startsWith('ftyp', 4)) return 'mov,mp4,m4a,3gp,3g2,mj2'; - if (data[0] === 0x1a && data[1] === 0x45 && data[2] === 0xdf && data[3] === 0xa3) { - return 'matroska,webm'; - } - if (data[0] === 0x00 && data[1] === 0x00 && data[2] === 0x01 && data[3] === 0xba) { - return 'mpeg'; - } -}; - -const assertPCM = (frames: AudioFrame[]) => { +const assertPCM = async (frames: AudioFrame[]) => { const frame = combineAudioFrames(frames); const data = new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength); - const container = detectCompressedContainer(data); - if (container) { + const container = await new Promise((resolve, reject) => { + const ffmpeg = spawn(ffmpegInstaller.path, [ + '-hide_banner', + '-probesize', + '32', + '-analyzeduration', + '0', + '-i', + 'pipe:0', + ]); + let stderr = ''; + + ffmpeg.stderr.setEncoding('utf8'); + ffmpeg.stderr.on('data', (chunk: string) => { + stderr += chunk; + }); + ffmpeg.stdin.on('error', () => {}); + ffmpeg.on('error', reject); + ffmpeg.on('close', () => { + resolve(stderr.match(/Input #0, (.*), from 'pipe:0':/)?.[1]); + }); + + ffmpeg.stdin.end(data); + }); + + if (container && compressedFormats.has(container)) { throw new Error(`Audio data isn't PCM (detected ${container})`); } }; @@ -100,7 +111,7 @@ export const tts = async ( frames.push(event.frame); } - assertPCM(frames); + await assertPCM(frames); await validate(frames, stt, TEXT, 0.2); stream.close(); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3f01e7345..d600520d7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1354,6 +1354,9 @@ importers: plugins/test: dependencies: + '@ffmpeg-installer/ffmpeg': + specifier: ^1.1.0 + version: 1.1.0 fastest-levenshtein: specifier: ^1.0.16 version: 1.0.16 From ccd732cc2d4983545bcb9eebf32961ff63c5f1d5 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Mon, 20 Jul 2026 14:12:40 -0400 Subject: [PATCH 3/3] test: use bundled ffmpeg from @livekit/av instead of @ffmpeg-installer --- plugins/test/package.json | 2 +- plugins/test/src/tts.ts | 9 ++++- pnpm-lock.yaml | 84 ++------------------------------------- 3 files changed, 11 insertions(+), 84 deletions(-) diff --git a/plugins/test/package.json b/plugins/test/package.json index da920e230..12c66e77d 100644 --- a/plugins/test/package.json +++ b/plugins/test/package.json @@ -39,7 +39,7 @@ "zod": "^3.25.76" }, "dependencies": { - "@ffmpeg-installer/ffmpeg": "^1.1.0", + "@livekit/av": "^10.0.0", "fastest-levenshtein": "^1.0.16", "vitest": "^4.0.17" }, diff --git a/plugins/test/src/tts.ts b/plugins/test/src/tts.ts index 6efcc1f54..c627f0fae 100644 --- a/plugins/test/src/tts.ts +++ b/plugins/test/src/tts.ts @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import ffmpegInstaller from '@ffmpeg-installer/ffmpeg'; import { type AudioBuffer, AudioByteStream, @@ -10,6 +9,7 @@ import { tokenize, tts as ttslib, } from '@livekit/agents'; +import { getFfmpegPath } from '@livekit/av'; import { type AudioFrame, combineAudioFrames } from '@livekit/rtc-node'; import { distance } from 'fastest-levenshtein'; import { spawn } from 'node:child_process'; @@ -31,11 +31,16 @@ const compressedFormats = new Set([ ]); const assertPCM = async (frames: AudioFrame[]) => { + const ffmpegPath = getFfmpegPath(); + if (!ffmpegPath) { + throw new Error('ffmpeg binary from @livekit/av is not available on this platform'); + } + const frame = combineAudioFrames(frames); const data = new Uint8Array(frame.data.buffer, frame.data.byteOffset, frame.data.byteLength); const container = await new Promise((resolve, reject) => { - const ffmpeg = spawn(ffmpegInstaller.path, [ + const ffmpeg = spawn(ffmpegPath, [ '-hide_banner', '-probesize', '32', diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bd9d5b529..a33393c50 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1385,9 +1385,9 @@ importers: plugins/test: dependencies: - '@ffmpeg-installer/ffmpeg': - specifier: ^1.1.0 - version: 1.1.0 + '@livekit/av': + specifier: ^10.0.0 + version: 10.0.0 fastest-levenshtein: specifier: ^1.0.16 version: 1.0.16 @@ -1954,49 +1954,6 @@ packages: resolution: {integrity: sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} - '@ffmpeg-installer/darwin-arm64@4.1.5': - resolution: {integrity: sha512-hYqTiP63mXz7wSQfuqfFwfLOfwwFChUedeCVKkBtl/cliaTM7/ePI9bVzfZ2c+dWu3TqCwLDRWNSJ5pqZl8otA==} - cpu: [arm64] - os: [darwin] - - '@ffmpeg-installer/darwin-x64@4.1.0': - resolution: {integrity: sha512-Z4EyG3cIFjdhlY8wI9aLUXuH8nVt7E9SlMVZtWvSPnm2sm37/yC2CwjUzyCQbJbySnef1tQwGG2Sx+uWhd9IAw==} - cpu: [x64] - os: [darwin] - - '@ffmpeg-installer/ffmpeg@1.1.0': - resolution: {integrity: sha512-Uq4rmwkdGxIa9A6Bd/VqqYbT7zqh1GrT5/rFwCwKM70b42W5gIjWeVETq6SdcL0zXqDtY081Ws/iJWhr1+xvQg==} - - '@ffmpeg-installer/linux-arm64@4.1.4': - resolution: {integrity: sha512-dljEqAOD0oIM6O6DxBW9US/FkvqvQwgJ2lGHOwHDDwu/pX8+V0YsDL1xqHbj1DMX/+nP9rxw7G7gcUvGspSoKg==} - cpu: [arm64] - os: [linux] - - '@ffmpeg-installer/linux-arm@4.1.3': - resolution: {integrity: sha512-NDf5V6l8AfzZ8WzUGZ5mV8O/xMzRag2ETR6+TlGIsMHp81agx51cqpPItXPib/nAZYmo55Bl2L6/WOMI3A5YRg==} - cpu: [arm] - os: [linux] - - '@ffmpeg-installer/linux-ia32@4.1.0': - resolution: {integrity: sha512-0LWyFQnPf+Ij9GQGD034hS6A90URNu9HCtQ5cTqo5MxOEc7Rd8gLXrJvn++UmxhU0J5RyRE9KRYstdCVUjkNOQ==} - cpu: [ia32] - os: [linux] - - '@ffmpeg-installer/linux-x64@4.1.0': - resolution: {integrity: sha512-Y5BWhGLU/WpQjOArNIgXD3z5mxxdV8c41C+U15nsE5yF8tVcdCGet5zPs5Zy3Ta6bU7haGpIzryutqCGQA/W8A==} - cpu: [x64] - os: [linux] - - '@ffmpeg-installer/win32-ia32@4.1.0': - resolution: {integrity: sha512-FV2D7RlaZv/lrtdhaQ4oETwoFUsUjlUiasiZLDxhEUPdNDWcH1OU9K1xTvqz+OXLdsmYelUDuBS/zkMOTtlUAw==} - cpu: [ia32] - os: [win32] - - '@ffmpeg-installer/win32-x64@4.1.0': - resolution: {integrity: sha512-Drt5u2vzDnIONf4ZEkKtFlbvwj6rI3kxw1Ck9fpudmtgaZIHD4ucsWB2lCZBXRxJgXR+2IMSti+4rtM4C4rXgg==} - cpu: [x64] - os: [win32] - '@google/genai@1.52.0': resolution: {integrity: sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==} engines: {node: '>=20.0.0'} @@ -6002,41 +5959,6 @@ snapshots: '@eslint/js@8.57.0': {} - '@ffmpeg-installer/darwin-arm64@4.1.5': - optional: true - - '@ffmpeg-installer/darwin-x64@4.1.0': - optional: true - - '@ffmpeg-installer/ffmpeg@1.1.0': - optionalDependencies: - '@ffmpeg-installer/darwin-arm64': 4.1.5 - '@ffmpeg-installer/darwin-x64': 4.1.0 - '@ffmpeg-installer/linux-arm': 4.1.3 - '@ffmpeg-installer/linux-arm64': 4.1.4 - '@ffmpeg-installer/linux-ia32': 4.1.0 - '@ffmpeg-installer/linux-x64': 4.1.0 - '@ffmpeg-installer/win32-ia32': 4.1.0 - '@ffmpeg-installer/win32-x64': 4.1.0 - - '@ffmpeg-installer/linux-arm64@4.1.4': - optional: true - - '@ffmpeg-installer/linux-arm@4.1.3': - optional: true - - '@ffmpeg-installer/linux-ia32@4.1.0': - optional: true - - '@ffmpeg-installer/linux-x64@4.1.0': - optional: true - - '@ffmpeg-installer/win32-ia32@4.1.0': - optional: true - - '@ffmpeg-installer/win32-x64@4.1.0': - optional: true - '@google/genai@1.52.0': dependencies: google-auth-library: 10.6.2