add zoom live scribe api support

audio/send_audio_to_zoom_scribe_transcribe_service_js/.env.example

@@ -21,15 +21,15 @@ zoomWSURLForEvents=
 # These are separate from ZOOM_CLIENT_ID / ZOOM_CLIENT_SECRET used for RTMS.
 ZOOM_API_KEY=
 ZOOM_API_SECRET=
-SCRIBE_BASE_URL=https://api.zoom.us/v2
+# Full WebSocket URL of the live transcription endpoint.
+SCRIBE_LIVE_URL=wss://api.zoom.us/v2/aiservices/scribe/live
 
-# Scribe transcription settings
+# Live Scribe (real-time streaming) settings.
 SCRIBE_LANGUAGE=en-US
-SCRIBE_WINDOW_SECONDS=10
-SCRIBE_MAX_WINDOWS=24
 SCRIBE_WORD_TIME_OFFSETS=true
 SCRIBE_TIMESTAMPS=true
 SCRIBE_DIARIZATION=false
 SCRIBE_CHANNEL_SEPARATION=false
 SCRIBE_PROFANITY_FILTER=false
 SCRIBE_OUTPUT_FORMAT=json
+

audio/send_audio_to_zoom_scribe_transcribe_service_js/README.md

@@ -1,19 +1,23 @@
-# Send Audio to Zoom Scribe Transcription Service
+# Send Audio to Zoom Scribe Live Transcription Service
 
-Stream Zoom RTMS meeting audio into short WAV windows and transcribe each window with the Zoom AI Services Scribe API fast-mode endpoint.
+Stream Zoom RTMS meeting audio into the Zoom AI Services Scribe **live** API — a
+real-time transcription WebSocket — and log transcripts as they arrive.
 
-> Built with `RTMSManager` and Zoom AI Services Scribe fast mode.
+> Built with `RTMSManager` and Zoom AI Services Scribe live streaming.
 
 ## What This Sample Does
 
 - Receives `meeting.rtms_started` and `meeting.rtms_stopped` events through `WebhookManager` or `WebsocketManager`.
 - Connects RTMSManager to the meeting media stream.
 - Requests audio-only RTMS media as 16 kHz mono L16 mixed audio.
-- Buffers RTMS PCM audio into short WAV chunks under `audio_windows/`.
-- Sends each chunk to Scribe fast mode with `POST /aiservices/scribe/transcribe`.
-- Logs each returned transcript to the console.
+- On `meeting.rtms_started`, opens a Scribe **live** WebSocket (`/aiservices/scribe/live`).
+- Forwards each RTMS PCM packet straight to the WebSocket as a binary frame (no file buffering, no resampling).
+- Logs `transcription.completed` events as they stream back.
+- On `meeting.rtms_stopped`, sends `session.close`, waits for the final transcript, and closes the socket.
 
-Scribe is file-oriented, not a live streaming WebSocket. This sample is a pseudo-streaming pattern: it sends one WAV chunk every `SCRIBE_WINDOW_SECONDS` seconds, waits for the fast-mode response, and then logs the returned transcript.
+Unlike the fast-mode `/transcribe` endpoint (which uploads whole audio files), the
+live endpoint is a true streaming WebSocket: audio flows in continuously and
+transcripts come back with low latency while the meeting is still in progress.
 
 ## Quick Start
 
@@ -52,17 +56,15 @@ ZOOM_API_KEY=
 ZOOM_API_SECRET=
 ```
 
-`ZOOM_API_KEY` and `ZOOM_API_SECRET` are the Zoom AI Services / Build-platform credentials used to sign the Scribe JWT.
+`ZOOM_API_KEY` and `ZOOM_API_SECRET` are the Zoom AI Services / Build-platform credentials used to sign the Scribe JWT (the same credential the fast-mode endpoint used).
 
 ## Optional Environment Variables
 
 ```env
 PORT=3000
 WEBHOOK_PATH=/webhook
-SCRIBE_BASE_URL=https://api.zoom.us/v2
+SCRIBE_LIVE_URL=wss://api.zoom.us/v2/aiservices/scribe/live
 SCRIBE_LANGUAGE=en-US
-SCRIBE_WINDOW_SECONDS=10
-SCRIBE_MAX_WINDOWS=24
 SCRIBE_WORD_TIME_OFFSETS=true
 SCRIBE_TIMESTAMPS=true
 SCRIBE_DIARIZATION=false
@@ -71,36 +73,42 @@ SCRIBE_PROFANITY_FILTER=false
 SCRIBE_OUTPUT_FORMAT=json
 ```
 
-Recommended starting window size is `10` seconds. Lower values give faster partial results but increase upload overhead and can cut words across windows.
+`SCRIBE_LIVE_URL` is the full WebSocket URL of the live transcription endpoint
+(defaults to `wss://api.zoom.us/v2/aiservices/scribe/live`).
 
 ## How It Works
 
 1. The app starts an Express server and initializes RTMSManager.
 2. The webhook endpoint receives `meeting.rtms_started`.
-3. RTMSManager connects to Zoom signaling/media sockets.
-4. RTMS audio packets arrive as raw 16 kHz mono PCM.
-5. `audioWindowBuffer.js` wraps each `SCRIBE_WINDOW_SECONDS` window in a WAV container.
-6. `scribeClient.js` signs a Build-platform JWT and submits the WAV chunk to Scribe fast mode.
-7. The returned transcript text is logged as `[ZoomScribe] Transcript result`.
+3. `scribeClient.js` mints a Build-platform JWT and opens `wss://.../aiservices/scribe/live`, then sends `session.update` (`audio.format=pcm16`, `language`).
+4. RTMSManager connects to Zoom signaling/media sockets; RTMS audio arrives as raw 16 kHz mono PCM16.
+5. Each audio packet is forwarded to the WebSocket as a binary frame. Audio that arrives before the session is ready is buffered and flushed on `session.updated`.
+6. The server streams back `transcription.completed` events, which are logged.
+7. On `meeting.rtms_stopped`, the client sends `session.close`, waits briefly for the final transcript, logs the full meeting transcript, and closes.
 
-## Scribe Request Shape
-
-The sample uses multipart upload:
+## Live WebSocket Protocol
 
 ```text
-POST https://api.zoom.us/v2/aiservices/scribe/transcribe
-Authorization: Bearer <Build-platform JWT>
-file=<WAV file>
-config={"language":"en-US","word_time_offsets":true,"timestamps":true}
+Connect: wss://api.zoom.us/v2/aiservices/scribe/live
+  Subprotocols: ["live-asr", "zoom-api-access-token.<Build-platform JWT>"]
+  (the JWT is carried in the "zoom-api-access-token.*" subprotocol)
+
+Client -> { "type": "session.update", "audio": { "format": "pcm16" }, "language": "en-US" }
+Client -> <binary PCM16 frames>                       # streamed RTMS audio
+Client -> { "type": "session.close" }                 # on meeting stop
+
+Server -> { "type": "session.created", "session_id": ... }
+Server -> { "type": "session.updated" }               # ready to receive audio
+Server -> { "type": "transcription.completed", "transcript": ..., "audio_start_ms": ..., "audio_end_ms": ... }
+Server -> { "type": "session.closed", "reason": ... }
 ```
 
 ## Files
 
 | File | Purpose |
 |------|---------|
-| `index.js` | Express, RTMSManager, webhook/websocket trigger, transcription queue |
-| `audioWindowBuffer.js` | Converts RTMS L16 PCM chunks into WAV windows |
-| `scribeClient.js` | Zoom Scribe JWT auth and fast-mode transcription client |
+| `index.js` | Express, RTMSManager, webhook/websocket trigger, forwards RTMS audio to the live client |
+| `scribeClient.js` | Scribe JWT auth + live streaming WebSocket client (connect, stream, event handling, cleanup) |
 | `.env.example` | Configuration template |
 
 ## Troubleshooting
@@ -109,11 +117,11 @@ config={"language":"en-US","word_time_offsets":true,"timestamps":true}
 |-------|-------|
 | Missing credential error | Set `ZOOM_CLIENT_ID`, `ZOOM_CLIENT_SECRET`, `ZOOM_SECRET_TOKEN`, `ZOOM_API_KEY`, and `ZOOM_API_SECRET`. |
 | No webhook received | Confirm the public HTTPS webhook URL points to `/webhook` or your configured `WEBHOOK_PATH`. |
-| No transcript text | Confirm RTMS is receiving audio and `audio_windows/` contains WAV files. |
-| Scribe 401/403 | Confirm `ZOOM_API_KEY` and `ZOOM_API_SECRET` are AI Services / Build-platform credentials. |
-| Slow updates | Lower `SCRIBE_WINDOW_SECONDS`, but expect more upload overhead. |
+| WebSocket 401/403 on connect | Confirm `ZOOM_API_KEY`/`ZOOM_API_SECRET` are AI Services / Build-platform credentials. |
+| Connects but no transcripts | Confirm RTMS is delivering audio (watch the `chunks=`/`sentBytes=` log line) and that `session.updated` was received. |
 
 ## Notes
 
-- This sample uses mixed meeting audio. For per-participant audio, request RTMS audio multi-streams and route windows by RTMS `userId`.
-- Scribe fast mode is best for short windows. For long recordings or archives, use Scribe batch jobs instead.
+- This sample uses mixed meeting audio. For per-participant audio, request RTMS audio multi-streams and open one live session per RTMS `userId`.
+- The live session has a server-side maximum duration; very long meetings may be closed by the server (the client logs `session.closed` with the reason). For archival transcription of long recordings, use Scribe batch jobs instead.
+- RTMS L16 at 16 kHz mono matches the live API's required `pcm16` format exactly, so audio is forwarded verbatim with no resampling or WAV wrapping.

audio/send_audio_to_zoom_scribe_transcribe_service_js/index.js

@@ -7,8 +7,14 @@ import { fileURLToPath } from 'url';
 import { RTMSManager } from '../../library/javascript/rtmsManager/RTMSManager.js';
 import WebhookManager from '../../library/javascript/webhookManager/WebhookManager.js';
 import WebsocketManager from '../../library/javascript/webSocketManager/WebsocketManager.js';
-import { AudioWindowBuffer } from './audioWindowBuffer.js';
-import { ScribeClient } from './scribeClient.js';
+import {
+  initializeLiveScribeSession,
+  sendAudioChunk,
+  cleanupMeeting,
+  closeLiveScribe,
+  liveScribeConfig,
+  activeSessionCount
+} from './scribeClient.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -70,6 +76,8 @@ const rtmsConfig = {
   },
   mediaParams: {
     audio: {
+      // The live Scribe API expects 16 kHz mono PCM16 (LE) — request exactly that
+      // from RTMS so audio can be forwarded verbatim, with no resampling.
       contentType: MEDIA_PARAMS.MEDIA_CONTENT_TYPE_RAW_AUDIO,
       sampleRate: MEDIA_PARAMS.AUDIO_SAMPLE_RATE_SR_16K,
       channel: MEDIA_PARAMS.AUDIO_CHANNEL_MONO,
@@ -82,97 +90,47 @@ const rtmsConfig = {
 
 const app = express();
 const server = http.createServer(app);
-const audioWindows = new AudioWindowBuffer({
-  outputDir: path.join(__dirname, 'audio_windows'),
-  sampleRate: 16000,
-  channels: 1,
-  bitsPerSample: 16,
-  windowSeconds: envNumber('SCRIBE_WINDOW_SECONDS', 10),
-  maxWindows: envNumber('SCRIBE_MAX_WINDOWS', 24)
-});
-const scribeClient = ScribeClient.fromEnv(process.env);
 
 let activeMeetingId = null;
 let activeStreamId = null;
-let transcriptionInFlight = false;
-const transcriptionQueue = [];
 
 console.log('[ZoomScribe] App Configuration:', appConfig);
 console.log('[ZoomScribe] RTMS Configuration:', RTMSManager.redactSecrets(rtmsConfig));
-console.log('[ZoomScribe] Scribe Configuration:', {
-  baseUrl: scribeClient.baseUrl,
-  language: scribeClient.language,
-  windowSeconds: audioWindows.windowSeconds,
-  sampleRate: audioWindows.sampleRate,
-  channels: audioWindows.channels
-});
+console.log('[ZoomScribe] Live Scribe Configuration:', liveScribeConfig());
 
 app.use(express.json());
 
 app.get('/health', (req, res) => {
   res.json({
     ok: true,
+    mode: 'live',
     activeMeetingId,
     activeStreamId,
-    queuedWindows: transcriptionQueue.length,
-    transcriptionInFlight
+    liveSessions: activeSessionCount()
   });
 });
 
 await RTMSManager.init(rtmsConfig);
 
-async function transcribeWindow(window) {
-  const startedAt = Date.now();
-  const result = await scribeClient.transcribeFile(window.filePath, {
-    meetingId: activeMeetingId,
-    streamId: activeStreamId,
-    window
-  });
-
-  console.log('[ZoomScribe] Transcript result:', {
-    fileName: window.fileName,
-    requestId: result.requestId,
-    model: result.model,
-    durationSec: result.durationSec,
-    elapsedMs: Date.now() - startedAt,
-    text: result.text || '(no transcript text)'
-  });
-}
-
-async function drainTranscriptionQueue() {
-  if (transcriptionInFlight) return;
-  transcriptionInFlight = true;
-
-  try {
-    while (transcriptionQueue.length > 0) {
-      const window = transcriptionQueue.shift();
-      try {
-        await transcribeWindow(window);
-      } catch (error) {
-        console.error('[ZoomScribe] Transcription failed:', {
-          fileName: window.fileName,
-          message: error.message
-        });
-      }
-    }
-  } finally {
-    transcriptionInFlight = false;
-  }
-}
-
+// Open/close the live transcription WebSocket in step with the meeting lifecycle.
 function updateActiveRtmsState(event, payload = {}) {
   if (event === 'meeting.rtms_started') {
     activeMeetingId = payload.meeting_uuid;
     activeStreamId = payload.rtms_stream_id;
-    audioWindows.reset();
-    transcriptionQueue.length = 0;
+    initializeLiveScribeSession(activeMeetingId);
   }
 
   if (event === 'meeting.rtms_stopped') {
-    activeMeetingId = null;
-    activeStreamId = null;
-    audioWindows.reset();
-    transcriptionQueue.length = 0;
+    const endingMeetingId = payload.meeting_uuid || activeMeetingId;
+    if (endingMeetingId) {
+      cleanupMeeting(endingMeetingId).catch((error) => {
+        console.error('[ZoomScribe] Live session cleanup failed:', error.message);
+      });
+    }
+    if (endingMeetingId === activeMeetingId) {
+      activeMeetingId = null;
+      activeStreamId = null;
+    }
   }
 }
 
@@ -212,29 +170,10 @@ if (appConfig.managerType === 'webhook') {
   console.log('[ZoomScribe] Websocket Manager initialized');
 }
 
-RTMSManager.on('audio', ({ buffer, userId, userName, timestamp, streamId }) => {
-  const windows = audioWindows.writeAudio(buffer, {
-    meetingId: activeMeetingId,
-    streamId,
-    userId: userId ?? null,
-    userName: userName ?? null,
-    timestamp: timestamp ?? Date.now()
-  });
-
-  for (const window of windows) {
-    console.log('[ZoomScribe] Audio window ready:', {
-      fileName: window.fileName,
-      bytes: window.size,
-      durationSeconds: window.durationSeconds,
-      sampleCount: window.sampleCount,
-      userName: window.userName
-    });
-    transcriptionQueue.push(window);
-  }
-
-  drainTranscriptionQueue().catch((error) => {
-    console.error('[ZoomScribe] Queue drain failed:', error.message);
-  });
+// Forward each RTMS audio packet straight to the live Scribe WebSocket.
+RTMSManager.on('audio', ({ buffer, userId }) => {
+  if (!activeMeetingId) return;
+  sendAudioChunk(buffer, activeMeetingId, userId ?? 0);
 });
 
 RTMSManager.on('error', (error) => {
@@ -251,6 +190,7 @@ server.listen(appConfig.port, () => {
 process.on('SIGINT', async () => {
   console.log('[ZoomScribe] Shutting down...');
   server.close();
+  await closeLiveScribe();
   await RTMSManager.stop();
   process.exit(0);
 });