From 55dc28e54ed445c4c9022fae0e25a05e720b76b4 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:11:56 -0400 Subject: [PATCH 1/3] fix: let the voice answer small talk itself again #2490 made GPT-Live delegate every utterance to Fast, greetings included, and forbade it from answering anything itself. On a real call that doubled the replies (the voice still answered, then read Fast's answer for the same greeting seconds later) and made the transcript read out of order, since a five-second Fast turn for "hi there" lands after the next question. It also turned the silence backstop into a Fast submission and latched the call into fallback-only mode after a single missed delegation. This restores the previous policy: greetings, thanks, reactions, and small talk stay with the voice and are recorded as heard and spoken turns; anything about code, tools, data, or the product goes to Fast. Two parts of #2490 are kept because they address the real problem it targeted: a grounding policy that forbids the voice from claiming it checked anything or stating how any product, repository, or tool works without backend commentary, and the "unverified" tag on direct voice words in Fast's history. --- .../voice-small-talk-stays-with-voice.md | 5 + apps/docs/voice.mdx | 6 +- .../FastSessionTranscript.client.test.tsx | 13 +- .../[sessionId]/FastSessionTranscript.tsx | 7 +- .../src/hooks/useLiveVoice.client.test.tsx | 117 +++++------------- apps/web/src/hooks/useLiveVoice.ts | 46 ++++--- apps/web/src/lib/server/voice.test.ts | 24 +--- apps/web/src/lib/server/voice.ts | 15 ++- .../web/src/trpc/commands/voice/index.test.ts | 2 +- apps/web/src/trpc/commands/voice/index.ts | 8 +- 10 files changed, 104 insertions(+), 139 deletions(-) create mode 100644 .changeset/voice-small-talk-stays-with-voice.md diff --git a/.changeset/voice-small-talk-stays-with-voice.md b/.changeset/voice-small-talk-stays-with-voice.md new file mode 100644 index 0000000000..fdec03f6cf --- /dev/null +++ b/.changeset/voice-small-talk-stays-with-voice.md @@ -0,0 +1,5 @@ +--- +"@roomote/web": patch +--- + +Voice answers greetings and small talk itself again instead of starting a Fast turn for every utterance, which doubled replies and read out of order; it still never states facts about code, tools, or the product without Fast. diff --git a/apps/docs/voice.mdx b/apps/docs/voice.mdx index 04fac38102..0006b89ffd 100644 --- a/apps/docs/voice.mdx +++ b/apps/docs/voice.mdx @@ -37,9 +37,9 @@ and receives only the negotiated session answer; it never receives the API key. confirms the call is open; a falling tone marks the end. A **Call started** marker appears in the Session. 3. Talk to Roomote the way you would on a phone call. It acknowledges each - utterance in a few words, hands it to the Fast Session, and reports the - response out loud when it lands. Every utterance is sent to the Fast Session, - where the selected model, tools, context, and safeguards handle the response. + request in a few words, hands the work to the Fast Session, and reports the + result out loud when it lands. Greetings, thanks, and small talk are + answered directly without starting Fast work. 4. Speak at any time to interrupt. Roomote keeps listening while it speaks, and follow-ups go back through the same Fast Session. You can also type in the composer during the call. diff --git a/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.client.test.tsx b/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.client.test.tsx index 380e0e5a4b..47c4f22d54 100644 --- a/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.client.test.tsx +++ b/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.client.test.tsx @@ -71,6 +71,7 @@ const { onUtterance: undefined as | ((text: string, delegationId: string | null) => void) | undefined, + onHeardTurn: undefined as ((text: string) => void) | undefined, onSpokenTurn: undefined as ((text: string) => void) | undefined, onHeardTurnDelta: undefined as ((text: string) => void) | undefined, onSpokenTurnDelta: undefined as ((text: string) => void) | undefined, @@ -80,16 +81,19 @@ const { vi.mock('@/hooks/useLiveVoice', () => ({ useLiveVoice: ({ onUtterance, + onHeardTurn, onSpokenTurn, onHeardTurnDelta, onSpokenTurnDelta, }: { onUtterance: (text: string, delegationId: string | null) => void; + onHeardTurn?: (text: string) => void; onSpokenTurn?: (text: string) => void; onHeardTurnDelta?: (text: string) => void; onSpokenTurnDelta?: (text: string) => void; }) => { liveVoiceState.onUtterance = onUtterance; + liveVoiceState.onHeardTurn = onHeardTurn; liveVoiceState.onSpokenTurn = onSpokenTurn; liveVoiceState.onHeardTurnDelta = onHeardTurnDelta; liveVoiceState.onSpokenTurnDelta = onSpokenTurnDelta; @@ -297,6 +301,7 @@ beforeEach(() => { recordVoiceCallEventMutate.mockResolvedValue({ eventId: 'voice-call:1' }); liveVoiceState.startedAt = null; liveVoiceState.deliveringUtterances = 0; + liveVoiceState.onHeardTurn = undefined; liveVoiceState.onSpokenTurn = undefined; liveVoiceState.active = false; liveVoiceState.status = 'idle'; @@ -3083,7 +3088,7 @@ describe('FastSessionTranscript', () => { expect(replyMutate).not.toHaveBeenCalled(); }); - it('transcribes the call into the Session: markers and spoken turns', async () => { + it('transcribes the call into the Session: markers, heard turns, and spoken turns', async () => { voiceStatusQuery.mockResolvedValue({ enabled: true }); const transcript = () => ( { ); act(() => { + liveVoiceState.onHeardTurn?.('Hi Roomote, how is it going'); liveVoiceState.onSpokenTurn?.('Good, thanks. What can I do for you?'); }); + expect(recordVoiceTurnMutate).toHaveBeenCalledWith({ + sessionId: 'session-1', + role: 'user', + text: 'Hi Roomote, how is it going', + }); expect(recordVoiceTurnMutate).toHaveBeenCalledWith({ sessionId: 'session-1', role: 'assistant', diff --git a/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.tsx b/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.tsx index 269961ac84..0f8f138b88 100644 --- a/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.tsx +++ b/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.tsx @@ -1123,6 +1123,7 @@ export function FastSessionTranscript({ const requestInFlightRef = useRef(false); const liveVoice = useLiveVoice({ onUtterance: enqueueVoiceUtterance, + onHeardTurn: (text) => recordVoiceTurnRef.current('user', text), onSpokenTurn: (text) => { if (requestInFlightRef.current) { heldSpokenTurnsRef.current.push(text); @@ -1258,9 +1259,9 @@ export function FastSessionTranscript({ } }, [messages, streamMessages, liveVoiceActive]); - // The call is transcribed into the Session: Fast turns record what the - // person said, this path records what the voice said, and call events mark - // where the conversation started and ended. + // The call is transcribed into the Session: what the person said when the + // voice answered directly, what the voice said, and where the call started + // and ended. Delegated requests are recorded by the Fast turn they start. const recordVoiceTurn = useCallback( (role: 'user' | 'assistant', text: string) => { // The finished words stay on screen until their persisted row arrives. diff --git a/apps/web/src/hooks/useLiveVoice.client.test.tsx b/apps/web/src/hooks/useLiveVoice.client.test.tsx index f3389ce54d..0cb09dea39 100644 --- a/apps/web/src/hooks/useLiveVoice.client.test.tsx +++ b/apps/web/src/hooks/useLiveVoice.client.test.tsx @@ -378,129 +378,72 @@ describe('useLiveVoice', () => { expect(playVoiceCue).not.toHaveBeenCalled(); }); - it('stays in fallback-only mode after a missed delegation', async () => { + it('records small talk GPT-Live handled itself as a heard turn, and what GPT-Live said as a spoken turn', async () => { const onUtterance = vi.fn(); - const { result } = renderHook(() => useLiveVoice({ onUtterance })); + const onHeardTurn = vi.fn(); + const onSpokenTurn = vi.fn(); + const { result } = renderHook(() => + useLiveVoice({ onUtterance, onHeardTurn, onSpokenTurn }), + ); await act(async () => result.current.start()); act(() => { FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', - delta: 'I want to talk about the browser, ', + delta: 'Thanks, that ', start_ms: 0, end_ms: 300, }); vi.advanceTimersByTime(1_000); FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', - delta: 'and how people might do more with it', + delta: 'looks right', start_ms: 300, end_ms: 600, }); vi.advanceTimersByTime(1_499); }); - expect(onUtterance).not.toHaveBeenCalled(); + expect(onHeardTurn).not.toHaveBeenCalled(); - await act(async () => { - vi.advanceTimersByTime(1); - }); - expect(cleanTranscriptMutate).toHaveBeenCalledWith({ - text: 'I want to talk about the browser, and how people might do more with it', - }); - expect(onUtterance).toHaveBeenCalledWith( - 'I want to talk about the browser, and how people might do more with it.', - null, - ); - - // Neither A's late delegation nor B's own delegation can be correlated - // after fallback, even well beyond the old three-second window. Both are - // ignored and B uses the same safe path. act(() => { - vi.advanceTimersByTime(10_000); - FakePeer.instance.channel.emit({ - type: 'session.delegation.created', - delegation: { id: 'item_late', target: 'client' }, - }); - vi.advanceTimersByTime(250); + vi.advanceTimersByTime(1); }); - expect(onUtterance).toHaveBeenCalledTimes(1); + expect(onHeardTurn).toHaveBeenCalledWith('Thanks, that looks right'); + expect(onUtterance).not.toHaveBeenCalled(); + // GPT-Live answers on its own; its words are recorded once it goes quiet. act(() => { FakePeer.instance.channel.emit({ - type: 'session.input_transcript.delta', - delta: 'Now check the build', - start_ms: 5_000, - end_ms: 5_400, + type: 'session.output_transcript.delta', + delta: 'Glad to ', }); FakePeer.instance.channel.emit({ - type: 'session.delegation.created', - delegation: { id: 'item_next', target: 'client' }, + type: 'session.output_transcript.delta', + delta: 'hear it.', }); - vi.advanceTimersByTime(1_500); + vi.advanceTimersByTime(1_200); }); - await act(async () => {}); - expect(onUtterance).toHaveBeenLastCalledWith('Now check the build.', null); - expect(onUtterance).toHaveBeenCalledTimes(2); + expect(onSpokenTurn).toHaveBeenCalledWith('Glad to hear it.'); - act(() => result.current.stop()); - await act(async () => result.current.start()); + // A delegation that shows up right after a silence flush belongs to that + // utterance and must not be held for the next one. act(() => { FakePeer.instance.channel.emit({ type: 'session.delegation.created', - delegation: { id: 'item_new_call', target: 'client' }, - }); - FakePeer.instance.channel.emit({ - type: 'session.input_transcript.delta', - delta: 'Fresh call', + delegation: { id: 'item_late', target: 'client' }, }); - vi.advanceTimersByTime(250); }); - await act(async () => {}); - expect(onUtterance).toHaveBeenLastCalledWith( - 'Fresh call.', - 'item_new_call', - ); - expect(onUtterance).toHaveBeenCalledTimes(3); - }); - - it('does not attach A late delegation after utterance B has started', async () => { - const onUtterance = vi.fn(); - const { result } = renderHook(() => useLiveVoice({ onUtterance })); - - await act(async () => result.current.start()); act(() => { FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', - delta: 'First request', + delta: 'Now check the build', + start_ms: 5_000, + end_ms: 5_400, }); vi.advanceTimersByTime(1_500); }); - await act(async () => {}); - expect(onUtterance).toHaveBeenCalledWith('First request.', null); - - act(() => { - FakePeer.instance.channel.emit({ - type: 'session.input_transcript.delta', - delta: 'Second ', - }); - FakePeer.instance.channel.emit({ - type: 'session.delegation.created', - delegation: { id: 'item_first_late', target: 'client' }, - }); - FakePeer.instance.channel.emit({ - type: 'session.input_transcript.delta', - delta: 'request', - }); - vi.advanceTimersByTime(250); - }); - expect(onUtterance).toHaveBeenCalledTimes(1); - - act(() => { - vi.advanceTimersByTime(1_250); - }); - await act(async () => {}); - expect(onUtterance).toHaveBeenLastCalledWith('Second request.', null); - expect(onUtterance).toHaveBeenCalledTimes(2); + expect(onHeardTurn).toHaveBeenLastCalledWith('Now check the build'); + expect(onUtterance).not.toHaveBeenCalled(); }); it('streams both sides of the call as they are spoken', async () => { @@ -549,9 +492,10 @@ describe('useLiveVoice', () => { it('drops sound annotations from what the person said', async () => { const onUtterance = vi.fn(); + const onHeardTurn = vi.fn(); const onHeardTurnDelta = vi.fn(); const { result } = renderHook(() => - useLiveVoice({ onUtterance, onHeardTurnDelta }), + useLiveVoice({ onUtterance, onHeardTurn, onHeardTurnDelta }), ); await act(async () => result.current.start()); @@ -583,7 +527,6 @@ describe('useLiveVoice', () => { }); // Annotation-only speech is not a turn at all. - onUtterance.mockClear(); act(() => { FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', @@ -593,7 +536,7 @@ describe('useLiveVoice', () => { }); vi.advanceTimersByTime(1_500); }); - expect(onUtterance).not.toHaveBeenCalled(); + expect(onHeardTurn).not.toHaveBeenCalled(); }); it('flushes the spoken turn when the person starts talking again', async () => { diff --git a/apps/web/src/hooks/useLiveVoice.ts b/apps/web/src/hooks/useLiveVoice.ts index 59bf051de8..72ee325e70 100644 --- a/apps/web/src/hooks/useLiveVoice.ts +++ b/apps/web/src/hooks/useLiveVoice.ts @@ -13,10 +13,13 @@ import { const DELEGATION_TRANSCRIPT_SETTLE_MS = 250; /** - * Backstop a GPT-Live delegation that never arrives. After this much silence, - * the utterance is sent through Fast without a delegation id. + * Speech GPT-Live answers itself never produces a delegation. After this much + * silence with no delegation the utterance is recorded as a heard turn so the + * Session transcript still has it. */ const UTTERANCE_SILENCE_FLUSH_MS = 1_500; +/** A delegation this soon after a silence flush belongs to that utterance. */ +const STALE_DELEGATION_WINDOW_MS = 3_000; /** GPT-Live has finished a spoken turn once its transcript stops growing. */ const SPOKEN_TURN_SETTLE_MS = 1_200; const SESSION_START_TIMEOUT_MS = 15_000; @@ -41,6 +44,12 @@ interface UseLiveVoiceOptions { * speaking a turn. This is the spoken record the Session persists. */ onSpokenTurn?: (text: string) => void; + /** + * Called with the raw transcript of what the person said each time GPT-Live + * handles it without delegating (small talk), so the Session still records + * it. Delegated utterances reach the Session through `onUtterance`. + */ + onHeardTurn?: (text: string) => void; /** Called with GPT-Live's words so far while it is speaking a turn. */ onSpokenTurnDelta?: (text: string) => void; /** Called with the person's words so far while they are speaking. */ @@ -111,6 +120,7 @@ async function waitForIceGathering(peer: RTCPeerConnection): Promise { export function useLiveVoice({ onUtterance, onSpokenTurn, + onHeardTurn, onSpokenTurnDelta, onHeardTurnDelta, disabled = false, @@ -125,6 +135,8 @@ export function useLiveVoice({ const [deliveringUtterances, setDeliveringUtterances] = useState(0); const onSpokenTurnRef = useRef(onSpokenTurn); onSpokenTurnRef.current = onSpokenTurn; + const onHeardTurnRef = useRef(onHeardTurn); + onHeardTurnRef.current = onHeardTurn; const onSpokenTurnDeltaRef = useRef(onSpokenTurnDelta); onSpokenTurnDeltaRef.current = onSpokenTurnDelta; const onHeardTurnDeltaRef = useRef(onHeardTurnDelta); @@ -154,10 +166,7 @@ export function useLiveVoice({ const pendingDelegationsRef = useRef([]); const delegationTimerRef = useRef(null); const silenceTimerRef = useRef(null); - // Delegations have no utterance identifier. After one is missed, accepting - // any later delegation could attach it to the wrong transcript, so the rest - // of this call uses the ordered silence fallback instead. - const fallbackOnlyRef = useRef(false); + const lastSilenceFlushAtRef = useRef(0); const speakingTimerRef = useRef(null); const deliveryChainRef = useRef>(Promise.resolve()); @@ -242,10 +251,9 @@ export function useLiveVoice({ ); }, [flushDelegation]); - // A missed or delayed GPT-Live delegation must not bypass Fast. Once the - // person has been quiet, submit the utterance without a delegation id. The - // call then stays in fallback-only mode because later delegation events - // cannot be correlated safely with a specific utterance. + // Speech GPT-Live handles itself (small talk) never produces a delegation. + // Once the person has been quiet for a moment, record what they said so the + // Session transcript stays the complete record of the call. const scheduleSilenceFlush = useCallback(() => { clearSilenceTimer(); silenceTimerRef.current = window.setTimeout(() => { @@ -253,11 +261,10 @@ export function useLiveVoice({ const utterance = stripVoiceAnnotations(inputTranscriptRef.current); if (pendingDelegationsRef.current.length > 0) return; inputTranscriptRef.current = ''; - if (!utterance) return; - fallbackOnlyRef.current = true; - deliverUtterance(utterance, null); + lastSilenceFlushAtRef.current = Date.now(); + if (utterance) onHeardTurnRef.current?.(utterance); }, UTTERANCE_SILENCE_FLUSH_MS); - }, [clearSilenceTimer, deliverUtterance]); + }, [clearSilenceTimer]); const handleServerEvent = useCallback( (raw: string) => { @@ -309,7 +316,15 @@ export function useLiveVoice({ break; case 'session.delegation.created': if (event.delegation?.target === 'client' && event.delegation.id) { - if (fallbackOnlyRef.current) break; + // A delegation arriving just after the silence flush already sent + // that utterance; attaching it to the next one would skew replies. + if ( + !inputTranscriptRef.current.trim() && + Date.now() - lastSilenceFlushAtRef.current < + STALE_DELEGATION_WINDOW_MS + ) { + break; + } pendingDelegationsRef.current.push(event.delegation.id); scheduleDelegationFlush(); } @@ -380,7 +395,6 @@ export function useLiveVoice({ inputTranscriptRef.current = ''; pendingDelegationsRef.current = []; - fallbackOnlyRef.current = false; setActive(false); setStatus('idle'); setStartedAt(null); diff --git a/apps/web/src/lib/server/voice.test.ts b/apps/web/src/lib/server/voice.test.ts index 4015e51ac5..8dc73b3713 100644 --- a/apps/web/src/lib/server/voice.test.ts +++ b/apps/web/src/lib/server/voice.test.ts @@ -108,30 +108,14 @@ describe('createVoiceLiveSession', () => { expect(body.session.instructions).toContain( 'Integrations the backend can use: GitHub.', ); - // The voice acknowledges, delegates every utterance, and reports results - // faithfully without originating answers or claims of inspection. + // The voice acknowledges, delegates real work, and reports results + // faithfully in its own words. expect(body.session.instructions).toContain('Backchannel policy'); + expect(body.session.instructions).toContain('Delegate to the backend when'); expect(body.session.instructions).toContain( - 'Delegate every complete utterance to the backend', - ); - expect(body.session.instructions).toContain( - 'Never answer, explain, clarify, offer an opinion, or state a fact yourself', - ); - expect(body.session.instructions).toContain( - 'Never claim that you checked a source', - ); - expect(body.session.instructions).toContain( - 'any product, repository, or connected tool discussed in the Session', - ); - expect(body.session.instructions).toContain( - 'Roomote when the platform itself is the topic', - ); - expect(body.session.instructions).not.toContain( - 'describe how Roomote works', - ); - expect(body.session.instructions).not.toContain( 'Do not delegate to the backend when', ); + expect(body.session.instructions).toContain('Grounding policy'); expect(body.session.instructions).toContain( 'keep every number, name, path, and link label exactly as given', ); diff --git a/apps/web/src/lib/server/voice.ts b/apps/web/src/lib/server/voice.ts index b17b62d818..3d42b8f1a9 100644 --- a/apps/web/src/lib/server/voice.ts +++ b/apps/web/src/lib/server/voice.ts @@ -177,7 +177,7 @@ The person will mostly talk about their code repositories, pull requests, issues ${formatVoiceWorkspaceContext(context)} -Backchannel policy: Acknowledge each utterance in a few words right away ("Sure.", "I'll check.") and then wait for the backend. Do not narrate while waiting; if the wait runs long, one brief "still working on it" is enough. +Backchannel policy: Acknowledge each request in a few words right away ("Sure.", "I'll check.") and then wait for the backend. Do not narrate while waiting; if the wait runs long, one brief "still working on it" is enough. Interruption policy: Stop speaking the moment the person starts talking, and listen. @@ -185,9 +185,16 @@ Delegation policy: Backend tools: - The backend is the Roomote Fast session: it reads and changes the repositories above, launches coding tasks in those environments, calls the listed integrations, reasons carefully, and returns results for you to report. -- Delegate every complete utterance to the backend, including greetings, thanks, reactions, small talk, corrections, follow-ups, and requests that need clarification. -- Your only self-generated speech is the brief acknowledgement above or one brief wait update. Never answer, explain, clarify, offer an opinion, or state a fact yourself. -- You cannot inspect code, documentation, tools, or deployment state yourself. Never claim that you checked a source or state how any product, repository, or connected tool discussed in the Session works unless backend commentary supplied that result. This includes Roomote when the platform itself is the topic. +Delegate to the backend when: +- The person asks about or for anything involving code, repositories, pull requests, issues, tasks, tools, data, or facts about their work. Anything you would have to guess at, delegate. +- The person corrects, refines, or follows up on earlier work. + +Do not delegate to the backend when: +- The person is only greeting you, thanking you, reacting ("cool", "nice"), or making small talk. Answer briefly yourself, and never start a Fast turn for it. +- You need a one-line clarification to understand what they mean before the backend could act. + +Grounding policy: +- You cannot inspect code, documentation, tools, or deployment state yourself. Never claim that you checked a source, and never state how any product, repository, or connected tool discussed in the Session works unless backend commentary supplied that result. This includes Roomote when the platform itself is the topic. When in doubt, delegate. Reporting policy: - Commentary is the backend's result. Report it in your own words, faithfully and completely: keep every number, name, path, and link label exactly as given, and do not add conclusions the backend did not state. Never claim work finished or a result exists before commentary says so. diff --git a/apps/web/src/trpc/commands/voice/index.test.ts b/apps/web/src/trpc/commands/voice/index.test.ts index 39c02a355a..34f30167f8 100644 --- a/apps/web/src/trpc/commands/voice/index.test.ts +++ b/apps/web/src/trpc/commands/voice/index.test.ts @@ -206,7 +206,7 @@ describe('cleanVoiceTranscriptCommand', () => { }); describe('recordVoiceTurnCommand', () => { - it('writes direct voice output as spoken but unverified in Fast history', async () => { + it('writes what the voice said as a spoken assistant turn and adds it to Fast history', async () => { mockFindAccessibleFastSession.mockResolvedValue({ id: 'fast-1' }); mockUpsertFastAgentMessage.mockResolvedValue({}); mockAppendFastAgentVisibleMessages.mockResolvedValue(undefined); diff --git a/apps/web/src/trpc/commands/voice/index.ts b/apps/web/src/trpc/commands/voice/index.ts index 1c8d311ae1..e097ea7a67 100644 --- a/apps/web/src/trpc/commands/voice/index.ts +++ b/apps/web/src/trpc/commands/voice/index.ts @@ -137,10 +137,10 @@ export async function cleanVoiceTranscriptCommand( const VOICE_MESSAGE_SOURCE = 'voice'; /** - * Record one spoken turn of a voice call in the Session transcript: legacy - * direct user speech (`user`), or what the voice said directly (`assistant`). - * Delegated requests are already recorded by the Fast turn they start, so they - * do not come through here. + * Record one spoken turn of a voice call in the Session transcript: what the + * person said when the voice answered them directly (`user`), or what the + * voice said (`assistant`). Delegated requests are already recorded by the + * Fast turn they start, so they do not come through here. * * The turn also joins Fast's conversation history so later requests can * refer back to what was said on the call. From 9ffbade8b87adec25fa970ed3192a9a363541372 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:20:45 -0400 Subject: [PATCH 2/3] fix: keep a delegation that arrives ahead of its request's transcript The three-second window after a small-talk flush dropped any delegation that arrived with nothing transcribed yet, including the delegation for the next request when GPT-Live emits it before that request's first transcript delta. The request was then filed as small talk and never reached Fast. A delegation that arrives with nothing transcribed now waits: speech arriving settles it as the next request's delegation, and only silence for three seconds drops it, so a late delegation for already-flushed small talk still cannot attach to a later request. --- .../src/hooks/useLiveVoice.client.test.tsx | 45 ++++++++++++++++-- apps/web/src/hooks/useLiveVoice.ts | 47 ++++++++++++++----- 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/apps/web/src/hooks/useLiveVoice.client.test.tsx b/apps/web/src/hooks/useLiveVoice.client.test.tsx index 0cb09dea39..1b3ba9edeb 100644 --- a/apps/web/src/hooks/useLiveVoice.client.test.tsx +++ b/apps/web/src/hooks/useLiveVoice.client.test.tsx @@ -425,12 +425,12 @@ describe('useLiveVoice', () => { }); expect(onSpokenTurn).toHaveBeenCalledWith('Glad to hear it.'); - // A delegation that shows up right after a silence flush belongs to that - // utterance and must not be held for the next one. + // A delegation that arrives before the next request's transcript is + // that request's delegation: the request must reach Fast. act(() => { FakePeer.instance.channel.emit({ type: 'session.delegation.created', - delegation: { id: 'item_late', target: 'client' }, + delegation: { id: 'item_next', target: 'client' }, }); }); act(() => { @@ -440,9 +440,46 @@ describe('useLiveVoice', () => { start_ms: 5_000, end_ms: 5_400, }); + vi.advanceTimersByTime(250); + }); + await act(async () => {}); + expect(onUtterance).toHaveBeenCalledWith( + 'Now check the build.', + 'item_next', + ); + expect(onHeardTurn).toHaveBeenCalledTimes(1); + }); + + it('drops a delegation that is never followed by speech, so it cannot attach to a later request', async () => { + const onUtterance = vi.fn(); + const onHeardTurn = vi.fn(); + const { result } = renderHook(() => + useLiveVoice({ onUtterance, onHeardTurn }), + ); + + await act(async () => result.current.start()); + // A late delegation for speech that was already flushed as small talk. + act(() => { + FakePeer.instance.channel.emit({ + type: 'session.delegation.created', + delegation: { id: 'item_stale', target: 'client' }, + }); + vi.advanceTimersByTime(3_000); + }); + + // The next thing said is small talk again: it is recorded as heard, not + // sent to Fast under the stale delegation. + act(() => { + FakePeer.instance.channel.emit({ + type: 'session.input_transcript.delta', + delta: 'Cool, thanks', + start_ms: 8_000, + end_ms: 8_400, + }); vi.advanceTimersByTime(1_500); }); - expect(onHeardTurn).toHaveBeenLastCalledWith('Now check the build'); + await act(async () => {}); + expect(onHeardTurn).toHaveBeenCalledWith('Cool, thanks'); expect(onUtterance).not.toHaveBeenCalled(); }); diff --git a/apps/web/src/hooks/useLiveVoice.ts b/apps/web/src/hooks/useLiveVoice.ts index 72ee325e70..b12fc5b6a5 100644 --- a/apps/web/src/hooks/useLiveVoice.ts +++ b/apps/web/src/hooks/useLiveVoice.ts @@ -18,8 +18,13 @@ const DELEGATION_TRANSCRIPT_SETTLE_MS = 250; * Session transcript still has it. */ const UTTERANCE_SILENCE_FLUSH_MS = 1_500; -/** A delegation this soon after a silence flush belongs to that utterance. */ -const STALE_DELEGATION_WINDOW_MS = 3_000; +/** + * A delegation that arrives with nothing transcribed belongs either to an + * utterance already flushed as small talk or to a request whose transcript is + * still on its way. Transcript arriving within this window settles it as the + * latter; otherwise it is dropped so it cannot attach to a later request. + */ +const UNSPOKEN_DELEGATION_EXPIRY_MS = 3_000; /** GPT-Live has finished a spoken turn once its transcript stops growing. */ const SPOKEN_TURN_SETTLE_MS = 1_200; const SESSION_START_TIMEOUT_MS = 15_000; @@ -166,7 +171,7 @@ export function useLiveVoice({ const pendingDelegationsRef = useRef([]); const delegationTimerRef = useRef(null); const silenceTimerRef = useRef(null); - const lastSilenceFlushAtRef = useRef(0); + const unspokenDelegationTimersRef = useRef(new Map()); const speakingTimerRef = useRef(null); const deliveryChainRef = useRef>(Promise.resolve()); @@ -261,7 +266,6 @@ export function useLiveVoice({ const utterance = stripVoiceAnnotations(inputTranscriptRef.current); if (pendingDelegationsRef.current.length > 0) return; inputTranscriptRef.current = ''; - lastSilenceFlushAtRef.current = Date.now(); if (utterance) onHeardTurnRef.current?.(utterance); }, UTTERANCE_SILENCE_FLUSH_MS); }, [clearSilenceTimer]); @@ -281,6 +285,11 @@ export function useLiveVoice({ // The person is talking again: whatever GPT-Live said is done. if (outputTranscriptRef.current) flushSpokenTurn(); inputTranscriptRef.current += event.delta; + // Speech arriving settles any delegation that came in ahead of it. + for (const timer of unspokenDelegationTimersRef.current.values()) { + window.clearTimeout(timer); + } + unspokenDelegationTimersRef.current.clear(); onHeardTurnDeltaRef.current?.( stripVoiceAnnotations(inputTranscriptRef.current), ); @@ -316,16 +325,24 @@ export function useLiveVoice({ break; case 'session.delegation.created': if (event.delegation?.target === 'client' && event.delegation.id) { - // A delegation arriving just after the silence flush already sent - // that utterance; attaching it to the next one would skew replies. - if ( - !inputTranscriptRef.current.trim() && - Date.now() - lastSilenceFlushAtRef.current < - STALE_DELEGATION_WINDOW_MS - ) { - break; + const delegationId = event.delegation.id; + pendingDelegationsRef.current.push(delegationId); + if (!inputTranscriptRef.current.trim()) { + // Nothing transcribed yet: either a late delegation for speech + // already flushed as small talk, or one for a request whose + // transcript is still arriving. Wait for speech to decide. + unspokenDelegationTimersRef.current.set( + delegationId, + window.setTimeout(() => { + unspokenDelegationTimersRef.current.delete(delegationId); + if (inputTranscriptRef.current.trim()) return; + pendingDelegationsRef.current = + pendingDelegationsRef.current.filter( + (id) => id !== delegationId, + ); + }, UNSPOKEN_DELEGATION_EXPIRY_MS), + ); } - pendingDelegationsRef.current.push(event.delegation.id); scheduleDelegationFlush(); } break; @@ -395,6 +412,10 @@ export function useLiveVoice({ inputTranscriptRef.current = ''; pendingDelegationsRef.current = []; + for (const timer of unspokenDelegationTimersRef.current.values()) { + window.clearTimeout(timer); + } + unspokenDelegationTimersRef.current.clear(); setActive(false); setStatus('idle'); setStartedAt(null); From 3712896f98949cec0f5c9cb75a63508de7bca4a6 Mon Sep 17 00:00:00 2001 From: Matt Rubens <2600+mrubens@users.noreply.github.com> Date: Fri, 11 Sep 2026 14:35:13 -0400 Subject: [PATCH 3/3] fix: attribute early delegations by session clock, not by the next speech --- .../src/hooks/useLiveVoice.client.test.tsx | 85 ++++++++++++- apps/web/src/hooks/useLiveVoice.ts | 118 +++++++++++++----- 2 files changed, 172 insertions(+), 31 deletions(-) diff --git a/apps/web/src/hooks/useLiveVoice.client.test.tsx b/apps/web/src/hooks/useLiveVoice.client.test.tsx index 1b3ba9edeb..7719cf63ac 100644 --- a/apps/web/src/hooks/useLiveVoice.client.test.tsx +++ b/apps/web/src/hooks/useLiveVoice.client.test.tsx @@ -425,11 +425,13 @@ describe('useLiveVoice', () => { }); expect(onSpokenTurn).toHaveBeenCalledWith('Glad to hear it.'); - // A delegation that arrives before the next request's transcript is - // that request's delegation: the request must reach Fast. + // A delegation made while the next request was being spoken, arriving + // before that request's transcript, is its delegation: the request must + // reach Fast. act(() => { FakePeer.instance.channel.emit({ type: 'session.delegation.created', + offset_ms: 5_900, delegation: { id: 'item_next', target: 'client' }, }); }); @@ -462,6 +464,7 @@ describe('useLiveVoice', () => { act(() => { FakePeer.instance.channel.emit({ type: 'session.delegation.created', + offset_ms: 2_000, delegation: { id: 'item_stale', target: 'client' }, }); vi.advanceTimersByTime(3_000); @@ -483,6 +486,84 @@ describe('useLiveVoice', () => { expect(onUtterance).not.toHaveBeenCalled(); }); + it('drops a late delegation for flushed small talk even when the next small talk starts right away', async () => { + const onUtterance = vi.fn(); + const onHeardTurn = vi.fn(); + const { result } = renderHook(() => + useLiveVoice({ onUtterance, onHeardTurn }), + ); + + await act(async () => result.current.start()); + act(() => { + FakePeer.instance.channel.emit({ + type: 'session.input_transcript.delta', + delta: 'Thanks', + start_ms: 0, + end_ms: 400, + }); + vi.advanceTimersByTime(1_500); + }); + expect(onHeardTurn).toHaveBeenCalledWith('Thanks'); + + // GPT-Live's delegation for "Thanks" lands after the silence flush, and + // the person starts talking again well inside the expiry window. + act(() => { + FakePeer.instance.channel.emit({ + type: 'session.delegation.created', + offset_ms: 2_200, + delegation: { id: 'item_stale', target: 'client' }, + }); + vi.advanceTimersByTime(500); + FakePeer.instance.channel.emit({ + type: 'session.input_transcript.delta', + delta: 'Cool, thanks', + start_ms: 2_900, + end_ms: 3_300, + }); + vi.advanceTimersByTime(1_500); + }); + await act(async () => {}); + expect(onHeardTurn).toHaveBeenLastCalledWith('Cool, thanks'); + expect(onUtterance).not.toHaveBeenCalled(); + }); + + it('sends a request under its own delegation, not a stale one that preceded it', async () => { + const onUtterance = vi.fn(); + const onHeardTurn = vi.fn(); + const { result } = renderHook(() => + useLiveVoice({ onUtterance, onHeardTurn }), + ); + + await act(async () => result.current.start()); + act(() => { + FakePeer.instance.channel.emit({ + type: 'session.delegation.created', + offset_ms: 2_200, + delegation: { id: 'item_stale', target: 'client' }, + }); + vi.advanceTimersByTime(500); + FakePeer.instance.channel.emit({ + type: 'session.input_transcript.delta', + delta: 'Now check the build', + start_ms: 2_900, + end_ms: 3_500, + }); + FakePeer.instance.channel.emit({ + type: 'session.delegation.created', + offset_ms: 4_100, + delegation: { id: 'item_fresh', target: 'client' }, + }); + vi.advanceTimersByTime(250); + }); + await act(async () => {}); + expect(onUtterance).toHaveBeenCalledTimes(1); + expect(onUtterance).toHaveBeenCalledWith( + 'Now check the build.', + 'item_fresh', + ); + expect(onHeardTurn).not.toHaveBeenCalled(); + }); + it('streams both sides of the call as they are spoken', async () => { const onSpokenTurnDelta = vi.fn(); const onHeardTurnDelta = vi.fn(); diff --git a/apps/web/src/hooks/useLiveVoice.ts b/apps/web/src/hooks/useLiveVoice.ts index b12fc5b6a5..423787ba6a 100644 --- a/apps/web/src/hooks/useLiveVoice.ts +++ b/apps/web/src/hooks/useLiveVoice.ts @@ -92,11 +92,27 @@ interface UseLiveVoiceReturn { type LiveServerEvent = { type?: string; delta?: string; + /** Session clock position at which GPT-Live emitted the event. */ offset_ms?: number; + /** Session clock range of a transcript fragment. */ + start_ms?: number; + end_ms?: number; delegation?: { id?: string; target?: string }; error?: { message?: string }; }; +interface PendingDelegation { + id: string; + /** When GPT-Live delegated, on the same clock as transcript fragments. */ + offsetMs: number | undefined; + /** + * Set while nothing has been transcribed since the delegation arrived. It + * discards the delegation if speech never follows, so a late delegation for + * speech already flushed as small talk cannot attach to a later request. + */ + expiryTimer: number | null; +} + async function waitForIceGathering(peer: RTCPeerConnection): Promise { if (peer.iceGatheringState === 'complete') return; @@ -168,10 +184,9 @@ export function useLiveVoice({ onUtteranceRef.current = onUtterance; const inputTranscriptRef = useRef(''); - const pendingDelegationsRef = useRef([]); + const pendingDelegationsRef = useRef([]); const delegationTimerRef = useRef(null); const silenceTimerRef = useRef(null); - const unspokenDelegationTimersRef = useRef(new Map()); const speakingTimerRef = useRef(null); const deliveryChainRef = useRef>(Promise.resolve()); @@ -233,19 +248,62 @@ export function useLiveVoice({ const flushDelegation = useCallback(() => { delegationTimerRef.current = null; - const delegationId = pendingDelegationsRef.current[0]; + const delegation = pendingDelegationsRef.current[0]; const utterance = inputTranscriptRef.current.trim(); // GPT-Live may delegate before the last transcript delta arrives. Retain // the delegation and let the next delta schedule another flush. - if (!delegationId || !utterance) return; + if (!delegation || !utterance) return; pendingDelegationsRef.current.shift(); inputTranscriptRef.current = ''; clearSilenceTimer(); - deliverUtterance(utterance, delegationId); + deliverUtterance(utterance, delegation.id); }, [clearSilenceTimer, deliverUtterance]); + const removePendingDelegation = useCallback( + (delegation: PendingDelegation) => { + if (delegation.expiryTimer !== null) { + window.clearTimeout(delegation.expiryTimer); + delegation.expiryTimer = null; + } + pendingDelegationsRef.current = pendingDelegationsRef.current.filter( + (pending) => pending !== delegation, + ); + }, + [], + ); + + /** + * Speech has started after delegations arrived with nothing transcribed. + * A delegation answers what GPT-Live was hearing when it delegated, so one + * made before this speech started belongs to earlier speech (already + * flushed as small talk) and must not carry this utterance to Fast. A + * delegation made while this speech was under way is this utterance's: its + * transcript merely lagged. + */ + const settleEarlyDelegations = useCallback( + (speechStartMs: number | undefined) => { + for (const delegation of [...pendingDelegationsRef.current]) { + if (delegation.expiryTimer === null) continue; + const madeBeforeSpeech = + speechStartMs !== undefined && + delegation.offsetMs !== undefined && + delegation.offsetMs <= speechStartMs; + if (madeBeforeSpeech) { + console.info( + `[voice] Ignoring delegation ${delegation.id}: made before this speech started`, + ); + removePendingDelegation(delegation); + } else { + window.clearTimeout(delegation.expiryTimer); + delegation.expiryTimer = null; + } + } + }, + [removePendingDelegation], + ); + const scheduleDelegationFlush = useCallback(() => { if (delegationTimerRef.current !== null) { window.clearTimeout(delegationTimerRef.current); @@ -284,12 +342,9 @@ export function useLiveVoice({ if (event.delta) { // The person is talking again: whatever GPT-Live said is done. if (outputTranscriptRef.current) flushSpokenTurn(); + const speechStarting = !inputTranscriptRef.current.trim(); inputTranscriptRef.current += event.delta; - // Speech arriving settles any delegation that came in ahead of it. - for (const timer of unspokenDelegationTimersRef.current.values()) { - window.clearTimeout(timer); - } - unspokenDelegationTimersRef.current.clear(); + if (speechStarting) settleEarlyDelegations(event.start_ms); onHeardTurnDeltaRef.current?.( stripVoiceAnnotations(inputTranscriptRef.current), ); @@ -325,24 +380,22 @@ export function useLiveVoice({ break; case 'session.delegation.created': if (event.delegation?.target === 'client' && event.delegation.id) { - const delegationId = event.delegation.id; - pendingDelegationsRef.current.push(delegationId); + const delegation: PendingDelegation = { + id: event.delegation.id, + offsetMs: event.offset_ms, + expiryTimer: null, + }; if (!inputTranscriptRef.current.trim()) { // Nothing transcribed yet: either a late delegation for speech // already flushed as small talk, or one for a request whose - // transcript is still arriving. Wait for speech to decide. - unspokenDelegationTimersRef.current.set( - delegationId, - window.setTimeout(() => { - unspokenDelegationTimersRef.current.delete(delegationId); - if (inputTranscriptRef.current.trim()) return; - pendingDelegationsRef.current = - pendingDelegationsRef.current.filter( - (id) => id !== delegationId, - ); - }, UNSPOKEN_DELEGATION_EXPIRY_MS), - ); + // transcript is still arriving. Speech starting decides by + // timestamp; silence for the whole window discards it. + delegation.expiryTimer = window.setTimeout(() => { + delegation.expiryTimer = null; + removePendingDelegation(delegation); + }, UNSPOKEN_DELEGATION_EXPIRY_MS); } + pendingDelegationsRef.current.push(delegation); scheduleDelegationFlush(); } break; @@ -354,7 +407,13 @@ export function useLiveVoice({ break; } }, - [flushSpokenTurn, scheduleDelegationFlush, scheduleSilenceFlush], + [ + flushSpokenTurn, + removePendingDelegation, + scheduleDelegationFlush, + scheduleSilenceFlush, + settleEarlyDelegations, + ], ); const release = useCallback( @@ -411,11 +470,12 @@ export function useLiveVoice({ release(peer, channel, mic, audio); inputTranscriptRef.current = ''; - pendingDelegationsRef.current = []; - for (const timer of unspokenDelegationTimersRef.current.values()) { - window.clearTimeout(timer); + for (const delegation of pendingDelegationsRef.current) { + if (delegation.expiryTimer !== null) { + window.clearTimeout(delegation.expiryTimer); + } } - unspokenDelegationTimersRef.current.clear(); + pendingDelegationsRef.current = []; setActive(false); setStatus('idle'); setStartedAt(null);