diff --git a/.changeset/voice-small-talk-stays-with-voice.md b/.changeset/voice-small-talk-stays-with-voice.md new file mode 100644 index 0000000000..fdec03f6cf --- /dev/null +++ b/.changeset/voice-small-talk-stays-with-voice.md @@ -0,0 +1,5 @@ +--- +"@roomote/web": patch +--- + +Voice answers greetings and small talk itself again instead of starting a Fast turn for every utterance, which doubled replies and read out of order; it still never states facts about code, tools, or the product without Fast. diff --git a/apps/docs/voice.mdx b/apps/docs/voice.mdx index 04fac38102..0006b89ffd 100644 --- a/apps/docs/voice.mdx +++ b/apps/docs/voice.mdx @@ -37,9 +37,9 @@ and receives only the negotiated session answer; it never receives the API key. confirms the call is open; a falling tone marks the end. A **Call started** marker appears in the Session. 3. Talk to Roomote the way you would on a phone call. It acknowledges each - utterance in a few words, hands it to the Fast Session, and reports the - response out loud when it lands. Every utterance is sent to the Fast Session, - where the selected model, tools, context, and safeguards handle the response. + request in a few words, hands the work to the Fast Session, and reports the + result out loud when it lands. Greetings, thanks, and small talk are + answered directly without starting Fast work. 4. Speak at any time to interrupt. Roomote keeps listening while it speaks, and follow-ups go back through the same Fast Session. You can also type in the composer during the call. diff --git a/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.client.test.tsx b/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.client.test.tsx index 380e0e5a4b..47c4f22d54 100644 --- a/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.client.test.tsx +++ b/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.client.test.tsx @@ -71,6 +71,7 @@ const { onUtterance: undefined as | ((text: string, delegationId: string | null) => void) | undefined, + onHeardTurn: undefined as ((text: string) => void) | undefined, onSpokenTurn: undefined as ((text: string) => void) | undefined, onHeardTurnDelta: undefined as ((text: string) => void) | undefined, onSpokenTurnDelta: undefined as ((text: string) => void) | undefined, @@ -80,16 +81,19 @@ const { vi.mock('@/hooks/useLiveVoice', () => ({ useLiveVoice: ({ onUtterance, + onHeardTurn, onSpokenTurn, onHeardTurnDelta, onSpokenTurnDelta, }: { onUtterance: (text: string, delegationId: string | null) => void; + onHeardTurn?: (text: string) => void; onSpokenTurn?: (text: string) => void; onHeardTurnDelta?: (text: string) => void; onSpokenTurnDelta?: (text: string) => void; }) => { liveVoiceState.onUtterance = onUtterance; + liveVoiceState.onHeardTurn = onHeardTurn; liveVoiceState.onSpokenTurn = onSpokenTurn; liveVoiceState.onHeardTurnDelta = onHeardTurnDelta; liveVoiceState.onSpokenTurnDelta = onSpokenTurnDelta; @@ -297,6 +301,7 @@ beforeEach(() => { recordVoiceCallEventMutate.mockResolvedValue({ eventId: 'voice-call:1' }); liveVoiceState.startedAt = null; liveVoiceState.deliveringUtterances = 0; + liveVoiceState.onHeardTurn = undefined; liveVoiceState.onSpokenTurn = undefined; liveVoiceState.active = false; liveVoiceState.status = 'idle'; @@ -3083,7 +3088,7 @@ describe('FastSessionTranscript', () => { expect(replyMutate).not.toHaveBeenCalled(); }); - it('transcribes the call into the Session: markers and spoken turns', async () => { + it('transcribes the call into the Session: markers, heard turns, and spoken turns', async () => { voiceStatusQuery.mockResolvedValue({ enabled: true }); const transcript = () => ( { ); act(() => { + liveVoiceState.onHeardTurn?.('Hi Roomote, how is it going'); liveVoiceState.onSpokenTurn?.('Good, thanks. What can I do for you?'); }); + expect(recordVoiceTurnMutate).toHaveBeenCalledWith({ + sessionId: 'session-1', + role: 'user', + text: 'Hi Roomote, how is it going', + }); expect(recordVoiceTurnMutate).toHaveBeenCalledWith({ sessionId: 'session-1', role: 'assistant', diff --git a/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.tsx b/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.tsx index 269961ac84..0f8f138b88 100644 --- a/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.tsx +++ b/apps/web/src/app/(sandbox)/sessions/[sessionId]/FastSessionTranscript.tsx @@ -1123,6 +1123,7 @@ export function FastSessionTranscript({ const requestInFlightRef = useRef(false); const liveVoice = useLiveVoice({ onUtterance: enqueueVoiceUtterance, + onHeardTurn: (text) => recordVoiceTurnRef.current('user', text), onSpokenTurn: (text) => { if (requestInFlightRef.current) { heldSpokenTurnsRef.current.push(text); @@ -1258,9 +1259,9 @@ export function FastSessionTranscript({ } }, [messages, streamMessages, liveVoiceActive]); - // The call is transcribed into the Session: Fast turns record what the - // person said, this path records what the voice said, and call events mark - // where the conversation started and ended. + // The call is transcribed into the Session: what the person said when the + // voice answered directly, what the voice said, and where the call started + // and ended. Delegated requests are recorded by the Fast turn they start. const recordVoiceTurn = useCallback( (role: 'user' | 'assistant', text: string) => { // The finished words stay on screen until their persisted row arrives. diff --git a/apps/web/src/hooks/useLiveVoice.client.test.tsx b/apps/web/src/hooks/useLiveVoice.client.test.tsx index f3389ce54d..7719cf63ac 100644 --- a/apps/web/src/hooks/useLiveVoice.client.test.tsx +++ b/apps/web/src/hooks/useLiveVoice.client.test.tsx @@ -378,53 +378,63 @@ describe('useLiveVoice', () => { expect(playVoiceCue).not.toHaveBeenCalled(); }); - it('stays in fallback-only mode after a missed delegation', async () => { + it('records small talk GPT-Live handled itself as a heard turn, and what GPT-Live said as a spoken turn', async () => { const onUtterance = vi.fn(); - const { result } = renderHook(() => useLiveVoice({ onUtterance })); + const onHeardTurn = vi.fn(); + const onSpokenTurn = vi.fn(); + const { result } = renderHook(() => + useLiveVoice({ onUtterance, onHeardTurn, onSpokenTurn }), + ); await act(async () => result.current.start()); act(() => { FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', - delta: 'I want to talk about the browser, ', + delta: 'Thanks, that ', start_ms: 0, end_ms: 300, }); vi.advanceTimersByTime(1_000); FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', - delta: 'and how people might do more with it', + delta: 'looks right', start_ms: 300, end_ms: 600, }); vi.advanceTimersByTime(1_499); }); - expect(onUtterance).not.toHaveBeenCalled(); + expect(onHeardTurn).not.toHaveBeenCalled(); - await act(async () => { + act(() => { vi.advanceTimersByTime(1); }); - expect(cleanTranscriptMutate).toHaveBeenCalledWith({ - text: 'I want to talk about the browser, and how people might do more with it', + expect(onHeardTurn).toHaveBeenCalledWith('Thanks, that looks right'); + expect(onUtterance).not.toHaveBeenCalled(); + + // GPT-Live answers on its own; its words are recorded once it goes quiet. + act(() => { + FakePeer.instance.channel.emit({ + type: 'session.output_transcript.delta', + delta: 'Glad to ', + }); + FakePeer.instance.channel.emit({ + type: 'session.output_transcript.delta', + delta: 'hear it.', + }); + vi.advanceTimersByTime(1_200); }); - expect(onUtterance).toHaveBeenCalledWith( - 'I want to talk about the browser, and how people might do more with it.', - null, - ); + expect(onSpokenTurn).toHaveBeenCalledWith('Glad to hear it.'); - // Neither A's late delegation nor B's own delegation can be correlated - // after fallback, even well beyond the old three-second window. Both are - // ignored and B uses the same safe path. + // A delegation made while the next request was being spoken, arriving + // before that request's transcript, is its delegation: the request must + // reach Fast. act(() => { - vi.advanceTimersByTime(10_000); FakePeer.instance.channel.emit({ type: 'session.delegation.created', - delegation: { id: 'item_late', target: 'client' }, + offset_ms: 5_900, + delegation: { id: 'item_next', target: 'client' }, }); - vi.advanceTimersByTime(250); }); - expect(onUtterance).toHaveBeenCalledTimes(1); - act(() => { FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', @@ -432,75 +442,126 @@ describe('useLiveVoice', () => { start_ms: 5_000, end_ms: 5_400, }); - FakePeer.instance.channel.emit({ - type: 'session.delegation.created', - delegation: { id: 'item_next', target: 'client' }, - }); - vi.advanceTimersByTime(1_500); + vi.advanceTimersByTime(250); }); await act(async () => {}); - expect(onUtterance).toHaveBeenLastCalledWith('Now check the build.', null); - expect(onUtterance).toHaveBeenCalledTimes(2); + expect(onUtterance).toHaveBeenCalledWith( + 'Now check the build.', + 'item_next', + ); + expect(onHeardTurn).toHaveBeenCalledTimes(1); + }); + + it('drops a delegation that is never followed by speech, so it cannot attach to a later request', async () => { + const onUtterance = vi.fn(); + const onHeardTurn = vi.fn(); + const { result } = renderHook(() => + useLiveVoice({ onUtterance, onHeardTurn }), + ); - act(() => result.current.stop()); await act(async () => result.current.start()); + // A late delegation for speech that was already flushed as small talk. act(() => { FakePeer.instance.channel.emit({ type: 'session.delegation.created', - delegation: { id: 'item_new_call', target: 'client' }, + offset_ms: 2_000, + delegation: { id: 'item_stale', target: 'client' }, }); + vi.advanceTimersByTime(3_000); + }); + + // The next thing said is small talk again: it is recorded as heard, not + // sent to Fast under the stale delegation. + act(() => { FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', - delta: 'Fresh call', + delta: 'Cool, thanks', + start_ms: 8_000, + end_ms: 8_400, }); - vi.advanceTimersByTime(250); + vi.advanceTimersByTime(1_500); }); await act(async () => {}); - expect(onUtterance).toHaveBeenLastCalledWith( - 'Fresh call.', - 'item_new_call', - ); - expect(onUtterance).toHaveBeenCalledTimes(3); + expect(onHeardTurn).toHaveBeenCalledWith('Cool, thanks'); + expect(onUtterance).not.toHaveBeenCalled(); }); - it('does not attach A late delegation after utterance B has started', async () => { + it('drops a late delegation for flushed small talk even when the next small talk starts right away', async () => { const onUtterance = vi.fn(); - const { result } = renderHook(() => useLiveVoice({ onUtterance })); + const onHeardTurn = vi.fn(); + const { result } = renderHook(() => + useLiveVoice({ onUtterance, onHeardTurn }), + ); await act(async () => result.current.start()); act(() => { FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', - delta: 'First request', + delta: 'Thanks', + start_ms: 0, + end_ms: 400, }); vi.advanceTimersByTime(1_500); }); - await act(async () => {}); - expect(onUtterance).toHaveBeenCalledWith('First request.', null); + expect(onHeardTurn).toHaveBeenCalledWith('Thanks'); + // GPT-Live's delegation for "Thanks" lands after the silence flush, and + // the person starts talking again well inside the expiry window. act(() => { + FakePeer.instance.channel.emit({ + type: 'session.delegation.created', + offset_ms: 2_200, + delegation: { id: 'item_stale', target: 'client' }, + }); + vi.advanceTimersByTime(500); FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', - delta: 'Second ', + delta: 'Cool, thanks', + start_ms: 2_900, + end_ms: 3_300, }); + vi.advanceTimersByTime(1_500); + }); + await act(async () => {}); + expect(onHeardTurn).toHaveBeenLastCalledWith('Cool, thanks'); + expect(onUtterance).not.toHaveBeenCalled(); + }); + + it('sends a request under its own delegation, not a stale one that preceded it', async () => { + const onUtterance = vi.fn(); + const onHeardTurn = vi.fn(); + const { result } = renderHook(() => + useLiveVoice({ onUtterance, onHeardTurn }), + ); + + await act(async () => result.current.start()); + act(() => { FakePeer.instance.channel.emit({ type: 'session.delegation.created', - delegation: { id: 'item_first_late', target: 'client' }, + offset_ms: 2_200, + delegation: { id: 'item_stale', target: 'client' }, }); + vi.advanceTimersByTime(500); FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', - delta: 'request', + delta: 'Now check the build', + start_ms: 2_900, + end_ms: 3_500, + }); + FakePeer.instance.channel.emit({ + type: 'session.delegation.created', + offset_ms: 4_100, + delegation: { id: 'item_fresh', target: 'client' }, }); vi.advanceTimersByTime(250); }); - expect(onUtterance).toHaveBeenCalledTimes(1); - - act(() => { - vi.advanceTimersByTime(1_250); - }); await act(async () => {}); - expect(onUtterance).toHaveBeenLastCalledWith('Second request.', null); - expect(onUtterance).toHaveBeenCalledTimes(2); + expect(onUtterance).toHaveBeenCalledTimes(1); + expect(onUtterance).toHaveBeenCalledWith( + 'Now check the build.', + 'item_fresh', + ); + expect(onHeardTurn).not.toHaveBeenCalled(); }); it('streams both sides of the call as they are spoken', async () => { @@ -549,9 +610,10 @@ describe('useLiveVoice', () => { it('drops sound annotations from what the person said', async () => { const onUtterance = vi.fn(); + const onHeardTurn = vi.fn(); const onHeardTurnDelta = vi.fn(); const { result } = renderHook(() => - useLiveVoice({ onUtterance, onHeardTurnDelta }), + useLiveVoice({ onUtterance, onHeardTurn, onHeardTurnDelta }), ); await act(async () => result.current.start()); @@ -583,7 +645,6 @@ describe('useLiveVoice', () => { }); // Annotation-only speech is not a turn at all. - onUtterance.mockClear(); act(() => { FakePeer.instance.channel.emit({ type: 'session.input_transcript.delta', @@ -593,7 +654,7 @@ describe('useLiveVoice', () => { }); vi.advanceTimersByTime(1_500); }); - expect(onUtterance).not.toHaveBeenCalled(); + expect(onHeardTurn).not.toHaveBeenCalled(); }); it('flushes the spoken turn when the person starts talking again', async () => { diff --git a/apps/web/src/hooks/useLiveVoice.ts b/apps/web/src/hooks/useLiveVoice.ts index 59bf051de8..423787ba6a 100644 --- a/apps/web/src/hooks/useLiveVoice.ts +++ b/apps/web/src/hooks/useLiveVoice.ts @@ -13,10 +13,18 @@ import { const DELEGATION_TRANSCRIPT_SETTLE_MS = 250; /** - * Backstop a GPT-Live delegation that never arrives. After this much silence, - * the utterance is sent through Fast without a delegation id. + * Speech GPT-Live answers itself never produces a delegation. After this much + * silence with no delegation the utterance is recorded as a heard turn so the + * Session transcript still has it. */ const UTTERANCE_SILENCE_FLUSH_MS = 1_500; +/** + * A delegation that arrives with nothing transcribed belongs either to an + * utterance already flushed as small talk or to a request whose transcript is + * still on its way. Transcript arriving within this window settles it as the + * latter; otherwise it is dropped so it cannot attach to a later request. + */ +const UNSPOKEN_DELEGATION_EXPIRY_MS = 3_000; /** GPT-Live has finished a spoken turn once its transcript stops growing. */ const SPOKEN_TURN_SETTLE_MS = 1_200; const SESSION_START_TIMEOUT_MS = 15_000; @@ -41,6 +49,12 @@ interface UseLiveVoiceOptions { * speaking a turn. This is the spoken record the Session persists. */ onSpokenTurn?: (text: string) => void; + /** + * Called with the raw transcript of what the person said each time GPT-Live + * handles it without delegating (small talk), so the Session still records + * it. Delegated utterances reach the Session through `onUtterance`. + */ + onHeardTurn?: (text: string) => void; /** Called with GPT-Live's words so far while it is speaking a turn. */ onSpokenTurnDelta?: (text: string) => void; /** Called with the person's words so far while they are speaking. */ @@ -78,11 +92,27 @@ interface UseLiveVoiceReturn { type LiveServerEvent = { type?: string; delta?: string; + /** Session clock position at which GPT-Live emitted the event. */ offset_ms?: number; + /** Session clock range of a transcript fragment. */ + start_ms?: number; + end_ms?: number; delegation?: { id?: string; target?: string }; error?: { message?: string }; }; +interface PendingDelegation { + id: string; + /** When GPT-Live delegated, on the same clock as transcript fragments. */ + offsetMs: number | undefined; + /** + * Set while nothing has been transcribed since the delegation arrived. It + * discards the delegation if speech never follows, so a late delegation for + * speech already flushed as small talk cannot attach to a later request. + */ + expiryTimer: number | null; +} + async function waitForIceGathering(peer: RTCPeerConnection): Promise { if (peer.iceGatheringState === 'complete') return; @@ -111,6 +141,7 @@ async function waitForIceGathering(peer: RTCPeerConnection): Promise { export function useLiveVoice({ onUtterance, onSpokenTurn, + onHeardTurn, onSpokenTurnDelta, onHeardTurnDelta, disabled = false, @@ -125,6 +156,8 @@ export function useLiveVoice({ const [deliveringUtterances, setDeliveringUtterances] = useState(0); const onSpokenTurnRef = useRef(onSpokenTurn); onSpokenTurnRef.current = onSpokenTurn; + const onHeardTurnRef = useRef(onHeardTurn); + onHeardTurnRef.current = onHeardTurn; const onSpokenTurnDeltaRef = useRef(onSpokenTurnDelta); onSpokenTurnDeltaRef.current = onSpokenTurnDelta; const onHeardTurnDeltaRef = useRef(onHeardTurnDelta); @@ -151,13 +184,9 @@ export function useLiveVoice({ onUtteranceRef.current = onUtterance; const inputTranscriptRef = useRef(''); - const pendingDelegationsRef = useRef([]); + const pendingDelegationsRef = useRef([]); const delegationTimerRef = useRef(null); const silenceTimerRef = useRef(null); - // Delegations have no utterance identifier. After one is missed, accepting - // any later delegation could attach it to the wrong transcript, so the rest - // of this call uses the ordered silence fallback instead. - const fallbackOnlyRef = useRef(false); const speakingTimerRef = useRef(null); const deliveryChainRef = useRef>(Promise.resolve()); @@ -219,19 +248,62 @@ export function useLiveVoice({ const flushDelegation = useCallback(() => { delegationTimerRef.current = null; - const delegationId = pendingDelegationsRef.current[0]; + const delegation = pendingDelegationsRef.current[0]; const utterance = inputTranscriptRef.current.trim(); // GPT-Live may delegate before the last transcript delta arrives. Retain // the delegation and let the next delta schedule another flush. - if (!delegationId || !utterance) return; + if (!delegation || !utterance) return; pendingDelegationsRef.current.shift(); inputTranscriptRef.current = ''; clearSilenceTimer(); - deliverUtterance(utterance, delegationId); + deliverUtterance(utterance, delegation.id); }, [clearSilenceTimer, deliverUtterance]); + const removePendingDelegation = useCallback( + (delegation: PendingDelegation) => { + if (delegation.expiryTimer !== null) { + window.clearTimeout(delegation.expiryTimer); + delegation.expiryTimer = null; + } + pendingDelegationsRef.current = pendingDelegationsRef.current.filter( + (pending) => pending !== delegation, + ); + }, + [], + ); + + /** + * Speech has started after delegations arrived with nothing transcribed. + * A delegation answers what GPT-Live was hearing when it delegated, so one + * made before this speech started belongs to earlier speech (already + * flushed as small talk) and must not carry this utterance to Fast. A + * delegation made while this speech was under way is this utterance's: its + * transcript merely lagged. + */ + const settleEarlyDelegations = useCallback( + (speechStartMs: number | undefined) => { + for (const delegation of [...pendingDelegationsRef.current]) { + if (delegation.expiryTimer === null) continue; + const madeBeforeSpeech = + speechStartMs !== undefined && + delegation.offsetMs !== undefined && + delegation.offsetMs <= speechStartMs; + if (madeBeforeSpeech) { + console.info( + `[voice] Ignoring delegation ${delegation.id}: made before this speech started`, + ); + removePendingDelegation(delegation); + } else { + window.clearTimeout(delegation.expiryTimer); + delegation.expiryTimer = null; + } + } + }, + [removePendingDelegation], + ); + const scheduleDelegationFlush = useCallback(() => { if (delegationTimerRef.current !== null) { window.clearTimeout(delegationTimerRef.current); @@ -242,10 +314,9 @@ export function useLiveVoice({ ); }, [flushDelegation]); - // A missed or delayed GPT-Live delegation must not bypass Fast. Once the - // person has been quiet, submit the utterance without a delegation id. The - // call then stays in fallback-only mode because later delegation events - // cannot be correlated safely with a specific utterance. + // Speech GPT-Live handles itself (small talk) never produces a delegation. + // Once the person has been quiet for a moment, record what they said so the + // Session transcript stays the complete record of the call. const scheduleSilenceFlush = useCallback(() => { clearSilenceTimer(); silenceTimerRef.current = window.setTimeout(() => { @@ -253,11 +324,9 @@ export function useLiveVoice({ const utterance = stripVoiceAnnotations(inputTranscriptRef.current); if (pendingDelegationsRef.current.length > 0) return; inputTranscriptRef.current = ''; - if (!utterance) return; - fallbackOnlyRef.current = true; - deliverUtterance(utterance, null); + if (utterance) onHeardTurnRef.current?.(utterance); }, UTTERANCE_SILENCE_FLUSH_MS); - }, [clearSilenceTimer, deliverUtterance]); + }, [clearSilenceTimer]); const handleServerEvent = useCallback( (raw: string) => { @@ -273,7 +342,9 @@ export function useLiveVoice({ if (event.delta) { // The person is talking again: whatever GPT-Live said is done. if (outputTranscriptRef.current) flushSpokenTurn(); + const speechStarting = !inputTranscriptRef.current.trim(); inputTranscriptRef.current += event.delta; + if (speechStarting) settleEarlyDelegations(event.start_ms); onHeardTurnDeltaRef.current?.( stripVoiceAnnotations(inputTranscriptRef.current), ); @@ -309,8 +380,22 @@ export function useLiveVoice({ break; case 'session.delegation.created': if (event.delegation?.target === 'client' && event.delegation.id) { - if (fallbackOnlyRef.current) break; - pendingDelegationsRef.current.push(event.delegation.id); + const delegation: PendingDelegation = { + id: event.delegation.id, + offsetMs: event.offset_ms, + expiryTimer: null, + }; + if (!inputTranscriptRef.current.trim()) { + // Nothing transcribed yet: either a late delegation for speech + // already flushed as small talk, or one for a request whose + // transcript is still arriving. Speech starting decides by + // timestamp; silence for the whole window discards it. + delegation.expiryTimer = window.setTimeout(() => { + delegation.expiryTimer = null; + removePendingDelegation(delegation); + }, UNSPOKEN_DELEGATION_EXPIRY_MS); + } + pendingDelegationsRef.current.push(delegation); scheduleDelegationFlush(); } break; @@ -322,7 +407,13 @@ export function useLiveVoice({ break; } }, - [flushSpokenTurn, scheduleDelegationFlush, scheduleSilenceFlush], + [ + flushSpokenTurn, + removePendingDelegation, + scheduleDelegationFlush, + scheduleSilenceFlush, + settleEarlyDelegations, + ], ); const release = useCallback( @@ -379,8 +470,12 @@ export function useLiveVoice({ release(peer, channel, mic, audio); inputTranscriptRef.current = ''; + for (const delegation of pendingDelegationsRef.current) { + if (delegation.expiryTimer !== null) { + window.clearTimeout(delegation.expiryTimer); + } + } pendingDelegationsRef.current = []; - fallbackOnlyRef.current = false; setActive(false); setStatus('idle'); setStartedAt(null); diff --git a/apps/web/src/lib/server/voice.test.ts b/apps/web/src/lib/server/voice.test.ts index 4015e51ac5..8dc73b3713 100644 --- a/apps/web/src/lib/server/voice.test.ts +++ b/apps/web/src/lib/server/voice.test.ts @@ -108,30 +108,14 @@ describe('createVoiceLiveSession', () => { expect(body.session.instructions).toContain( 'Integrations the backend can use: GitHub.', ); - // The voice acknowledges, delegates every utterance, and reports results - // faithfully without originating answers or claims of inspection. + // The voice acknowledges, delegates real work, and reports results + // faithfully in its own words. expect(body.session.instructions).toContain('Backchannel policy'); + expect(body.session.instructions).toContain('Delegate to the backend when'); expect(body.session.instructions).toContain( - 'Delegate every complete utterance to the backend', - ); - expect(body.session.instructions).toContain( - 'Never answer, explain, clarify, offer an opinion, or state a fact yourself', - ); - expect(body.session.instructions).toContain( - 'Never claim that you checked a source', - ); - expect(body.session.instructions).toContain( - 'any product, repository, or connected tool discussed in the Session', - ); - expect(body.session.instructions).toContain( - 'Roomote when the platform itself is the topic', - ); - expect(body.session.instructions).not.toContain( - 'describe how Roomote works', - ); - expect(body.session.instructions).not.toContain( 'Do not delegate to the backend when', ); + expect(body.session.instructions).toContain('Grounding policy'); expect(body.session.instructions).toContain( 'keep every number, name, path, and link label exactly as given', ); diff --git a/apps/web/src/lib/server/voice.ts b/apps/web/src/lib/server/voice.ts index b17b62d818..3d42b8f1a9 100644 --- a/apps/web/src/lib/server/voice.ts +++ b/apps/web/src/lib/server/voice.ts @@ -177,7 +177,7 @@ The person will mostly talk about their code repositories, pull requests, issues ${formatVoiceWorkspaceContext(context)} -Backchannel policy: Acknowledge each utterance in a few words right away ("Sure.", "I'll check.") and then wait for the backend. Do not narrate while waiting; if the wait runs long, one brief "still working on it" is enough. +Backchannel policy: Acknowledge each request in a few words right away ("Sure.", "I'll check.") and then wait for the backend. Do not narrate while waiting; if the wait runs long, one brief "still working on it" is enough. Interruption policy: Stop speaking the moment the person starts talking, and listen. @@ -185,9 +185,16 @@ Delegation policy: Backend tools: - The backend is the Roomote Fast session: it reads and changes the repositories above, launches coding tasks in those environments, calls the listed integrations, reasons carefully, and returns results for you to report. -- Delegate every complete utterance to the backend, including greetings, thanks, reactions, small talk, corrections, follow-ups, and requests that need clarification. -- Your only self-generated speech is the brief acknowledgement above or one brief wait update. Never answer, explain, clarify, offer an opinion, or state a fact yourself. -- You cannot inspect code, documentation, tools, or deployment state yourself. Never claim that you checked a source or state how any product, repository, or connected tool discussed in the Session works unless backend commentary supplied that result. This includes Roomote when the platform itself is the topic. +Delegate to the backend when: +- The person asks about or for anything involving code, repositories, pull requests, issues, tasks, tools, data, or facts about their work. Anything you would have to guess at, delegate. +- The person corrects, refines, or follows up on earlier work. + +Do not delegate to the backend when: +- The person is only greeting you, thanking you, reacting ("cool", "nice"), or making small talk. Answer briefly yourself, and never start a Fast turn for it. +- You need a one-line clarification to understand what they mean before the backend could act. + +Grounding policy: +- You cannot inspect code, documentation, tools, or deployment state yourself. Never claim that you checked a source, and never state how any product, repository, or connected tool discussed in the Session works unless backend commentary supplied that result. This includes Roomote when the platform itself is the topic. When in doubt, delegate. Reporting policy: - Commentary is the backend's result. Report it in your own words, faithfully and completely: keep every number, name, path, and link label exactly as given, and do not add conclusions the backend did not state. Never claim work finished or a result exists before commentary says so. diff --git a/apps/web/src/trpc/commands/voice/index.test.ts b/apps/web/src/trpc/commands/voice/index.test.ts index 39c02a355a..34f30167f8 100644 --- a/apps/web/src/trpc/commands/voice/index.test.ts +++ b/apps/web/src/trpc/commands/voice/index.test.ts @@ -206,7 +206,7 @@ describe('cleanVoiceTranscriptCommand', () => { }); describe('recordVoiceTurnCommand', () => { - it('writes direct voice output as spoken but unverified in Fast history', async () => { + it('writes what the voice said as a spoken assistant turn and adds it to Fast history', async () => { mockFindAccessibleFastSession.mockResolvedValue({ id: 'fast-1' }); mockUpsertFastAgentMessage.mockResolvedValue({}); mockAppendFastAgentVisibleMessages.mockResolvedValue(undefined); diff --git a/apps/web/src/trpc/commands/voice/index.ts b/apps/web/src/trpc/commands/voice/index.ts index 1c8d311ae1..e097ea7a67 100644 --- a/apps/web/src/trpc/commands/voice/index.ts +++ b/apps/web/src/trpc/commands/voice/index.ts @@ -137,10 +137,10 @@ export async function cleanVoiceTranscriptCommand( const VOICE_MESSAGE_SOURCE = 'voice'; /** - * Record one spoken turn of a voice call in the Session transcript: legacy - * direct user speech (`user`), or what the voice said directly (`assistant`). - * Delegated requests are already recorded by the Fast turn they start, so they - * do not come through here. + * Record one spoken turn of a voice call in the Session transcript: what the + * person said when the voice answered them directly (`user`), or what the + * voice said (`assistant`). Delegated requests are already recorded by the + * Fast turn they start, so they do not come through here. * * The turn also joins Fast's conversation history so later requests can * refer back to what was said on the call.