Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 23 additions & 10 deletions app/(main)/chat/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,13 @@
import {
configToBlob,
createLLMCall,
extractAssistantAudio,
extractAssistantText,
pollLLMCall,
} from "@/app/lib/chatClient";
import { useChatStore } from "@/app/lib/store/chat";
import {
ChatAudioPayload,
ChatMessage,
LLMCallRequest,
LLMInput,
Expand Down Expand Up @@ -97,6 +99,7 @@
signal: AbortSignal;
}): Promise<{
text: string;
audio: ChatAudioPayload | null;
jobId: string;
conversationId: string | null;
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}> {
Expand All @@ -111,10 +114,11 @@
}
const jobId = created.data.job_id;
const result = await pollLLMCall(jobId, args.apiKey, { signal: args.signal });
const text = extractAssistantText(result.llm_response?.response);
const newConversationId =
result.llm_response?.response?.conversation_id ?? args.conversationId;
return { text, jobId, conversationId: newConversationId };
const response = result.llm_response?.response;
const audio = extractAssistantAudio(response);
const text = audio ? "" : extractAssistantText(response);
const newConversationId = response?.conversation_id ?? args.conversationId;
return { text, audio, jobId, conversationId: newConversationId };
}

function checkTextConfig(
Expand All @@ -140,7 +144,7 @@
return null;
}

export default function ChatPage() {

Check warning on line 147 in app/(main)/chat/page.tsx

View workflow job for this annotation

GitHub Actions / lint-and-build

Function 'ChatPage' has a complexity of 22. Maximum allowed is 10

Check warning on line 147 in app/(main)/chat/page.tsx

View workflow job for this annotation

GitHub Actions / lint-and-build

Function 'ChatPage' has too many statements (35). Maximum allowed is 20
const { sidebarCollapsed } = useApp();
const { isAuthenticated, activeKey, isHydrated } = useAuth();
const apiKey = activeKey?.key ?? "";
Expand Down Expand Up @@ -196,7 +200,7 @@
);

const sendMessage = useCallback(
async (input: SendInput): Promise<string | null> => {

Check warning on line 203 in app/(main)/chat/page.tsx

View workflow job for this annotation

GitHub Actions / lint-and-build

Async arrow function has a complexity of 26. Maximum allowed is 10

Check warning on line 203 in app/(main)/chat/page.tsx

View workflow job for this annotation

GitHub Actions / lint-and-build

Async arrow function has too many statements (47). Maximum allowed is 20
if (input.kind === "text" && !input.text.trim()) return null;

if (!isAuthenticated) {
Expand Down Expand Up @@ -267,6 +271,7 @@

const {
text,
audio,
jobId,
conversationId: newConversationId,
} = await executeChatCall({
Expand All @@ -280,12 +285,20 @@
if (newConversationId && newConversationId !== conversationId) {
setConversationId(newConversationId);
}
updateMessageInStore(assistantMessage.id, {
content:
text ||
"(The assistant returned an empty response — try again or pick a different configuration.)",
status: "complete",
});
if (audio) {
updateMessageInStore(assistantMessage.id, {
content: "",
audio,
status: "complete",
});
} else {
updateMessageInStore(assistantMessage.id, {
content:
text ||
"(The assistant returned an empty response — try again or pick a different configuration.)",
status: "complete",
});
}
finishAbort();
return text || null;
} catch (err) {
Expand Down Expand Up @@ -344,7 +357,7 @@
const textConfigReady =
!activeConfig || (activeConfig.type?.toLowerCase() ?? "text") !== "stt";

const handleStartVoice = useCallback(async () => {

Check warning on line 360 in app/(main)/chat/page.tsx

View workflow job for this annotation

GitHub Actions / lint-and-build

Async arrow function has too many statements (21). Maximum allowed is 20
if (!isAuthenticated) {
setShowLoginModal(true);
return;
Expand Down
4 changes: 4 additions & 0 deletions app/(main)/configurations/prompt-editor/page.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import { configState } from "@/app/lib/store/configStore";
import { DEFAULT_CONFIG } from "@/app/lib/constants";

function PromptEditorContent() {

Check warning on line 27 in app/(main)/configurations/prompt-editor/page.tsx

View workflow job for this annotation

GitHub Actions / lint-and-build

Function 'PromptEditorContent' has a complexity of 12. Maximum allowed is 10

Check warning on line 27 in app/(main)/configurations/prompt-editor/page.tsx

View workflow job for this annotation

GitHub Actions / lint-and-build

Function 'PromptEditorContent' has too many statements (41). Maximum allowed is 20
const searchParams = useSearchParams();
const { sidebarCollapsed } = useApp();
const { activeKey } = useAuth();
Expand Down Expand Up @@ -238,6 +238,9 @@
]);

const handleSave = async () => {
const wasNewConfig = !allConfigMeta.find(
(m) => m.name === currentConfigName.trim(),
);
const ok = await saveConfig({
currentConfigName,
currentConfigBlob,
Expand All @@ -248,6 +251,7 @@
if (ok) {
setHasUnsavedChanges(false);
setCommitMessage("");
if (wasNewConfig) resetEditor();
}
};

Expand Down
50 changes: 48 additions & 2 deletions app/components/chat/ChatMessage.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,46 @@ function AssistantBody({
);
}

function AssistantAudioPlayer({
url,
mimeType,
}: {
url: string;
mimeType: string;
}) {
const [error, setError] = useState(false);

if (error) {
return (
<div className="inline-flex items-center gap-2.5 rounded-2xl bg-status-error-bg text-status-error-text border border-status-error-border px-3 py-2 text-[13px]">
<SpeakerIcon className="w-4 h-4 shrink-0" />
<span>
Audio is no longer available (the signed URL may have expired).
</span>
</div>
);
}

return (
<div className="inline-flex flex-col gap-2 rounded-2xl bg-bg-secondary px-3 py-2 max-w-full">
<div className="flex items-center gap-2 text-[12px] font-medium text-text-secondary">
<SpeakerIcon className="w-3.5 h-3.5 text-accent-primary" />
<span>Voice reply</span>
</div>
<audio
src={url}
controls
preload="metadata"
onError={() => setError(true)}
className="max-w-full"
>
<source src={url} type={mimeType} />
Your browser doesn&apos;t support inline audio playback.
</audio>
</div>
);
}

function AssistantMessage({ message }: { message: ChatMessageType }) {
const isPending = message.status === "pending";
const isError = message.status === "error";
Expand All @@ -141,16 +181,22 @@ function AssistantMessage({ message }: { message: ChatMessageType }) {
}
};

const showToolbar = !isPending && !isError && !!message.content;
const hasAudio = !!message.audio?.url;
const showToolbar = !isPending && !isError && !hasAudio && !!message.content;

return (
<div className="flex gap-3 px-4 sm:px-6">
<div className="w-8 h-8 rounded-full bg-accent-primary text-white flex items-center justify-center shrink-0 mt-1">
<ChatIcon className="w-4 h-4" />
</div>
<div className="flex-1 min-w-0 pt-1">
{isPending && !message.content ? (
{isPending && !message.content && !hasAudio ? (
<ThinkingDots />
) : hasAudio && message.audio ? (
<AssistantAudioPlayer
url={message.audio.url}
mimeType={message.audio.mimeType}
/>
) : (
<AssistantBody
text={message.content}
Expand Down
8 changes: 6 additions & 2 deletions app/components/chat/VoiceInput.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"use client";

import { CloseIcon, SendIcon } from "@/app/components/icons";
import type { VoiceStatus } from "@/app/hooks/useVoiceChat";
import { VoiceStatus } from "@/app/lib/types/voiceChat";

interface VoiceInputProps {
status: VoiceStatus;
Expand All @@ -19,6 +19,8 @@ const BAR_COUNT = 24;

function StatusLabel({ status }: { status: VoiceStatus }) {
switch (status) {
case "requesting":
return <>Waiting for microphone permission…</>;
case "listening":
return <>Listening…</>;
case "sending":
Expand Down Expand Up @@ -113,7 +115,9 @@ export default function VoiceInput({
? "Tap ↑ to send · ✕ to cancel"
: status === "sending"
? "Sending your message…"
: "Preparing microphone…"}
: status === "requesting"
? "Allow microphone access in the browser prompt to start"
: "Preparing microphone…"}
</p>
</div>
</div>
Expand Down
42 changes: 29 additions & 13 deletions app/components/prompt-editor/ConfigEditorPane.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@ import {
ConfigVersionItems,
CompletionConfig,
} from "@/app/lib/types/configs";
import { MODEL_OPTIONS, isGpt5Model } from "@/app/lib/models";
import {
ConfigType,
MODEL_OPTIONS,
getModelsForType,
isGpt5Model,
} from "@/app/lib/models";
import { PROVIDER_TYPES, PROVIDES_OPTIONS } from "@/app/lib/constants";
import GuardrailsSection from "./GuardrailsSection";
import SaveConfigModal from "./SaveConfigModal";
Expand Down Expand Up @@ -104,25 +109,39 @@ export default function ConfigEditorPane({
? allConfigMeta.find((m) => m.name === configName.trim())
: undefined;

const currentType = (configBlob.completion.type || "text") as ConfigType;

const handleProviderChange = (newProvider: string) => {
const candidates = getModelsForType(newProvider, currentType);
const fallback = MODEL_OPTIONS[newProvider]?.[0]?.value ?? "";
const nextModel = candidates[0]?.value ?? fallback;
Comment thread
Ayush8923 marked this conversation as resolved.
onConfigChange({
...configBlob,
completion: {
...configBlob.completion,
provider: newProvider as CompletionConfig["provider"],
params: {
...params,
model:
MODEL_OPTIONS[newProvider as keyof typeof MODEL_OPTIONS][0].value,
},
params: { ...params, model: nextModel },
},
});
};

const handleTypeChange = (newType: "text" | "stt" | "tts") => {
const handleTypeChange = (newType: ConfigType) => {
const provider = configBlob.completion.provider;
const candidates = getModelsForType(provider, newType);
const currentModel = params.model;
const stillValid = candidates.some((m) => m.value === currentModel);
const nextModel = stillValid
? currentModel
: (candidates[0]?.value ??
MODEL_OPTIONS[provider]?.[0]?.value ??
currentModel);
onConfigChange({
...configBlob,
completion: { ...configBlob.completion, type: newType },
completion: {
...configBlob.completion,
type: newType,
params: { ...params, model: nextModel },
},
});
};

Expand Down Expand Up @@ -247,7 +266,7 @@ export default function ConfigEditorPane({
<select
value={configBlob.completion.type || "text"}
onChange={(e) =>
handleTypeChange(e.target.value as "text" | "stt")
handleTypeChange(e.target.value as "text" | "stt" | "tts")
}
className={inputClass}
>
Expand All @@ -273,10 +292,7 @@ export default function ConfigEditorPane({
onChange={(e) => handleModelChange(e.target.value)}
className={inputClass}
>
{(
MODEL_OPTIONS[provider as keyof typeof MODEL_OPTIONS] ??
MODEL_OPTIONS.openai
).map((model) => (
{getModelsForType(provider, currentType).map((model) => (
<option key={model.value} value={model.value}>
{model.label}
</option>
Expand Down
51 changes: 8 additions & 43 deletions app/hooks/useVoiceChat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,49 +6,13 @@ import {
audioBufferToMono16k,
encodeWav,
} from "@/app/lib/audio/codec";

export type VoiceStatus = "idle" | "listening" | "sending" | "error";

interface UseVoiceChatArgs {
onSubmitAudio: (audio: {
base64: string;
mimeType: string;
transcript: string;
}) => Promise<string | null | undefined>;
}

interface UseVoiceChatResult {
status: VoiceStatus;
error: string | null;
audioLevel: number; // 0..1 — for the live waveform
transcript: string; // live transcript captured during recording
start: () => Promise<void>;
submit: () => Promise<void>;
cancel: () => void;
}

interface SpeechRecognitionEventLike {
resultIndex: number;
results: ArrayLike<{
0: { transcript: string };
isFinal: boolean;
length: number;
}>;
}
interface SpeechRecognitionLike {
continuous: boolean;
interimResults: boolean;
lang: string;
start(): void;
stop(): void;
abort(): void;
onresult: ((e: SpeechRecognitionEventLike) => void) | null;
onerror: ((e: unknown) => void) | null;
onend: (() => void) | null;
}
interface SpeechRecognitionCtor {
new (): SpeechRecognitionLike;
}
import {
VoiceStatus,
UseVoiceChatArgs,
UseVoiceChatResult,
SpeechRecognitionLike,
SpeechRecognitionCtor,
} from "@/app/lib/types/voiceChat";

/**
* AudioWorkletProcessor source — captures the first input channel each
Expand Down Expand Up @@ -290,6 +254,7 @@ export function useVoiceChat({
// wired up later) is the authoritative transcript path.
transcriptRef.current = "";
setTranscript("");
setStatus("requesting");
startRecognition();

let stream: MediaStream;
Expand Down
33 changes: 33 additions & 0 deletions app/lib/chatClient.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
Tool,
} from "@/app/lib/types/configs";
import {
ChatAudioPayload,
LLMCallCreateResponse,
LLMCallRequest,
LLMCallStatusData,
Expand Down Expand Up @@ -168,6 +169,38 @@ export function extractAssistantText(
}
}

interface AudioOutputContent {
format?: string;
value?: string;
mime_type?: string;
uri?: string;
}

interface AudioOutputNode {
type?: string;
content?: AudioOutputContent;
}

export function extractAssistantAudio(
response?: LLMResponseBody | null,
): ChatAudioPayload | null {
const output = response?.output as AudioOutputNode | null | undefined;
if (!output || typeof output !== "object") return null;
if (output.type !== "audio" || !output.content) return null;
const { uri, value, format, mime_type } = output.content;
const mimeType = mime_type || "audio/wav";
if (format === "url" && typeof value === "string" && value) {
return { url: value, mimeType };
}
if (typeof uri === "string" && uri) {
return { url: uri, mimeType };
}
if (format === "base64" && typeof value === "string" && value) {
return { url: `data:${mimeType};base64,${value}`, mimeType };
}
return null;
}

/**
* The UI keeps `tools` as an array; the backend wants the equivalent fields
* flattened onto `params` (`knowledge_base_ids`, `max_num_results`). Mirrors
Expand Down
2 changes: 1 addition & 1 deletion app/lib/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ export const PROVIDER_TYPES = [
{
value: "tts",
label: "Text-to-Speech",
description: "Text input into transcribe audio",
description: "Synthesise audio from a text prompt",
},
];

Expand Down
Loading
Loading