From 8637262d4a38bdce377f95afbb6b38b46703f9de Mon Sep 17 00:00:00 2001 From: MarkXian Date: Sat, 29 Aug 2026 18:48:07 +0800 Subject: [PATCH] feat(stt): make OpenAI transcription model configurable Signed-off-by: MarkXian --- README.md | 2 ++ README.zh-CN.md | 2 ++ config.toml.example | 1 + docs/configuration.md | 2 ++ docs/configuration.zh-CN.md | 2 ++ docs/providers.md | 2 +- docs/providers.zh-CN.md | 2 +- internal/config/config.go | 9 +++++-- internal/config/config_test.go | 27 +++++++++++++++++++ internal/stt/openai.go | 35 +++++++++++++----------- internal/stt/openai_test.go | 49 ++++++++++++++++++++++++++++++++++ internal/stt/stt.go | 2 +- 12 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 internal/stt/openai_test.go diff --git a/README.md b/README.md index b4e00bc..2e5962b 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,8 @@ Details and code: [Bring your own agent](./docs/bring-your-own-agent.md) · [Age Providers: Deepgram, AssemblyAI, OpenAI, Cartesia, ElevenLabs, MiniMax, Speechify, Ollama, VibeVoice (local), xAI Grok Voice (speech-to-speech), pgvector/Supabase for retrieval. See [Providers](./docs/providers.md). +OpenAI STT supports `whisper-1`, `gpt-4o-transcribe`, and `gpt-4o-mini-transcribe` through the independent `openai.stt_model` setting. + ## Documentation | Page | What's in it | diff --git a/README.zh-CN.md b/README.zh-CN.md index 3006a9b..f7a5600 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -115,6 +115,8 @@ StreamCore 位于「提示词 + 工具」类框架的下一层：媒体链路。 服务商：Deepgram、AssemblyAI、OpenAI、Cartesia、ElevenLabs、MiniMax、Speechify、Ollama、VibeVoice（本地）、xAI Grok Voice（语音到语音），检索支持 pgvector / Supabase。见[服务商](./docs/providers.zh-CN.md)。 +OpenAI STT 可通过独立的 `openai.stt_model` 配置选择 `whisper-1`、`gpt-4o-transcribe` 或 `gpt-4o-mini-transcribe`。 + ## 文档 | 页面 | 内容 | diff --git a/config.toml.example b/config.toml.example index a775c85..5d80012 100644 --- a/config.toml.example +++ b/config.toml.example @@ -98,6 +98,7 @@ keyterms = [] # Bias the decoder toward domain words [openai] api_key = "" # Required for LLM, and for OpenAI STT if selected model = "gpt-4o-mini" +stt_model = "whisper-1" # STT model: whisper-1, gpt-4o-transcribe, or gpt-4o-mini-transcribe system_prompt = "You are a helpful AI voice assistant. Keep your responses concise and conversational." base_url = "" # Optional. Any endpoint speaking OpenAI's protocol — DeepSeek # (https://api.deepseek.com/v1, model deepseek-chat), Moonshot diff --git a/docs/configuration.md b/docs/configuration.md index b77964d..1abb650 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -90,6 +90,7 @@ utterance_end_ms = "1000" # Silence (ms) before UtteranceEnd; flushes [openai] api_key = "" model = "gpt-4o-mini" +stt_model = "whisper-1" # whisper-1 | gpt-4o-transcribe | gpt-4o-mini-transcribe system_prompt = "You are a helpful AI voice assistant. Keep your responses concise and conversational." [ollama] @@ -162,6 +163,7 @@ Notes: - `pipeline.readback_bargein_guard_enabled` keeps weak corrections and backchannels from cutting off a confirmation readback. Only explicit commands (stop, cancel, hang up) interrupt. Off by default. - `deepgram.endpointing` and `deepgram.utterance_end_ms` tune when a turn is considered finished upstream; the turn-merge debounce runs on top of them. - `deepgram.tts_model` picks the Aura voice; STT (`model`) and TTS (`tts_model`) share the one API key. Voices are named `[family]-[voice]-[language]` — see [Deepgram's voice list](https://developers.deepgram.com/docs/tts-models). +- `openai.stt_model` selects the batch transcription model independently of the chat `model`; it defaults to `whisper-1`. - `cartesia.max_concurrency` should match your plan's TTS concurrency limit — Cartesia counts active generations, not calls, and returns 429 past the limit. - `minimax.base_url` selects the region. Leave it unset for the global endpoint; mainland-China accounts must point it at `https://api.minimaxi.com/v1`, since keys do not work across the two platforms. - `minimax.model` must match your plan: a Token Plan key (`sk-cp-`) only covers `speech-2.8-hd`, while any other model bills pay-as-you-go and errors with `2056` on a zero balance. diff --git a/docs/configuration.zh-CN.md b/docs/configuration.zh-CN.md index be23237..247743e 100644 --- a/docs/configuration.zh-CN.md +++ b/docs/configuration.zh-CN.md @@ -84,6 +84,7 @@ utterance_end_ms = "1000" # Silence (ms) before UtteranceEnd; flushes [openai] api_key = "" model = "gpt-4o-mini" +stt_model = "whisper-1" # whisper-1 | gpt-4o-transcribe | gpt-4o-mini-transcribe system_prompt = "You are a helpful AI voice assistant. Keep your responses concise and conversational." [ollama] @@ -154,6 +155,7 @@ voice = "en-Emma_woman" - `pipeline.readback_bargein_guard_enabled` 可避免弱纠正与回应词打断智能体的确认复述。只有明确的命令（stop、cancel、hang up）才会打断。默认关闭。 - `deepgram.endpointing` 与 `deepgram.utterance_end_ms` 调节上游认定一轮结束的时机；轮次合并去抖运行在它们之上。 - `deepgram.tts_model` 选择 Aura 音色；STT（`model`）与 TTS（`tts_model`）共用同一个 API key。音色命名规则为 `[family]-[voice]-[language]` —— 见 [Deepgram 音色列表](https://developers.deepgram.com/docs/tts-models)。 +- `openai.stt_model` 独立于对话 `model` 选择批量转写模型，默认值为 `whisper-1`。 - `cartesia.max_concurrency` 应与你套餐的 TTS 并发上限一致 —— Cartesia 统计的是进行中的生成数而不是通话数，超限会返回 429。 - `minimax.base_url` 用于选择区域。留空即使用全球端点；中国大陆账号必须指向 `https://api.minimaxi.com/v1`，因为两个平台的 key 不通用。 - `minimax.model` 必须与你的套餐匹配：Token Plan 的 key（`sk-cp-`）只覆盖 `speech-2.8-hd`，其他模型走按量计费，余额为零时报错 `2056`。 diff --git a/docs/providers.md b/docs/providers.md index 8ebd530..06e0402 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -12,7 +12,7 @@ Notes: -- `stt.provider = "openai"` uses Whisper-style final transcription instead of streaming partials. +- `stt.provider = "openai"` uses batch final transcription instead of streaming partials; choose `whisper-1`, `gpt-4o-transcribe`, or `gpt-4o-mini-transcribe` with `openai.stt_model`. - `llm.provider = "ollama"` targets any Ollama-compatible endpoint via `base_url` — local or on your own infrastructure. - `llm.provider = "agent"` POSTs each turn to an HTTP endpoint you host; your agent owns memory, prompting, and tools, and replies stream back as SSE, chunked text, or JSON. See [Bring your own agent](./bring-your-own-agent.md). - `stt.provider = "vibevoice"` and `tts.provider = "vibevoice"` use local models; start the Python sidecars first. diff --git a/docs/providers.zh-CN.md b/docs/providers.zh-CN.md index 5759843..c53a66f 100644 --- a/docs/providers.zh-CN.md +++ b/docs/providers.zh-CN.md @@ -12,7 +12,7 @@ 注意： -- `stt.provider = "openai"` 使用 Whisper 式的最终转写，而不是流式中间结果。 +- `stt.provider = "openai"` 使用批量最终转写而不是流式中间结果；可通过 `openai.stt_model` 选择 `whisper-1`、`gpt-4o-transcribe` 或 `gpt-4o-mini-transcribe`。 - `llm.provider = "ollama"` 通过 `base_url` 指向任何兼容 Ollama 的端点 —— 本地或你自己的基础设施均可。 - `llm.provider = "agent"` 把每一轮对话 POST 到你托管的 HTTP 端点；记忆、提示词与工具都由你的智能体掌控，回复以 SSE、分块文本或 JSON 流式返回。见[接入你自己的智能体](./bring-your-own-agent.zh-CN.md)。 - `stt.provider = "vibevoice"` 与 `tts.provider = "vibevoice"` 使用本地模型；请先启动 Python 边车进程。 diff --git a/internal/config/config.go b/internal/config/config.go index 39c4a59..2c86957 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -232,8 +232,12 @@ type AssemblyAIConfig struct { } type OpenAIConfig struct { - APIKey string `toml:"api_key"` - Model string `toml:"model"` + APIKey string `toml:"api_key"` + Model string `toml:"model"` + // STTModel selects the transcription model used when stt.provider = + // "openai". It is separate from Model because chat and transcription use + // different model families. Empty defaults to whisper-1. + STTModel string `toml:"stt_model"` SystemPrompt string `toml:"system_prompt"` // BaseURL points the client at an OpenAI-compatible endpoint other than // OpenAI's own — DeepSeek, Moonshot, Qwen and MiniMax all speak the same @@ -433,6 +437,7 @@ func Load(path string) (*Config, error) { setDefault(&cfg.LLM.Provider, "openai") setDefault(&cfg.TTS.Provider, "cartesia") setDefault(&cfg.OpenAI.Model, "gpt-4o-mini") + setDefault(&cfg.OpenAI.STTModel, "whisper-1") setDefault(&cfg.OpenAI.SystemPrompt, "You are a helpful AI voice assistant having a natural phone conversation. Keep responses to 1-2 sentences unless asked for detail. When interrupted (indicated by bracketed context), respond the way a patient human would: if they say 'no', address the disagreement; if they redirect, follow their lead. Never repeat what you already said, never ask 'would you like me to continue', and never mention that you were interrupted.") setDefault(&cfg.Ollama.BaseURL, "http://localhost:11434") setDefault(&cfg.Ollama.Model, "llama3.2") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 4baa7de..2edd3b8 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -217,6 +217,33 @@ api_key = "file-key" } } +func TestLoadOpenAISTTModel(t *testing.T) { + clearEnvOverrides(t) + + t.Run("defaults to whisper-1", func(t *testing.T) { + cfg, err := Load(writeConfig(t, "")) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.OpenAI.STTModel != "whisper-1" { + t.Errorf("OpenAI.STTModel = %q, want %q", cfg.OpenAI.STTModel, "whisper-1") + } + }) + + t.Run("reads an explicit transcription model", func(t *testing.T) { + cfg, err := Load(writeConfig(t, ` +[openai] +stt_model = "gpt-4o-transcribe" +`)) + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.OpenAI.STTModel != "gpt-4o-transcribe" { + t.Errorf("OpenAI.STTModel = %q, want %q", cfg.OpenAI.STTModel, "gpt-4o-transcribe") + } + }) +} + func TestLoadWarnsAboutUnknownKeys(t *testing.T) { clearEnvOverrides(t) logs := captureLogs(t) diff --git a/internal/stt/openai.go b/internal/stt/openai.go index 2e958fe..c034610 100644 --- a/internal/stt/openai.go +++ b/internal/stt/openai.go @@ -14,11 +14,11 @@ import ( ) const ( - whisperModel = "whisper-1" - sampleRate = 16000 - numChannels = 1 - bitsPerSample = 16 - bytesPerSample = bitsPerSample / 8 + defaultOpenAISTTModel = "whisper-1" + sampleRate = 16000 + numChannels = 1 + bitsPerSample = 16 + bytesPerSample = bitsPerSample / 8 // VAD parameters speechEnergyThreshold = 500.0 // RMS threshold to detect speech @@ -27,12 +27,13 @@ const ( minSpeechDuration = 200 * time.Millisecond // Ignore very short bursts ) -// openaiClient implements STT using the OpenAI Whisper API. -// Since Whisper is a batch API, audio is buffered and sent for transcription +// openaiClient implements STT using the OpenAI transcription API. +// Since the API is batch-oriented, audio is buffered and sent for transcription // when silence is detected after speech (simple energy-based VAD). // Unlike Deepgram, only final transcripts are produced (no partials). type openaiClient struct { client *openai.Client + model string ctx context.Context cancel context.CancelFunc mu sync.Mutex @@ -49,11 +50,15 @@ type openaiClient struct { done chan struct{} } -func NewOpenAIClient(ctx context.Context, apiKey string, onResult func(TranscriptResult)) (*openaiClient, error) { +func NewOpenAIClient(ctx context.Context, apiKey, model string, onResult func(TranscriptResult)) (*openaiClient, error) { sttCtx, cancel := context.WithCancel(ctx) + if model == "" { + model = defaultOpenAISTTModel + } c := &openaiClient{ client: openai.NewClient(apiKey), + model: model, ctx: sttCtx, cancel: cancel, onResult: onResult, @@ -63,11 +68,11 @@ func NewOpenAIClient(ctx context.Context, apiKey string, onResult func(Transcrip c.vadTicker = time.NewTicker(100 * time.Millisecond) go c.vadLoop() - log.Println("[stt] OpenAI Whisper client ready") + log.Printf("[stt] OpenAI transcription client ready (model=%s)", model) return c, nil } -// vadLoop periodically checks if we should flush buffered speech to Whisper. +// vadLoop periodically checks if we should flush buffered speech to OpenAI. func (c *openaiClient) vadLoop() { defer close(c.done) for { @@ -138,7 +143,7 @@ func (c *openaiClient) SendAudio(data []byte) error { if !c.speaking { c.speaking = true c.speechStart = time.Now() - log.Println("[stt:whisper] speech started") + log.Println("[stt:openai] speech started") } c.lastSpeech = time.Now() } @@ -155,13 +160,13 @@ func (c *openaiClient) transcribe(pcmData []byte) { wavData := encodeWAV(pcmData, sampleRate, numChannels, bitsPerSample) resp, err := c.client.CreateTranscription(c.ctx, openai.AudioRequest{ - Model: whisperModel, + Model: c.model, Reader: bytes.NewReader(wavData), FilePath: "audio.wav", }) if err != nil { if c.ctx.Err() == nil { - log.Printf("[stt:whisper] transcription error: %v", err) + log.Printf("[stt:openai] transcription error: %v", err) } return } @@ -171,7 +176,7 @@ func (c *openaiClient) transcribe(pcmData []byte) { return } - log.Printf("[stt:whisper] transcript: %s", text) + log.Printf("[stt:openai] transcript: %s", text) c.onResult(TranscriptResult{ Text: text, IsFinal: true, @@ -182,7 +187,7 @@ func (c *openaiClient) Close() { c.cancel() c.vadTicker.Stop() <-c.done // Wait for vadLoop to finish - log.Println("[stt:whisper] closed") + log.Println("[stt:openai] closed") } // rmsEnergy calculates the root-mean-square energy of linear16 PCM samples. diff --git a/internal/stt/openai_test.go b/internal/stt/openai_test.go new file mode 100644 index 0000000..af77d79 --- /dev/null +++ b/internal/stt/openai_test.go @@ -0,0 +1,49 @@ +package stt + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + openai "github.com/sashabaranov/go-openai" +) + +func TestOpenAITranscriptionUsesConfiguredModel(t *testing.T) { + const wantModel = "gpt-4o-transcribe" + + var gotModel string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := r.ParseMultipartForm(1 << 20); err != nil { + t.Errorf("parse transcription request: %v", err) + http.Error(w, "invalid multipart form", http.StatusBadRequest) + return + } + gotModel = r.FormValue("model") + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, `{"text":"hello"}`) + })) + defer server.Close() + + clientConfig := openai.DefaultConfig("test-key") + clientConfig.BaseURL = server.URL + "/v1" + var gotTranscript string + client := &openaiClient{ + client: openai.NewClientWithConfig(clientConfig), + model: wantModel, + ctx: context.Background(), + onResult: func(result TranscriptResult) { + gotTranscript = result.Text + }, + } + + client.transcribe([]byte{0, 0, 1, 0}) + + if gotModel != wantModel { + t.Errorf("transcription model = %q, want %q", gotModel, wantModel) + } + if gotTranscript != "hello" { + t.Errorf("transcript = %q, want %q", gotTranscript, "hello") + } +} diff --git a/internal/stt/stt.go b/internal/stt/stt.go index bd758c3..7dc41ba 100644 --- a/internal/stt/stt.go +++ b/internal/stt/stt.go @@ -35,7 +35,7 @@ func NewClient(ctx context.Context, cfg *config.Config, onResult func(Transcript if cfg.OpenAI.APIKey == "" { return nil, fmt.Errorf("stt provider %q requires [openai] api_key to be set", cfg.STT.Provider) } - return NewOpenAIClient(ctx, cfg.OpenAI.APIKey, onResult) + return NewOpenAIClient(ctx, cfg.OpenAI.APIKey, cfg.OpenAI.STTModel, onResult) case "assemblyai": if cfg.AssemblyAI.APIKey == "" { return nil, fmt.Errorf("stt provider %q requires [assemblyai] api_key to be set", cfg.STT.Provider)