diff --git a/.env.example b/.env.example index aabb7273..d1f01996 100644 --- a/.env.example +++ b/.env.example @@ -28,3 +28,14 @@ EMBEDDINGS_MODEL=text-embedding-3-small # 未配置 SEARCH_API_KEY 时深度研究开关会提示不可用，普通对话不受影响。 SEARCH_API_KEY= SEARCH_BASE_URL=https://api.tavily.com + +# === Langfuse（可选，遥测 + 评测） === +# 未配置时遥测整体停用（零开销），聊天与点赞/点踩 UI 不受影响（反馈被静默丢弃）。 +# 密钥来自 Langfuse 项目设置（cloud.langfuse.com 免费注册，或 docker compose 自托管）。 +# BASE_URL：EU 区 https://cloud.langfuse.com、US 区 https://us.cloud.langfuse.com、 +# 自托管填实例地址。 +LANGFUSE_PUBLIC_KEY= +LANGFUSE_SECRET_KEY= +LANGFUSE_BASE_URL=https://cloud.langfuse.com +# pnpm eval 用 Langfuse dataset 替代内置样例集时指定 dataset 名（可选） +# LANGFUSE_EVAL_DATASET= diff --git a/README.md b/README.md index 27c6b431..794927fd 100644 --- a/README.md +++ b/README.md @@ -69,3 +69,11 @@ RAG 依赖 Postgres 的 pgvector 扩展（迁移会自动 `CREATE EXTENSION vect - 配置 `SEARCH_API_KEY`（默认 [Tavily](https://tavily.com)，`SEARCH_BASE_URL` 可换兼容服务）即可启用；未配置时开关会提示不可用，普通对话不受影响。 - 编排走 AI SDK v7 多步工具循环（`streamText` + `webSearch`/`readUrl` 工具 + `stopWhen` 步数上限），与现有 chat 链路同源。 - 设计与调研结论见 `docs/deep-research/设计说明.md`。 + +## 遥测与评测（Langfuse） + +AI SDK v7 原生遥测（OpenTelemetry）+ [Langfuse](https://langfuse.com)：每轮对话一条 trace（含每步 LLM 调用、工具执行、token 用量与耗时），线程聚合为 session；assistant 消息下的 👍/👎 作为 score 回写到对应 trace。 + +- 配置 `LANGFUSE_PUBLIC_KEY` / `LANGFUSE_SECRET_KEY` / `LANGFUSE_BASE_URL`（云端免费额度或 docker compose 自托管均可）即可启用；未配置时遥测整体停用、零开销，点赞/点踩被静默丢弃。 +- `pnpm eval` 跑离线评测（规则断言 + LLM-as-a-judge），结果作为 experiment run 上报，可在 Langfuse UI 跨 run 对比；设 `LANGFUSE_EVAL_DATASET` 可改用远端 dataset。 +- 调研对比（AI SDK 遥测现状、Langfuse vs LangSmith）与设计取舍见 `docs/observability/`。 diff --git a/app/api/chat/route.ts b/app/api/chat/route.ts index 47a20ac3..ef32e583 100644 --- a/app/api/chat/route.ts +++ b/app/api/chat/route.ts @@ -5,14 +5,34 @@ import { tool, type UIMessage, } from "ai" +import { after } from "next/server" import { frontendTools } from "@assistant-ui/react-ai-sdk" import type { ToolJSONSchema } from "assistant-stream" +import { + getActiveTraceId, + propagateAttributes, + startActiveObservation, + type LangfuseSpan, +} from "@langfuse/tracing" import { z } from "zod" import { resolveAttachmentParts } from "@/lib/chat/resolve-attachments" import { minimaxChatModel } from "@/lib/ai/minimax" import { researchTools } from "@/lib/chat/research-tools" import { isSearchConfigured } from "@/lib/ai/search" -import { RESEARCH_MAX_STEPS, RESEARCH_SYSTEM_PROMPT } from "@/constants/research" +import { + flushLangfuseSpans, + isLangfuseConfigured, + isValidTraceId, +} from "@/lib/observability/langfuse" +import { + CHAT_TRACE_NAME, + TELEMETRY_FUNCTION_IDS, + TRACE_TAGS, +} from "@/constants/observability" +import { + RESEARCH_MAX_STEPS, + RESEARCH_SYSTEM_PROMPT, +} from "@/constants/research" // 深度研究可能多步循环，耗时较长，放宽单次请求时长上限 export const maxDuration = 120 @@ -24,7 +44,13 @@ const getWeather = tool({ }), execute: async ({ location }) => { // Deterministic mock reading (hashed from the city name) - no real weather API/key involved. - const conditions = ["Sunny", "Partly Cloudy", "Cloudy", "Light Rain", "Clear"] + const conditions = [ + "Sunny", + "Partly Cloudy", + "Cloudy", + "Light Rain", + "Clear", + ] const seed = [...location].reduce((acc, c) => acc + c.charCodeAt(0), 0) return { location, @@ -42,27 +68,47 @@ const compareTable = tool({ inputSchema: z.object({ title: z.string(), unit: z.string().optional(), - columns: z.array(z.string()).describe("Category labels, e.g. country names"), + columns: z + .array(z.string()) + .describe("Category labels, e.g. country names"), series: z.array( z.object({ name: z.string(), - values: z.array(z.number()).describe("One value per column, same order as columns"), - }), + values: z + .array(z.number()) + .describe("One value per column, same order as columns"), + }) ), }), execute: async (input) => input, }) -export async function POST(req: Request) { - const { - messages, - tools, - deepResearch, - }: { - messages: UIMessage[] - tools?: Record - deepResearch?: boolean - } = await req.json() +type ChatRequestBody = { + messages: UIMessage[] + tools?: Record + deepResearch?: boolean + /** useChat 的 chat id == assistant-ui threadListItem.id == threads.id */ + id?: string +} + +/** trace 根观测的 input 记录最后一条用户消息的纯文本（完整 prompt 在 generation 观测里已有） */ +function lastUserText(messages: UIMessage[]): string { + const lastUser = messages.findLast((m) => m.role === "user") + if (!lastUser) return "" + return lastUser.parts + .filter( + (part): part is Extract => + part.type === "text" + ) + .map((part) => part.text) + .join("\n") +} + +async function runChat( + body: ChatRequestBody, + turn?: LangfuseSpan +): Promise { + const { messages, tools, deepResearch } = body // 研究模式：加入联网检索/深读工具、放宽步数、注入研究系统提示 const research = deepResearch === true @@ -84,14 +130,72 @@ export async function POST(req: Request) { : "用户开启了深度研究，但服务端未配置搜索服务（SEARCH_API_KEY），请如实告知该功能暂不可用，并基于已有知识尽力回答。" : undefined + turn?.update({ input: lastUserText(messages) }) + const result = streamText({ model: minimaxChatModel(), system, - messages: await convertToModelMessages(resolvedMessages, { tools: allTools }), + messages: await convertToModelMessages(resolvedMessages, { + tools: allTools, + }), tools: allTools, // 研究模式允许更多工具轮次；普通对话维持原来的小步数 stopWhen: isStepCount(research && searchReady ? RESEARCH_MAX_STEPS : 5), + telemetry: { functionId: TELEMETRY_FUNCTION_IDS.chat }, + // handler 返回后流仍在继续，根观测在流真正结束/出错/中止时才收尾 + ...(turn && { + onEnd: ({ text }) => { + turn.update({ output: text }).end() + }, + onError: ({ error }) => { + turn + .update({ + level: "ERROR", + statusMessage: + error instanceof Error ? error.message : String(error), + }) + .end() + }, + onAbort: () => { + turn.update({ statusMessage: "aborted by client" }).end() + }, + }), + }) + + // serverless 下函数在响应后可能立刻冻结，响应结束后冲刷 span 批次 + if (turn) after(() => flushLangfuseSpans()) + + // 服务端把 traceId 下发为 assistant 消息 id：前端点赞/点踩时直接以消息 id 回写 score， + // 无需另建 message↔trace 映射。未启用遥测（或拿到无效 traceId）时交回 AI SDK 默认生成。 + const traceId = getActiveTraceId() + return result.toUIMessageStreamResponse({ + ...(traceId && + isValidTraceId(traceId) && { generateMessageId: () => traceId }), }) +} + +export async function POST(req: Request) { + const body: ChatRequestBody = await req.json() + + if (!isLangfuseConfigured()) return runChat(body) - return result.toUIMessageStreamResponse() + return startActiveObservation( + CHAT_TRACE_NAME, + (turn) => + propagateAttributes( + { + traceName: CHAT_TRACE_NAME, + // threadId 作为 sessionId，同一线程的多轮对话在 Langfuse 里聚成一个 session + sessionId: body.id, + tags: [ + body.deepResearch === true + ? TRACE_TAGS.deepResearch + : TRACE_TAGS.chat, + ], + }, + () => runChat(body, turn) + ), + // 流式响应在 handler 返回后才结束，span 由 onEnd/onError/onAbort 收尾 + { endOnExit: false } + ) } diff --git a/app/api/feedback/route.ts b/app/api/feedback/route.ts new file mode 100644 index 00000000..e4316c1e --- /dev/null +++ b/app/api/feedback/route.ts @@ -0,0 +1,43 @@ +import { z } from "zod" +import { + getLangfuseClient, + isLangfuseConfigured, + isValidTraceId, +} from "@/lib/observability/langfuse" +import { USER_FEEDBACK_SCORE_NAME } from "@/constants/observability" + +// 用户对 assistant 消息的点赞/点踩，作为 score 回写到 Langfuse 对应 trace。 +// assistant 消息 id 由 chat route 下发，值即该轮对话的 traceId（见 chat/route.ts）。 + +const bodySchema = z.object({ + messageId: z.string(), + type: z.enum(["positive", "negative"]), + comment: z.string().max(500).optional(), +}) + +export async function POST(req: Request) { + // 未启用遥测：静默接受，反馈不落任何地方（前端无需感知配置状态） + if (!isLangfuseConfigured()) return new Response(null, { status: 204 }) + + const parsed = bodySchema.safeParse(await req.json().catch(() => null)) + if (!parsed.success) return new Response("Bad request", { status: 400 }) + const { messageId, type, comment } = parsed.data + + // 只有 traceId 格式的消息 id 才可回写；历史消息或遥测未启用期间生成的消息直接忽略 + if (!isValidTraceId(messageId)) return new Response(null, { status: 204 }) + + const langfuse = getLangfuseClient() + langfuse.score.create({ + // 幂等 id：同一条消息改票时覆盖同一个 score，不产生重复计数 + id: `${USER_FEEDBACK_SCORE_NAME}-${messageId}`, + traceId: messageId, + name: USER_FEEDBACK_SCORE_NAME, + value: type === "positive" ? 1 : 0, + dataType: "BOOLEAN", + ...(comment && { comment }), + }) + // route handler 生命周期短，立即冲刷而不是等批量间隔 + await langfuse.score.flush() + + return new Response(null, { status: 204 }) +} diff --git a/app/page.tsx b/app/page.tsx index d32c3676..c01e7843 100644 --- a/app/page.tsx +++ b/app/page.tsx @@ -1,8 +1,15 @@ "use client" import { useMemo } from "react" -import { AssistantRuntimeProvider, useRemoteThreadListRuntime } from "@assistant-ui/react" -import { AssistantChatTransport, useChatRuntime } from "@assistant-ui/react-ai-sdk" +import { + AssistantRuntimeProvider, + useRemoteThreadListRuntime, + type FeedbackAdapter, +} from "@assistant-ui/react" +import { + AssistantChatTransport, + useChatRuntime, +} from "@assistant-ui/react-ai-sdk" import { Base } from "@/components/examples/base" import { AssistantTools } from "@/components/assistant-ui/tools" import { postgresThreadListAdapter } from "@/lib/chat/thread-list-adapter" @@ -10,6 +17,19 @@ import { usePostgresThreadHistoryAdapter } from "@/lib/chat/use-thread-history-a import { r2AttachmentAdapter } from "@/lib/chat/attachment-adapter" import { useResearchMode } from "@/lib/chat/research-mode" +// 点赞/点踩 → /api/feedback → Langfuse score。assistant 消息 id 即该轮 traceId +//（chat route 下发），服务端未启用遥测时该请求会被静默吞掉。fire-and-forget， +// UI 的已提交态由 assistant-ui 本地维护，不依赖请求结果。 +const feedbackAdapter: FeedbackAdapter = { + submit({ message, type }) { + void fetch("/api/feedback", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ messageId: message.id, type }), + }).catch(() => {}) + }, +} + function useMyChatRuntime() { const history = usePostgresThreadHistoryAdapter() // 把「深度研究」开关状态随每条消息发给 chat route。用 getState() 而非闭包快照， @@ -17,7 +37,13 @@ function useMyChatRuntime() { const transport = useMemo( () => new AssistantChatTransport({ - prepareSendMessagesRequest: ({ id, messages, trigger, messageId, body }) => ({ + prepareSendMessagesRequest: ({ + id, + messages, + trigger, + messageId, + body, + }) => ({ body: { ...body, id, @@ -28,9 +54,16 @@ function useMyChatRuntime() { }, }), }), - [], + [] ) - return useChatRuntime({ transport, adapters: { history, attachments: r2AttachmentAdapter } }) + return useChatRuntime({ + transport, + adapters: { + history, + attachments: r2AttachmentAdapter, + feedback: feedbackAdapter, + }, + }) } export default function Page() { diff --git a/components/examples/base.tsx b/components/examples/base.tsx index bcf8fb42..cedabc83 100644 --- a/components/examples/base.tsx +++ b/components/examples/base.tsx @@ -1,48 +1,48 @@ -"use client"; +"use client" import { ComposerAddAttachment, ComposerAttachments, UserMessageAttachments, -} from "@/components/assistant-ui/attachment"; -import { ComposerPdfInsights } from "@/components/assistant-ui/pdf-insights"; -import { DeepResearchToggle } from "@/components/assistant-ui/deep-research-toggle"; +} from "@/components/assistant-ui/attachment" +import { ComposerPdfInsights } from "@/components/assistant-ui/pdf-insights" +import { DeepResearchToggle } from "@/components/assistant-ui/deep-research-toggle" import { ResearchProgress, RESEARCH_TOOL_NAMES, -} from "@/components/assistant-ui/research-panel"; -import { MarkdownText } from "@/components/assistant-ui/markdown-text"; -import { DotMatrix } from "@/components/assistant-ui/dot-matrix"; -import { MessageTiming } from "@/components/assistant-ui/message-timing"; -import { ToolFallback } from "@/components/assistant-ui/tool-fallback"; +} from "@/components/assistant-ui/research-panel" +import { MarkdownText } from "@/components/assistant-ui/markdown-text" +import { DotMatrix } from "@/components/assistant-ui/dot-matrix" +import { MessageTiming } from "@/components/assistant-ui/message-timing" +import { ToolFallback } from "@/components/assistant-ui/tool-fallback" import { ToolGroupContent, ToolGroupRoot, ToolGroupTrigger, -} from "@/components/assistant-ui/tool-group"; +} from "@/components/assistant-ui/tool-group" import { ThreadList, ThreadListItems, ThreadListNew, ThreadListRoot, -} from "@/components/assistant-ui/thread-list"; -import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button"; +} from "@/components/assistant-ui/thread-list" +import { TooltipIconButton } from "@/components/assistant-ui/tooltip-icon-button" import { Reasoning, ReasoningContent, ReasoningRoot, ReasoningText, ReasoningTrigger, -} from "@/components/assistant-ui/reasoning"; -import { Button } from "@/components/ui/button"; -import { cn } from "@/lib/utils"; -import icon from "@/public/favicon/icon.svg"; +} from "@/components/assistant-ui/reasoning" +import { Button } from "@/components/ui/button" +import { cn } from "@/lib/utils" +import icon from "@/public/favicon/icon.svg" import { ComposerQuotePreview, QuoteBlock, SelectionToolbar, -} from "@/components/assistant-ui/quote"; -import { ComposerTriggerPopover } from "@/components/assistant-ui/composer-trigger-popover"; -import { DirectiveText } from "@/components/assistant-ui/directive-text"; +} from "@/components/assistant-ui/quote" +import { ComposerTriggerPopover } from "@/components/assistant-ui/composer-trigger-popover" +import { DirectiveText } from "@/components/assistant-ui/directive-text" import { ActionBarMorePrimitive, ActionBarPrimitive, @@ -59,7 +59,7 @@ import { useAui, useAuiState, type Unstable_SlashCommand, -} from "@assistant-ui/react"; +} from "@assistant-ui/react" import { ArrowDownIcon, ArrowUpIcon, @@ -86,23 +86,25 @@ import { ShareIcon, SlashIcon, SquareIcon, + ThumbsDownIcon, + ThumbsUpIcon, WrenchIcon, -} from "lucide-react"; +} from "lucide-react" import { LexicalComposerInput, type DirectiveChipProps, -} from "@assistant-ui/react-lexical"; -import Image from "next/image"; -import { useState, type FC, type ReactNode } from "react"; -import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet"; +} from "@assistant-ui/react-lexical" +import Image from "next/image" +import { useState, type FC, type ReactNode } from "react" +import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet" import { Tooltip, TooltipContent, TooltipTrigger, -} from "@/components/ui/tooltip"; -import { ModelSelector } from "@/components/assistant-ui/model-selector"; -import { docsModelOptions } from "@/components/docs/assistant/docs-model-options"; -import { DEFAULT_MODEL_ID } from "@/constants/model"; +} from "@/components/ui/tooltip" +import { ModelSelector } from "@/components/assistant-ui/model-selector" +import { docsModelOptions } from "@/components/docs/assistant/docs-model-options" +import { DEFAULT_MODEL_ID } from "@/constants/model" const Logo: FC = () => { return (
@@ -113,20 +115,20 @@ const Logo: FC = () => { /> assistant-ui
- ); -}; + ) +} const Sidebar: FC<{ collapsed?: boolean }> = ({ collapsed }) => { return ( - ); -}; + ) +} const MobileSidebar: FC = () => { return ( @@ -208,9 +210,9 @@ const MobileSidebar: FC = () => { - ); -}; -const models = docsModelOptions(); + ) +} +const models = docsModelOptions() const ModelPicker: FC = () => { return ( { size="sm" className="h-7 rounded-full" /> - ); -}; + ) +} const ThreadTitle: FC = () => { const title = useAuiState( (s) => - s.threads.threadItems.find((t) => t.id === s.threads.mainThreadId) - ?.title, - ); + s.threads.threadItems.find((t) => t.id === s.threads.mainThreadId)?.title + ) return ( {title ?? "New Chat"} - ); -}; + ) +} const Header: FC<{ - sidebarCollapsed: boolean; - onToggleSidebar: () => void; + sidebarCollapsed: boolean + onToggleSidebar: () => void }> = ({ sidebarCollapsed, onToggleSidebar }) => { return (
@@ -263,18 +264,17 @@ const Header: FC<{
- ); -}; + ) +} // Startup exposes a loading placeholder thread; treat it as a new chat so // the composer mounts centered. Loads after startup keep the docked layout. const isNewChatView = (s: AssistantState) => - s.thread.messages.length === 0 && - (!s.thread.isLoading || s.threads.isLoading); + s.thread.messages.length === 0 && (!s.thread.isLoading || s.threads.isLoading) const Thread: FC = () => { - const isEmpty = useAuiState(isNewChatView); + const isEmpty = useAuiState(isNewChatView) return ( { data-slot="aui_thread-viewport" className={cn( "relative flex flex-1 flex-col overflow-x-auto overflow-y-scroll scroll-smooth px-4 pt-4", - isEmpty && "justify-center", + isEmpty && "justify-center" )} > @@ -300,16 +300,16 @@ const Thread: FC = () => { > {({ message }) => { - if (message.composer.isEditing) return ; - if (message.role === "user") return ; - return ; + if (message.composer.isEditing) return + if (message.role === "user") return + return }} @@ -325,35 +325,35 @@ const Thread: FC = () => { - ); -}; + ) +} const ThreadScrollToBottom: FC = () => { return ( - ); -}; + ) +} const ThreadWelcome: FC = () => { return (
-

+

How can I help you today?

- ); -}; + ) +} type SuggestionGroup = { - label: string; - icon: ReactNode; - options: { label: string; prompt: string }[]; -}; + label: string + icon: ReactNode + options: { label: string; prompt: string }[] +} const SUGGESTION_GROUPS: SuggestionGroup[] = [ { label: "Weather", @@ -417,7 +417,8 @@ const SUGGESTION_GROUPS: SuggestionGroup[] = [ options: [ { label: "React vs Vue vs Svelte", - prompt: "Compare npm weekly downloads of React, Vue, and Svelte in a table", + prompt: + "Compare npm weekly downloads of React, Vue, and Svelte in a table", }, { label: "GDP of US, China, Japan", @@ -448,22 +449,22 @@ const SUGGESTION_GROUPS: SuggestionGroup[] = [ }, ], }, -]; +] const suggestionChipClass = - "aui-thread-welcome-suggestion text-foreground hover:bg-muted border-border/60 h-auto gap-1.5 rounded-full border px-3.5 py-1.5 text-sm font-normal whitespace-nowrap transition-colors [&_svg]:size-4"; + "aui-thread-welcome-suggestion text-foreground hover:bg-muted border-border/60 h-auto gap-1.5 rounded-full border px-3.5 py-1.5 text-sm font-normal whitespace-nowrap transition-colors [&_svg]:size-4" const ThreadSuggestions: FC = () => { - const aui = useAui(); - const [expandedLabel, setExpandedLabel] = useState(null); + const aui = useAui() + const [expandedLabel, setExpandedLabel] = useState(null) const expandedGroup = SUGGESTION_GROUPS.find( - (group) => group.label === expandedLabel, - ); + (group) => group.label === expandedLabel + ) const sendPrompt = (prompt: string) => { - if (aui.thread().getState().isRunning) return; + if (aui.thread().getState().isRunning) return aui.thread().append({ content: [{ type: "text", text: prompt }], runConfig: aui.composer().getState().runConfig, - }); - }; + }) + } return (
@@ -474,11 +475,11 @@ const ThreadSuggestions: FC = () => { variant="ghost" className={cn( suggestionChipClass, - group.label === expandedLabel && "bg-muted", + group.label === expandedLabel && "bg-muted" )} onClick={() => setExpandedLabel( - group.label === expandedLabel ? null : group.label, + group.label === expandedLabel ? null : group.label ) } > @@ -491,7 +492,7 @@ const ThreadSuggestions: FC = () => { {expandedGroup && (
{expandedGroup.options.map((option) => ( @@ -508,8 +509,8 @@ const ThreadSuggestions: FC = () => {
)}
- ); -}; + ) +} const slashCommands: readonly Unstable_SlashCommand[] = [ { id: "summarize", @@ -535,16 +536,16 @@ const slashCommands: readonly Unstable_SlashCommand[] = [ icon: "HelpCircle", execute: () => console.log("[base example] /help invoked"), }, -]; +] const slashIconMap: Record> = { FileText: FileTextIcon, Languages: LanguagesIcon, Globe: GlobeIcon, HelpCircle: HelpCircleIcon, -}; +} function DirectiveChip(props: DirectiveChipProps) { - const { directiveId, directiveType, label } = props; - const showWrench = directiveType !== "command"; + const { directiveId, directiveType, label } = props + const showWrench = directiveType !== "command" return ( {label} - ); + ) } const Composer: FC = () => { - const mention = unstable_useMentionAdapter({ fallbackIcon: WrenchIcon }); + const mention = unstable_useMentionAdapter({ fallbackIcon: WrenchIcon }) const slash = unstable_useSlashCommandAdapter({ commands: slashCommands, iconMap: slashIconMap, fallbackIcon: SlashIcon, - }); + }) return (
@@ -581,7 +582,7 @@ const Composer: FC = () => {
@@ -594,8 +595,8 @@ const Composer: FC = () => { /> - ); -}; + ) +} const ComposerAction: FC = () => { return (
@@ -629,7 +630,7 @@ const ComposerAction: FC = () => { type="button" variant="ghost" size="icon" - className="aui-composer-stop-dictation text-destructive size-7 rounded-full" + className="aui-composer-stop-dictation size-7 rounded-full text-destructive" aria-label="Stop voice input" > @@ -667,29 +668,29 @@ const ComposerAction: FC = () => {
- ); -}; + ) +} const MessageError: FC = () => { return ( - + - ); -}; + ) +} const AssistantWorkingIndicator: FC = () => { - const isEmpty = useAuiState((s) => s.message.content.length === 0); + const isEmpty = useAuiState((s) => s.message.content.length === 0) if (isEmpty) { return ( Connecting - ); + ) } return ( { > {"●"} - ); -}; + ) +} // 工具组渲染：纯研究工具组不再单独展示（由 ResearchProgress 面板统一呈现）， // 其余工具组沿用默认的可折叠分组。 const ToolGroupBlock: FC<{ - part: { indices: readonly number[]; status: { type: string } }; - children: ReactNode; + part: { indices: readonly number[]; status: { type: string } } + children: ReactNode }> = ({ part, children }) => { const allResearch = useAuiState((s) => { const content = s.message.content as unknown as { - type: string; - toolName?: string; - }[]; + type: string + toolName?: string + }[] return part.indices.every((i) => { - const p = content[i]; + const p = content[i] return ( p?.type === "tool-call" && !!p.toolName && RESEARCH_TOOL_NAMES.has(p.toolName) - ); - }); - }); - if (allResearch) return null; + ) + }) + }) + if (allResearch) return null return ( {children} - ); -}; + ) +} const AssistantMessage: FC = () => { // reserves space for action bar and compensates with `-mb` for consistent msg spacing // keeps hovered action bar from shifting layout (autohide doesn't support absolute positioning well) // for pt-[n] use -mb-[n + 6] & min-h-[n + 6] to preserve compensation - const ACTION_BAR_PT = "pt-1.5"; - const ACTION_BAR_HEIGHT = `-mb-7.5 min-h-7.5 ${ACTION_BAR_PT}`; + const ACTION_BAR_PT = "pt-1.5" + const ACTION_BAR_HEIGHT = `-mb-7.5 min-h-7.5 ${ACTION_BAR_PT}` return (
{/* grok 风格研究面板：把本条消息的联网检索/深读聚成一个可折叠时间线 */} @@ -761,11 +762,11 @@ const AssistantMessage: FC = () => { {({ part, children }) => { switch (part.type) { case "group-chainOfThought": - return
{children}
; + return
{children}
case "group-tool": - return {children}; + return {children} case "group-reasoning": { - const running = part.status.type === "running"; + const running = part.status.type === "running" return ( @@ -773,23 +774,23 @@ const AssistantMessage: FC = () => { {children} - ); + ) } case "text": - return ; + return case "reasoning": - return ; + return case "tool-call": // 研究工具（webSearch/readUrl）统一由 ResearchProgress 面板展示， // 这里不再单独渲染，避免重复 - if (RESEARCH_TOOL_NAMES.has(part.toolName)) return null; - return part.toolUI ?? ; + if (RESEARCH_TOOL_NAMES.has(part.toolName)) return null + return part.toolUI ?? case "indicator": - return ; + return case "data": - return part.dataRendererUI; + return part.dataRendererUI default: - return null; + return null } }} @@ -803,25 +804,42 @@ const AssistantMessage: FC = () => {
- ); -}; + ) +} const AssistantActionBar: FC = () => { return ( s.message.isCopied}> - + !s.message.isCopied}> - + + {/* 点赞/点踩 → FeedbackAdapter → Langfuse score；已提交的一侧高亮（data-submitted） */} + + + + + + + + + + @@ -840,10 +858,10 @@ const AssistantActionBar: FC = () => { side="bottom" align="start" sideOffset={6} - className="aui-action-bar-more-content bg-popover/95 text-popover-foreground data-[state=open]:fade-in-0 data-[state=open]:zoom-in-95 data-[state=open]:animate-in data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[state=closed]:animate-out data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 min-w-[8rem] overflow-hidden rounded-xl border p-1.5 shadow-lg backdrop-blur-sm" + className="aui-action-bar-more-content z-50 min-w-[8rem] overflow-hidden rounded-xl border bg-popover/95 p-1.5 text-popover-foreground shadow-lg backdrop-blur-sm data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[state=open]:animate-in data-[state=open]:fade-in-0 data-[state=open]:zoom-in-95" > - + Export as Markdown @@ -852,18 +870,18 @@ const AssistantActionBar: FC = () => { - ); -}; + ) +} const UserMessage: FC = () => { return (
-
+
{(quote) => } @@ -878,8 +896,8 @@ const UserMessage: FC = () => { className="col-span-full col-start-1 row-start-3 -mr-1 justify-end" /> - ); -}; + ) +} const UserActionBar: FC = () => { return ( { - ); -}; + ) +} const EditComposer: FC = () => { return ( { className="mx-auto flex w-full max-w-(--thread-max-width) flex-col px-2" > - +
@@ -927,8 +945,8 @@ const EditComposer: FC = () => { - ); -}; + ) +} const BranchPicker: FC = ({ className, ...rest @@ -937,8 +955,8 @@ const BranchPicker: FC = ({ @@ -956,17 +974,17 @@ const BranchPicker: FC = ({ - ); -}; + ) +} export const Base: FC = () => { - const [sidebarCollapsed, setSidebarCollapsed] = useState(false); + const [sidebarCollapsed, setSidebarCollapsed] = useState(false) return ( -
+
-
+
setSidebarCollapsed(!sidebarCollapsed)} @@ -977,5 +995,5 @@ export const Base: FC = () => {
- ); -}; + ) +} diff --git a/constants/observability.ts b/constants/observability.ts new file mode 100644 index 00000000..d4e1befb --- /dev/null +++ b/constants/observability.ts @@ -0,0 +1,23 @@ +// 可观测与评测相关的共享命名：这些名字会同时出现在代码与 Langfuse 看板/查询里， +// 一旦上报就成为数据的一部分，改名会割裂历史数据，务必集中定义、谨慎变更。 + +/** chat 主链路的 trace/根观测名（Langfuse trace 列表按它筛选） */ +export const CHAT_TRACE_NAME = "chat-turn" + +/** 各 LLM 调用点的 functionId（遥测分组标识） */ +export const TELEMETRY_FUNCTION_IDS = { + chat: "chat", + attachmentInsights: "attachment-insights", + ragEmbedTexts: "rag-embed-texts", + ragEmbedQuery: "rag-embed-query", + evalChat: "eval-chat", +} as const + +/** 用户反馈（点赞/点踩）score 名，BOOLEAN：1=赞 0=踩 */ +export const USER_FEEDBACK_SCORE_NAME = "user-feedback" + +/** trace 上区分普通对话 / 深度研究的 tag */ +export const TRACE_TAGS = { + chat: "chat", + deepResearch: "deep-research", +} as const diff --git "a/docs/observability/01-\350\260\203\347\240\224\346\212\245\345\221\212.md" "b/docs/observability/01-\350\260\203\347\240\224\346\212\245\345\221\212.md" new file mode 100644 index 00000000..c5f2db4e --- /dev/null +++ "b/docs/observability/01-\350\260\203\347\240\224\346\212\245\345\221\212.md" @@ -0,0 +1,101 @@ +# 遥测与评测（Observability & Evals）调研报告 + +> 调研日期：2026-07-06。所有 API 名称均经过本地 `node_modules`（`ai@7.0.14`、`@ai-sdk/otel@1.0.16`、Next.js 16.2.6 自带文档）与官方线上文档/npm registry 交叉验证，非凭记忆书写。 + +## 1. AI SDK 自身的可观测能力（v7） + +### 1.1 v7 遥测体系已彻底重构 + +本仓库用的 `ai@7.0.14`（AI SDK 7 于 2026-06-25 发布）。相比 v5/v6 时代"每次调用传 `experimental_telemetry: { isEnabled: true }`"的旧模式，v7 变化很大： + +| 变化点 | v5/v6 | v7 | +| --- | --- | --- | +| 调用参数名 | `experimental_telemetry`（`TelemetrySettings`） | **`telemetry`**（`TelemetryOptions`）；旧名保留但已标 `@deprecated` | +| 开关语义 | opt-in（默认关，需逐调用 `isEnabled: true`） | **opt-out**（注册集成后默认全开，`isEnabled: false` 按调用退出） | +| OTel 产出 | 内置在 `ai` 包里 | **拆到独立包 `@ai-sdk/otel`**，启动时 `registerTelemetry(new OpenTelemetry())` 注册一次、全局生效 | +| 业务元数据 | `telemetry.metadata` | **已移除**，改由调用参数 `runtimeContext` + `telemetry.includeRuntimeContext` 白名单接替 | +| 自定义 tracer | `telemetry.tracer` | 移除，改为 `new OpenTelemetry({ tracer })` 构造参数 | +| 集成机制 | 无 | 新增 **`Telemetry` 生命周期接口**（`onStart/onStepEnd/onToolExecutionEnd/onEnd/...`），观测厂商可实现该接口直接接事件流，不必依赖 OTel span；Node 下另有 `ai:telemetry` diagnostics channel | + +最小启用方式（Next.js）： + +```ts +// instrumentation.ts（Next 15 起 stable，Next 16 开箱即用，无需任何 experimental 开关） +import { registerTelemetry } from "ai" +import { OpenTelemetry } from "@ai-sdk/otel" + +export function register() { + // 注册某个 OTel TracerProvider（@vercel/otel 或 NodeTracerProvider）... + registerTelemetry(new OpenTelemetry()) +} +``` + +之后所有 `streamText`/`generateText`/`embed`/`embedMany`/`rerank` 调用自动发遥测，调用处最多加一个 `telemetry: { functionId: "chat" }` 用于分组。 + +### 1.2 产出的 span（两套格式） + +- **新格式（`OpenTelemetry` 集成，推荐）**：遵循 OpenTelemetry GenAI 语义约定（`gen_ai.*` 属性）。`streamText` 产出三层 span：`invoke_agent {modelId}`（整个多步操作）→ `chat {modelId}`（每次 LLM 调用，含 `gen_ai.usage.input_tokens/output_tokens`、首包耗时等）→ `execute_tool {toolName}`（每次工具执行，含入参/结果/耗时）。输入输出消息记录在 `gen_ai.input.messages` / `gen_ai.output.messages`。 +- **旧格式（`LegacyOpenTelemetry` 集成，兼容保留）**：v5/v6 时代的 `ai.streamText` → `ai.streamText.doStream` → `ai.toolCall` span 与 `ai.*` 属性，默认不再产出。 + +隐私控制：`recordInputs` / `recordOutputs` 可按调用关闭输入/输出记录（默认全开）。 + +### 1.3 AI SDK 没有内置评测（evals） + +确认 `ai@7.0.14` 的 exports 里没有任何 evals 入口，发布公告与文档也无此规划。Vercel 官方立场是"方法论指导 + 交给第三方"——观测集成目录（https://ai-sdk.dev/providers/observability）列出约 20 家，AI SDK 7 首发适配伙伴包括 **Langfuse、LangSmith**、Braintrust、Datadog、Sentry、Laminar、Raindrop。 + +**结论：AI SDK 只解决"埋点/产出遥测"这一半，落库、看板、打分、数据集评测必须选一个外部平台。** + +## 2. Langfuse vs LangSmith 对比 + +两家都是 AI SDK 7 首发适配伙伴，都提供 tracing + 用户反馈打分 + 数据集/实验 + LLM-as-a-judge 全套能力，核心差异在**开放性、部署形态与生态绑定**。 + +| 维度 | Langfuse | LangSmith | +| --- | --- | --- | +| 开源/自托管 | **平台核心 MIT 开源**，Docker Compose / Helm 免费自托管，核心功能无限制（仅 RBAC/审计日志等企业功能需 EE license） | 平台**闭源**（仅 SDK 是 MIT）；自托管是 **Enterprise 专属附加项**，需联系销售拿 license | +| 云免费额度 | Hobby：**50k units/月**、30 天数据、2 用户（unit = trace/observation/score 各计一条） | Developer：**5k base traces/月**（14 天保留），超出 $2.50/1k | +| 付费起步 | Core $29/月（100k units） | Plus $39/席位/月（10k traces） | +| AI SDK v7 集成 | 专用包 `@langfuse/vercel-ai-sdk`（5.9.1）：`registerTelemetry(new LangfuseVercelAiSdkIntegration())`，底层仍是 OTel span（`LangfuseSpanProcessor` 导出），要求 Node ≥ 22 | `langsmith`（0.7.16）：`registerTelemetry(LangSmithTelemetry())`，走自家 run tree 上报（非 OTel span），无需 instrumentation.ts | +| OTel 开放性 | **OTel 原生**（v5 SDK 整体建立在 OTel 上），span 可同时发给任何 OTLP 后端，数据可移植 | 支持 OTLP ingestion 端点（`api.smith.langchain.com/otel`），但 SDK 主路径是私有协议 | +| 用户反馈打分 | Scores API（NUMERIC/CATEGORICAL/BOOLEAN/TEXT），可挂 trace/observation/session；官方 FAQ 给出"traceId 作为 message id 下发 → 前端/后端回写 score"的完整方案；支持幂等 id 防重复投票 | `client.createFeedback(runId, key, { score })`；前端直传有 presigned feedback token 方案 | +| 数据集/实验 | Datasets（版本化）+ `langfuse.experiment.run({ data, task, evaluators })` SDK runner + UI 对比 | Datasets + `evaluate()`（`langsmith/evaluation`）+ jest/vitest 集成，实验对比 UI 成熟 | +| LLM-as-a-judge | 托管评审器，UI 配置（也有 API），作用于生产流量（可采样）/历史回填/实验 | Online evaluators，UI 配置为主 | +| 生态绑定 | 框架无关 | LangChain 生态一等公民（本项目不用 LangChain，此优势吃不到） | +| 旧集成注意 | v3 时代的 `langfuse-vercel`（LangfuseExporter）已停止演进，文档不再提及；当前主推 v5（`@langfuse/*` 5.9.1） | 旧 `AISDKExporter`（`langsmith/vercel`）**已在 0.4.0 移除**，老教程代码不可用 | + +### 2.1 选型结论：Langfuse + +1. **部署自由度**：本项目是个人项目（本地 Docker Postgres 起家），Langfuse 可以随时 `docker compose up` 免费自托管，也可以先用云端 Hobby（50k units/月对个人对话量非常充裕）；LangSmith 自托管锁在 Enterprise 后面，免费云额度也小一个量级。 +2. **技术路线一致**：Langfuse v5 SDK 是 OTel 原生，与 AI SDK v7 官方遥测体系（OTel GenAI 语义约定）同一技术栈，未来想把 span 同时发给 Grafana/Datadog 等任何 OTLP 后端，加一个 span processor 即可，无迁移成本。 +3. **反馈闭环有官方成熟方案**：`getActiveTraceId()` 把 traceId 直接下发为消息 id，点赞/点踩即可精确回写到对应 trace，无需自建映射表。 +4. LangSmith 的差异化优势（LangChain 深度集成、更成熟的实验对比 UI）在本项目（不用 LangChain、个人规模）价值有限。 + +### 2.2 落地约束确认 + +- `@langfuse/vercel-ai-sdk` peerDeps：`ai >=7.0.0 <8`（本仓库 7.0.14 ✓）、Node ≥ 22（本环境 v22.22.2 ✓）。 +- Next.js 16 `instrumentation.ts` 自 v15 起 stable，放项目根目录导出 `register()` 即可。 +- MiniMax 走 `@ai-sdk/openai-compatible`：遥测与 provider 无关，正常工作；仅 `gen_ai.provider.name` 会记录自报名称 `minimax` 而非知名 provider 枚举值，属预期。 +- Serverless/Vercel 部署需在响应后冲刷：`after(() => langfuseSpanProcessor.forceFlush())`；本地长驻 dev 进程不依赖此步但加上无害。 + +## 3. 实现范围（对应 02-设计方案） + +- **遥测**：`instrumentation.ts` 注册（未配置 Langfuse 环境变量时整体跳过，零开销降级，与 R2/embeddings/search 的可选降级惯例一致）；chat 主链路 + PDF 洞察 + RAG embedding 全部自动追踪并打 `functionId`；chat trace 关联 `sessionId`（= threadId）与研究模式 tag。 +- **评测**： + - 在线（用户反馈）：assistant 消息点赞/点踩 → `/api/feedback` → Langfuse score（BOOLEAN，幂等防改票重复计数）。 + - 离线（数据集实验）：`pnpm eval` 脚本，用 Langfuse Datasets + experiment runner 跑固定测试集，LLM-as-a-judge（复用 MiniMax）+ 规则断言两类 evaluator。 + - 生产在线 LLM-as-a-judge：Langfuse 云端 UI 配置即可，无需代码（文档说明）。 + +## 4. 主要来源 + +- AI SDK v7 telemetry 文档：https://ai-sdk.dev/docs/ai-sdk-core/telemetry （与本地 `node_modules/ai/docs/03-ai-sdk-core/60-telemetry.mdx` 一致） +- AI SDK 7 迁移指南（telemetry 一节）：https://ai-sdk.dev/docs/migration-guides/migration-guide-7-0 +- AI SDK 7 发布公告：https://vercel.com/blog/ai-sdk-7 +- 观测集成目录：https://ai-sdk.dev/providers/observability +- Langfuse × Vercel AI SDK：https://langfuse.com/integrations/frameworks/vercel-ai-sdk +- Langfuse v4→v5 迁移：https://langfuse.com/docs/observability/sdk/upgrade-path/js-v4-to-v5 +- Langfuse Scores / 用户反馈：https://langfuse.com/docs/evaluation/evaluation-methods/scores-via-sdk 、https://langfuse.com/faq/all/user-feedback +- Langfuse Trace IDs：https://langfuse.com/docs/observability/features/trace-ids-and-distributed-tracing +- Langfuse 实验：https://langfuse.com/docs/evaluation/experiments/experiments-via-sdk +- Langfuse 定价/自托管/许可：https://langfuse.com/pricing 、https://langfuse.com/self-hosting 、https://github.com/langfuse/langfuse/blob/main/LICENSE +- LangSmith × Vercel AI SDK：https://docs.langchain.com/langsmith/trace-with-vercel-ai-sdk （旧 `AISDKExporter` 已移除，见 legacy 页说明） +- LangSmith 定价/自托管：https://www.langchain.com/pricing 、https://docs.langchain.com/langsmith/self-hosted +- Next.js instrumentation：https://nextjs.org/docs/app/api-reference/file-conventions/instrumentation （本地 16.2.6 文档验证 v15 起 stable） diff --git "a/docs/observability/02-\350\256\276\350\256\241\346\226\271\346\241\210.md" "b/docs/observability/02-\350\256\276\350\256\241\346\226\271\346\241\210.md" new file mode 100644 index 00000000..1290d313 --- /dev/null +++ "b/docs/observability/02-\350\256\276\350\256\241\346\226\271\346\241\210.md" @@ -0,0 +1,84 @@ +# 遥测与评测 设计说明 + +> 选型依据见 [01-调研报告](./01-调研报告.md)。结论：AI SDK v7 原生遥测（OTel）+ Langfuse（开源、可自托管、OTel 原生）。本功能与 R2/embeddings/search 一样是**可选降级**的：不配 `LANGFUSE_*` 时零开销停用，主流程不受影响。 + +## 1. 总体数据流 + +``` +instrumentation.ts（服务启动，Next 15+ stable） + │ 已配置 LANGFUSE_* 才注册，否则整体跳过 + ▼ +NodeTracerProvider + LangfuseSpanProcessor（batched，smart filter 只导出 LLM/Langfuse span） +registerTelemetry(new LangfuseVercelAiSdkIntegration()) ← AI SDK v7 全局遥测集成 + │ 注册后所有 streamText/generateText/embed 默认发遥测 + ▼ +POST /api/chat + │ startActiveObservation("chat-turn", …, { endOnExit: false }) ← 根观测 + │ propagateAttributes({ sessionId: threadId, tags: [chat|deep-research] }) + │ streamText({ telemetry: { functionId: "chat" }, onEnd/onError/onAbort → 收尾根观测 }) + │ after(() => forceFlush()) ← serverless 冲刷 + ▼ +toUIMessageStreamResponse({ generateMessageId: () => traceId }) + │ assistant 消息 id ≡ 本轮 traceId（32-hex），随消息持久化进 Postgres + ▼ +前端 👍/👎（ActionBarPrimitive.FeedbackPositive/Negative + FeedbackAdapter） + │ POST /api/feedback { messageId, type } + ▼ +langfuse.score.create({ traceId: messageId, name: "user-feedback", value: 1|0, + id: "user-feedback-" }) ← 幂等，改票不重复计数 +``` + +## 2. 关键决策 + +| 决策 | 选择 | 理由 | +| --- | --- | --- | +| 遥测集成 | `@langfuse/vercel-ai-sdk` 的 `LangfuseVercelAiSdkIntegration`（AI SDK v7 callback 集成），而非通用 `@ai-sdk/otel` | Langfuse 官方为 v7 提供的专用集成，观测类型（generation/tool）、usage/cost 映射开箱正确；底层仍是 OTel span，不失开放性 | +| OTel provider | `NodeTracerProvider`（`@opentelemetry/sdk-trace-node`），不用 `@vercel/otel`/`NodeSDK` | 只需要 trace 一条能力，行为最可预期；Langfuse Next.js 指南主示例同款 | +| message↔trace 关联 | **服务端把 traceId 下发为 assistant 消息 id**（`generateMessageId`） | Langfuse 官方 FAQ 方案；id 随消息进 Postgres，反馈时零查表回写；重新生成=新消息 id=新 trace，分支天然各自可评 | +| trace 粒度 | 一次 POST /api/chat = 一条 trace（根观测 `chat-turn`） | 与「一条 assistant 消息」一一对应，反馈/评测的最小单元 | +| 会话分组 | `sessionId = threadId`（请求体里的 `id`，与 `threads.id` 一致） | Langfuse Sessions 视图直接还原整个对话线程 | +| 根观测收尾 | `endOnExit: false` + `onEnd`/`onError`/`onAbort` | 流式响应在 handler 返回后才结束，退出即 end 会把时长截断成首包时间 | +| 反馈上报路径 | 前端 → 自家 `/api/feedback` → `@langfuse/client`（secret key 留在服务端） | 不引入 `NEXT_PUBLIC_*` 密钥与 `@langfuse/browser`，与项目现有 route handler 风格一致 | +| 反馈幂等 | score id = `user-feedback-` | 同一消息改票覆盖同一条 score，不产生重复样本 | +| 历史消息反馈 | messageId 非 32-hex traceId 时 `/api/feedback` 直接 204 忽略 | 功能上线前的旧消息、未启用遥测期间的消息天然无 trace 可挂 | +| 全零 traceId 防护 | `isValidTraceId()` 排除 `0{32}`；未配置时整体绕过 trace 包装 | OTel 无 provider 时 no-op span 的 traceId 是全零，若下发会让所有消息共享同一 id，破坏持久化与分支 | + +## 3. 触点清单 + +| 文件 | 作用 | +| --- | --- | +| `instrumentation.ts` | Next 启动钩子，nodejs runtime 下动态加载注册逻辑 | +| `lib/observability/register.ts` | 注册 NodeTracerProvider + AI SDK 遥测集成（含重复注册防护） | +| `lib/observability/langfuse.ts` | 配置检测、SpanProcessor/LangfuseClient 全局单例（跨 bundle/HMR 共享）、flush、traceId 校验 | +| `constants/observability.ts` | trace 名、score 名、functionId 等对外可见命名（进了 Langfuse 就是数据，集中管理） | +| `app/api/chat/route.ts` | 根观测包装、sessionId/tags 传播、functionId、traceId 下发、after() 冲刷 | +| `app/api/feedback/route.ts` | 点赞/点踩 → score 回写（幂等） | +| `app/page.tsx` | FeedbackAdapter（fire-and-forget POST） | +| `components/examples/base.tsx` | ActionBar 加 👍/👎（`data-submitted` 高亮已提交侧） | +| `lib/attachments/insights.ts`、`lib/ai/embeddings.ts` | 打 functionId（注册集成后自动被追踪，独立成 trace） | +| `scripts/eval.ts`（`pnpm eval`） | 离线评测：experiment run + 规则断言 + LLM-as-a-judge | + +## 4. 评测（三层） + +1. **在线用户反馈**：上述 👍/👎 → `user-feedback` score（BOOLEAN）。Langfuse 里可按 score 过滤 trace，定位差评对话的完整执行轨迹（提示词、工具调用、每步 token）。 +2. **在线 LLM-as-a-judge（无代码）**：Langfuse 控制台 → Evaluators 配置托管评审器，对生产 trace 按采样率自动打分（如 helpfulness/toxicity），也可回填历史。trace 根观测的 input/output 已按其要求写入（`chat-turn` 的 input=最后一条用户消息，output=最终回答）。 +3. **离线数据集评测**：`pnpm eval`。 + - 数据源：默认内置 3 条中文样例；设 `LANGFUSE_EVAL_DATASET=` 改用 Langfuse Datasets（UI 里维护测试集，支持版本化）。 + - task：与线上同款 `minimaxChatModel()` 跑一遍输入。 + - evaluator：`output-well-formed`（规则：非空且无 `` 泄漏）+ `llm-judge-quality`（MiniMax 按「评分要点」给 0~1 分，judge 自身调用不进遥测）。 + - 每次运行是一次 experiment run，换模型/改提示词前后各跑一次即可在 UI 对比回归。 + +## 5. 验证记录（本地，假密钥） + +- 配置 Langfuse（导出地址故意不可达）：`/api/chat` 响应流 `start` part 的 `messageId` 为 32-hex traceId；span/score 导出失败仅 SDK 日志告警，响应不受影响（`flushLangfuseSpans` 吞掉 after() 里的网络错误）。 +- 未配置：`start` part 无 `messageId`（回落到前端生成）；`/api/feedback` 静默 204。 +- `/api/feedback`：合法 body → 204；非 traceId 格式 messageId → 204 跳过；坏 body → 400。 +- `pnpm eval` 无密钥时明确报缺配置退出。 +- `pnpm typecheck`：本次改动 0 新增错误（仓库现存 15 个历史错误，见下）。 + +## 6. 已知事项 / 后续 + +- **仓库现存 15 个 typecheck 错误**（`assistant-modal.tsx`、`mcp-config.tsx`、`tooltip-icon-button.tsx` 等，`render`/`delayDuration` 等 prop 类型漂移 + `@/public/favicon/icon.svg` 声明缺失），系此前依赖升级遗留，`pnpm build` 因此在 type check 阶段失败。与本功能无关，建议单独修复。 +- 反馈 UI 的已提交状态是内存态（assistant-ui `submittedFeedback`），刷新后按钮高亮消失，但 Langfuse 侧记录仍在且幂等；如需回显可后续把反馈也落库。 +- `metadata`（`propagateAttributes`）要求值 ≤200 字符的字符串，目前未用；接入用户体系后可加 `userId`。 +- 深度研究的检索/深读工具调用会自动成为 trace 里的 tool 观测（`execute_tool`），无需额外埋点。 diff --git a/instrumentation.ts b/instrumentation.ts new file mode 100644 index 00000000..ad609399 --- /dev/null +++ b/instrumentation.ts @@ -0,0 +1,9 @@ +// Next.js 服务实例启动时执行一次（v15 起 stable，无需任何 experimental 开关）。 +// OTel 的 Node SDK 只能跑在 nodejs runtime，edge 下直接跳过。 +export async function register() { + if (process.env.NEXT_RUNTIME === "nodejs") { + const { registerObservability } = + await import("./lib/observability/register") + registerObservability() + } +} diff --git a/lib/ai/embeddings.ts b/lib/ai/embeddings.ts index 960a3ff9..f60afdd7 100644 --- a/lib/ai/embeddings.ts +++ b/lib/ai/embeddings.ts @@ -1,5 +1,6 @@ import { createOpenAICompatible } from "@ai-sdk/openai-compatible" import { embed, embedMany } from "ai" +import { TELEMETRY_FUNCTION_IDS } from "@/constants/observability" // Embedding 模型走独立的、可配置的 OpenAI 兼容 provider。 // MiniMax 国际站没有可用的 embeddings，因此 RAG 的向量化交给任意 OpenAI 兼容服务 @@ -22,12 +23,17 @@ export async function embedTexts(texts: string[]): Promise { const { embeddings } = await embedMany({ model: embeddingModel(), values: texts, + telemetry: { functionId: TELEMETRY_FUNCTION_IDS.ragEmbedTexts }, }) return embeddings } /** 单条向量化（查询时用） */ export async function embedQuery(text: string): Promise { - const { embedding } = await embed({ model: embeddingModel(), value: text }) + const { embedding } = await embed({ + model: embeddingModel(), + value: text, + telemetry: { functionId: TELEMETRY_FUNCTION_IDS.ragEmbedQuery }, + }) return embedding } diff --git a/lib/attachments/insights.ts b/lib/attachments/insights.ts index ae6c51cc..69ec96c1 100644 --- a/lib/attachments/insights.ts +++ b/lib/attachments/insights.ts @@ -4,6 +4,7 @@ import { INSIGHTS_INPUT_CHAR_LIMIT, SUGGESTED_QUESTION_COUNT, } from "@/constants/attachment" +import { TELEMETRY_FUNCTION_IDS } from "@/constants/observability" // 上传后基于 PDF 文本生成「摘要 + 建议问题」，解决用户面对空白输入框的冷启动问题。 // 用 generateText + 容错 JSON 解析（而非 generateObject），以兼容任意 OpenAI 兼容端点。 @@ -65,6 +66,7 @@ export async function generateInsights( const { text: raw } = await generateText({ model: minimaxModel(), prompt: buildPrompt(text), + telemetry: { functionId: TELEMETRY_FUNCTION_IDS.attachmentInsights }, }) return parseInsights(raw) } diff --git a/lib/observability/langfuse.ts b/lib/observability/langfuse.ts new file mode 100644 index 00000000..bcdc20a5 --- /dev/null +++ b/lib/observability/langfuse.ts @@ -0,0 +1,51 @@ +import { LangfuseSpanProcessor } from "@langfuse/otel" +import { LangfuseClient } from "@langfuse/client" + +// Langfuse 可观测后端的共享单例。 +// 未配置 LANGFUSE_* 时，各入口应先判 isLangfuseConfigured() 并整体跳过， +// 让遥测零开销降级（与 R2/embeddings/search 的可选降级惯例一致）。 +// +// instrumentation.ts（注册 span processor）与 route handler（响应后 forceFlush） +// 在 Next.js 里属于不同的编译产物，模块级变量不互通，因此单例挂在 globalThis 上； +// 这同时避免了 dev HMR 重复实例化。 + +declare global { + var __langfuseSpanProcessor: LangfuseSpanProcessor | undefined + var __langfuseClient: LangfuseClient | undefined +} + +export function isLangfuseConfigured() { + return Boolean( + process.env.LANGFUSE_PUBLIC_KEY && process.env.LANGFUSE_SECRET_KEY + ) +} + +/** span 上报通道（batched）。密钥/地址走 LANGFUSE_* 环境变量。 */ +export function getLangfuseSpanProcessor(): LangfuseSpanProcessor { + globalThis.__langfuseSpanProcessor ??= new LangfuseSpanProcessor() + return globalThis.__langfuseSpanProcessor +} + +/** REST 客户端：score 上报、datasets/experiments 用 */ +export function getLangfuseClient(): LangfuseClient { + globalThis.__langfuseClient ??= new LangfuseClient() + return globalThis.__langfuseClient +} + +/** + * 冲刷待发送的 span 批次。serverless 部署（Vercel 等）中函数在响应后可能立刻冻结， + * 必须在 next/server 的 after() 里调用；本地长驻 dev 进程调用无害。 + * 导出失败只损失遥测数据，SDK 自会记日志，不向上抛（避免 after() 里刷未处理错误）。 + */ +export async function flushLangfuseSpans() { + if (!globalThis.__langfuseSpanProcessor) return + await globalThis.__langfuseSpanProcessor.forceFlush().catch(() => {}) +} + +/** + * 服务端下发的 assistant 消息 id 就是 W3C traceId（32 位小写 hex），反馈打分靠它回写。 + * 全零是 OTel 的 INVALID_TRACEID（无 TracerProvider 时 no-op span 会给出），必须排除。 + */ +export function isValidTraceId(id: string): boolean { + return /^[0-9a-f]{32}$/.test(id) && id !== "00000000000000000000000000000000" +} diff --git a/lib/observability/register.ts b/lib/observability/register.ts new file mode 100644 index 00000000..fabe8955 --- /dev/null +++ b/lib/observability/register.ts @@ -0,0 +1,28 @@ +import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node" +import { registerTelemetry } from "ai" +import { LangfuseVercelAiSdkIntegration } from "@langfuse/vercel-ai-sdk" +import { getLangfuseSpanProcessor, isLangfuseConfigured } from "./langfuse" + +// 服务启动时（instrumentation.ts 的 register()）调用一次： +// 1. 注册 OTel TracerProvider，span 经 LangfuseSpanProcessor 批量发往 Langfuse +// （processor 自带 smart filter，只导出 Langfuse/GenAI 相关 span）； +// 2. 注册 AI SDK v7 的全局遥测集成，此后所有 streamText/generateText/embed 调用 +// 默认发出遥测事件，由 Langfuse 集成转为 generation/tool 观测。 + +declare global { + var __observabilityRegistered: boolean | undefined +} + +export function registerObservability() { + if (!isLangfuseConfigured()) return + // dev 下 instrumentation 可能随进程内重建再次执行；重复注册 provider 会告警且泄漏 + if (globalThis.__observabilityRegistered) return + globalThis.__observabilityRegistered = true + + const provider = new NodeTracerProvider({ + spanProcessors: [getLangfuseSpanProcessor()], + }) + provider.register() + + registerTelemetry(new LangfuseVercelAiSdkIntegration()) +} diff --git a/package.json b/package.json index 0e6f7b82..9feb4967 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "db:migrate": "drizzle-kit migrate", "db:push": "drizzle-kit push", "db:studio": "drizzle-kit studio", + "eval": "tsx scripts/eval.ts", "openspec:validate": "openspec validate --all --strict" }, "dependencies": { @@ -30,6 +31,11 @@ "@aws-sdk/client-s3": "^3.1079.0", "@aws-sdk/s3-request-presigner": "^3.1079.0", "@base-ui/react": "^1.6.0", + "@langfuse/client": "^5.9.1", + "@langfuse/otel": "^5.9.1", + "@langfuse/tracing": "^5.9.1", + "@langfuse/vercel-ai-sdk": "^5.9.1", + "@opentelemetry/sdk-trace-node": "^2.9.0", "@shadcn/react": "^0.2.0", "@types/react-syntax-highlighter": "^15.5.13", "ai": "^7.0.14", @@ -80,6 +86,7 @@ "prettier": "^3.8.3", "prettier-plugin-tailwindcss": "^0.8.0", "tailwindcss": "^4", + "tsx": "^4.23.0", "typescript": "^5" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f22eef25..31a54241 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -47,6 +47,21 @@ importers: '@base-ui/react': specifier: ^1.6.0 version: 1.6.0(@date-fns/tz@1.5.0)(@types/react@19.2.17)(date-fns@4.4.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@langfuse/client': + specifier: ^5.9.1 + version: 5.9.1(@opentelemetry/api@1.9.1) + '@langfuse/otel': + specifier: ^5.9.1 + version: 5.9.1(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.9.0(@opentelemetry/api@1.9.1))(@opentelemetry/exporter-trace-otlp-http@0.220.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.9.0(@opentelemetry/api@1.9.1)) + '@langfuse/tracing': + specifier: ^5.9.1 + version: 5.9.1(@opentelemetry/api@1.9.1) + '@langfuse/vercel-ai-sdk': + specifier: ^5.9.1 + version: 5.9.1(@opentelemetry/api@1.9.1)(ai@7.0.14(zod@4.4.3))(zod@4.4.3) + '@opentelemetry/sdk-trace-node': + specifier: ^2.9.0 + version: 2.9.0(@opentelemetry/api@1.9.1) '@shadcn/react': specifier: ^0.2.0 version: 0.2.0(@types/react@19.2.17)(react@19.2.4) @@ -192,6 +207,9 @@ importers: tailwindcss: specifier: ^4 version: 4.3.2 + tsx: + specifier: ^4.23.0 + version: 4.23.0 typescript: specifier: ^5 version: 5.9.3 @@ -210,6 +228,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/gateway@4.0.2': + resolution: {integrity: sha512-Jz1BiiTSvhDsCBJrkFRSqLHDRMVjFtYk9GdbSi3UOqY+/epza+oIESMDzfN4m+YHT/1IYmNEmxaMfjXOvxKDjQ==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/mcp@1.0.58': resolution: {integrity: sha512-9Rglb6acqh+yNS8sLbRDBrS4SPHiJYRCBZJ2r7lYfzJx0UtccB/kBxnK7kvYA3Nuwn0eG7m/lLf0s0WP4kAHYw==} engines: {node: '>=18'} @@ -228,12 +252,22 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/otel@1.0.2': + resolution: {integrity: sha512-vmuuwOxJ5OuwMYDktABTLb9V33hwR2k77QIDPZ8mSHSo04seT88IXCmzZwLjTskHfIa5C9IRizzNy6sr2qOQWQ==} + engines: {node: '>=22'} + '@ai-sdk/provider-utils@4.0.35': resolution: {integrity: sha512-bjYld/2KGPLt78kpqbya+fD4LYS7BqVQJyUjE3qAHrYB0FR2Q90BaWEVIBZaguTWXf/A8L6uG1zO1v9TxVlGWg==} engines: {node: '>=18'} peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider-utils@5.0.0': + resolution: {integrity: sha512-zj66M02jc6ASYwIgWZowsooDUwaVngeNZQ3H10GwcPMZ+KR6gHMhcUuKl6tkai+JPXTKDyHY1pnszuxRtw2D4A==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/provider-utils@5.0.5': resolution: {integrity: sha512-oI0t3dvCoqWNV1I8o1Rybi2DXDvHES5r/TrwtJW90tuFLVepgJlftPxrcjh8vaSvjqC2diTuA2vXyjKAyHJm4A==} engines: {node: '>=22'} @@ -244,6 +278,10 @@ packages: resolution: {integrity: sha512-ZPtVYt5QIJzOta1kdUiDuCx4HhFkvNPv/rvmZ2b1iXwybYjJsCnNYR4PAw4kW7rgVfDARvHXcU64efWuqNp6bw==} engines: {node: '>=18'} + '@ai-sdk/provider@4.0.0': + resolution: {integrity: sha512-fr9Gs89prDWiuox/T+kCA+i2cJkHpxU5S+tr4megjTzRC27ZsvFhwjU/+XrqqMbvBUlfmXxTOYWy8ng45dsjIg==} + engines: {node: '>=22'} + '@ai-sdk/provider@4.0.2': resolution: {integrity: sha512-pfPoy9J1B1xV7cqJ8MYHOsDYrMv5tR3+EMNfI249OhkD2uRakvav3Fo7XpD2luuN/YNCBY7KfEQc7vEV7KEtyw==} engines: {node: '>=22'} @@ -1496,6 +1534,38 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@langfuse/client@5.9.1': + resolution: {integrity: sha512-v9lb9GXc7ujmY2k97830MfIg+m1LQWkQNzXP+u2F3KwHVaEvD8zy4v1gLhbTkvemTggAx/tR+tv8BC9CeojTrA==} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + + '@langfuse/core@5.9.1': + resolution: {integrity: sha512-KvyAskAO+2ixJwr9wy148ttR/Zn3oapvOJ2Br4Xcp6zhDpjSIIGB4jW/jkp53/KU9MyEHj0RSFPK/yKoxLC4KA==} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + + '@langfuse/otel@5.9.1': + resolution: {integrity: sha512-viM5Qq/AIPZPXfO7YdSmDHxEDSrNU/MyGzuE9zKA6hGBII1iLowU+qL99O+TjnXqxoADqlNibhY2pg1/WTIPcw==} + engines: {node: '>=20'} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + '@opentelemetry/core': ^2.0.1 + '@opentelemetry/exporter-trace-otlp-http': '>=0.202.0 <1.0.0' + '@opentelemetry/sdk-trace-base': ^2.0.1 + + '@langfuse/tracing@5.9.1': + resolution: {integrity: sha512-tJRyVAv1JkuOPh4Uz5eWUNH8U4jcJVtK2F5QNy5cZUzXCSrXobCSHusPbxY6VFZcLcFgtpDtSaxL7ev1tV2JNQ==} + engines: {node: '>=20'} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + + '@langfuse/vercel-ai-sdk@5.9.1': + resolution: {integrity: sha512-gZPZJ3JXYTkT7W61VhbNHXgn/88+laoAQGMc082ESeXBwfuMVVPKDueZzM0xy6s0BTtPLW5WmbJ5NkLNCtR3Nw==} + engines: {node: '>=22'} + peerDependencies: + '@opentelemetry/api': ^1.9.0 + ai: '>=7.0.0 <8' + '@lexical/clipboard@0.45.0': resolution: {integrity: sha512-9oDu2SNj/EZGjpXJTruz74Ls3VzF2Q41v1QB7JnTq5lh24byvIOdgEpOFHtr/7WVHssfXCbMtgFb5tW8rMaZ2g==} @@ -1664,10 +1734,84 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} + '@opentelemetry/api-logs@0.220.0': + resolution: {integrity: sha512-CmVa4ImJ+ynfrPMNaAXHET6Bhb44SwzmfyVJFq9ni2jgXJR/l7C6gfVFddNmHP+ZOkP9cf4f9DBe68qVLTHc9w==} + engines: {node: '>=8.0.0'} + '@opentelemetry/api@1.9.1': resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} engines: {node: '>=8.0.0'} + '@opentelemetry/context-async-hooks@2.9.0': + resolution: {integrity: sha512-OQ0vzvbZBiUhjqLnUaoNfYmP8553Crr3aggB4y0ZUi815mZ7idpdJXQmoKdeBKJelYttoBlLSSHubmyw3wvX4w==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/core@2.9.0': + resolution: {integrity: sha512-m2nckMT80NnmjTYSPjJQObBJ+8dgkoajEOUbznL8AHZ3T3yHRk2P7gI1PhEBc1+lOnrYE9UWrWHqJDsmqjmNbw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/exporter-trace-otlp-http@0.220.0': + resolution: {integrity: sha512-/+ExB3lRkf+erv4PnoywyL7RHKITidxtUpUTS55k7OQ0dB42S7gEF1gry7swb9MSm1hYLUhJg4QQh9W8SpwwqA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-exporter-base@0.220.0': + resolution: {integrity: sha512-CXYo8UD5Mn9YbgebO2EL4wejtA+gxLmLiu6HCk2KH2BR7XhFN6/6p1UlCb23DYCjeYkndevLHuejCCN1yx4+OQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/otlp-transformer@0.220.0': + resolution: {integrity: sha512-lXGrv7KXZ0gNH9SVNUaa6vv6phVYGvJxfXAlMbzbakiXru75f5MZl8Z7oqiMMQD77riVHJCFlQvbZs/VVN2/4A==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': ^1.3.0 + + '@opentelemetry/resources@2.9.0': + resolution: {integrity: sha512-jyA5MBLQ+Dkl3+JsZkUoUvL7yHvU64kLsvpXKarWm6347Sl1t1bXFTFykUePNpT5WH5pm9a2Qtt03iIYQhZ1Fg==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-logs@0.220.0': + resolution: {integrity: sha512-WywcTkQtv2iNmt+6y5Kcd4rzvx9bLVsBa2Nwcmg01IUaBTkTow3W4d9KE5vNBpEDtb9tp21WcRBY/lANRrApYA==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.4.0 <1.10.0' + + '@opentelemetry/sdk-metrics@2.9.0': + resolution: {integrity: sha512-Xx8RGS4H5XEBl01WuCreMIpiah9cCXMbSkeuIePPdD2cUpq/vUzYmj8E/MK1OsbOc93FuAD4jfn2WOacKwLn7Q==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.9.0 <1.10.0' + + '@opentelemetry/sdk-trace-base@2.9.0': + resolution: {integrity: sha512-cp9zmTl62R8PJrpvFcmc8N2JQU/xfa0S+61q511Nji+QxCfZ8Ifvg7H27G8cANe4crg4RTrWsVvanHiXjSp6ag==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/sdk-trace-node@2.9.0': + resolution: {integrity: sha512-ec9a7ps37huy5itYk0MalaZdSLlM6AXWp/FhtEjgMpp5leEGojBDvAl/UWttQnkMZOvFHKzRESn8TD3yKTF5nQ==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' + + '@opentelemetry/sdk-trace@2.9.0': + resolution: {integrity: sha512-sGA19HvtrrSKYsseHphluH6j3p6Xa3fqc7c7y8f/7mYWejc1lyDFcpSdD1kYa50HCLUeEo4zA5bW0pniaPszuw==} + engines: {node: ^18.19.0 || >=20.6.0} + peerDependencies: + '@opentelemetry/api': '>=1.3.0 <1.10.0' + + '@opentelemetry/semantic-conventions@1.42.0': + resolution: {integrity: sha512-icc5xCzndZfhuJMy5oqk5AvloWquR7jtae74qzpkKkhGp8BivK+oCcEXgGnjCdTfp8hA44l+w8gE8yYJbocJJw==} + engines: {node: '>=14'} + '@posthog/core@1.39.6': resolution: {integrity: sha512-o6ajIwN5zXoNP0D4H/QPmOyibNTUkSyOR6ya7AG5U2ywXx4awo72L2KnCoiZPQM5x/bXv6jPBdimH8M18Ax0aw==} @@ -2859,6 +3003,12 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + ai@7.0.2: + resolution: {integrity: sha512-VMU08jHIDJnnKDrbC9AFa5ZsPpOTfAPRLvTRHtJk4FGAoeldmJROMxvZ2ak5lCjEJ2GP2OLPQbMRyEK8w0+S4A==} + engines: {node: '>=22'} + peerDependencies: + zod: ^3.25.76 || ^4.1.8 + ajv-formats@2.1.1: resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==} peerDependencies: @@ -4662,6 +4812,10 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + mute-stream@2.0.0: resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==} engines: {node: ^18.17.0 || >=20.5.0} @@ -5849,6 +6003,13 @@ snapshots: '@vercel/oidc': 3.2.0 zod: 4.4.3 + '@ai-sdk/gateway@4.0.2(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 4.0.0 + '@ai-sdk/provider-utils': 5.0.0(zod@4.4.3) + '@vercel/oidc': 3.2.0 + zod: 4.4.3 + '@ai-sdk/mcp@1.0.58(zod@4.4.3)': dependencies: '@ai-sdk/provider': 3.0.13 @@ -5869,6 +6030,14 @@ snapshots: '@ai-sdk/provider-utils': 5.0.5(zod@4.4.3) zod: 4.4.3 + '@ai-sdk/otel@1.0.2(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 4.0.0 + '@opentelemetry/api': 1.9.1 + ai: 7.0.2(zod@4.4.3) + transitivePeerDependencies: + - zod + '@ai-sdk/provider-utils@4.0.35(zod@4.4.3)': dependencies: '@ai-sdk/provider': 3.0.13 @@ -5876,6 +6045,14 @@ snapshots: eventsource-parser: 3.1.0 zod: 4.4.3 + '@ai-sdk/provider-utils@5.0.0(zod@4.4.3)': + dependencies: + '@ai-sdk/provider': 4.0.0 + '@standard-schema/spec': 1.1.0 + '@workflow/serde': 4.1.0 + eventsource-parser: 3.1.0 + zod: 4.4.3 + '@ai-sdk/provider-utils@5.0.5(zod@4.4.3)': dependencies: '@ai-sdk/provider': 4.0.2 @@ -5888,6 +6065,10 @@ snapshots: dependencies: json-schema: 0.4.0 + '@ai-sdk/provider@4.0.0': + dependencies: + json-schema: 0.4.0 + '@ai-sdk/provider@4.0.2': dependencies: json-schema: 0.4.0 @@ -7084,6 +7265,39 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@langfuse/client@5.9.1(@opentelemetry/api@1.9.1)': + dependencies: + '@langfuse/core': 5.9.1(@opentelemetry/api@1.9.1) + '@langfuse/tracing': 5.9.1(@opentelemetry/api@1.9.1) + '@opentelemetry/api': 1.9.1 + mustache: 4.2.0 + + '@langfuse/core@5.9.1(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@langfuse/otel@5.9.1(@opentelemetry/api@1.9.1)(@opentelemetry/core@2.9.0(@opentelemetry/api@1.9.1))(@opentelemetry/exporter-trace-otlp-http@0.220.0(@opentelemetry/api@1.9.1))(@opentelemetry/sdk-trace-base@2.9.0(@opentelemetry/api@1.9.1))': + dependencies: + '@langfuse/core': 5.9.1(@opentelemetry/api@1.9.1) + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/exporter-trace-otlp-http': 0.220.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/api@1.9.1) + + '@langfuse/tracing@5.9.1(@opentelemetry/api@1.9.1)': + dependencies: + '@langfuse/core': 5.9.1(@opentelemetry/api@1.9.1) + '@opentelemetry/api': 1.9.1 + + '@langfuse/vercel-ai-sdk@5.9.1(@opentelemetry/api@1.9.1)(ai@7.0.14(zod@4.4.3))(zod@4.4.3)': + dependencies: + '@ai-sdk/otel': 1.0.2(zod@4.4.3) + '@langfuse/core': 5.9.1(@opentelemetry/api@1.9.1) + '@opentelemetry/api': 1.9.1 + ai: 7.0.14(zod@4.4.3) + transitivePeerDependencies: + - zod + '@lexical/clipboard@0.45.0': dependencies: '@lexical/extension': 0.45.0 @@ -7357,8 +7571,90 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} + '@opentelemetry/api-logs@0.220.0': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api@1.9.1': {} + '@opentelemetry/context-async-hooks@2.9.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + + '@opentelemetry/core@2.9.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/semantic-conventions': 1.42.0 + + '@opentelemetry/exporter-trace-otlp-http@0.220.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-exporter-base': 0.220.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.220.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.9.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/otlp-exporter-base@0.220.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/otlp-transformer': 0.220.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/otlp-transformer@0.220.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.220.0 + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-logs': 0.220.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-metrics': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.9.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/resources@2.9.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.42.0 + + '@opentelemetry/sdk-logs@0.220.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/api-logs': 0.220.0 + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.42.0 + + '@opentelemetry/sdk-metrics@2.9.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.9.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-trace-base@2.9.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.42.0 + + '@opentelemetry/sdk-trace-node@2.9.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/context-async-hooks': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/sdk-trace-base': 2.9.0(@opentelemetry/api@1.9.1) + + '@opentelemetry/sdk-trace@2.9.0(@opentelemetry/api@1.9.1)': + dependencies: + '@opentelemetry/api': 1.9.1 + '@opentelemetry/core': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/resources': 2.9.0(@opentelemetry/api@1.9.1) + '@opentelemetry/semantic-conventions': 1.42.0 + + '@opentelemetry/semantic-conventions@1.42.0': {} + '@posthog/core@1.39.6': dependencies: '@posthog/types': 1.392.1 @@ -8564,6 +8860,13 @@ snapshots: '@ai-sdk/provider-utils': 5.0.5(zod@4.4.3) zod: 4.4.3 + ai@7.0.2(zod@4.4.3): + dependencies: + '@ai-sdk/gateway': 4.0.2(zod@4.4.3) + '@ai-sdk/provider': 4.0.0 + '@ai-sdk/provider-utils': 5.0.0(zod@4.4.3) + zod: 4.4.3 + ajv-formats@2.1.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -10649,6 +10952,8 @@ snapshots: ms@2.1.3: {} + mustache@4.2.0: {} + mute-stream@2.0.0: {} nanoid@3.3.15: {} diff --git a/scripts/eval.ts b/scripts/eval.ts new file mode 100644 index 00000000..b5cb58f7 --- /dev/null +++ b/scripts/eval.ts @@ -0,0 +1,143 @@ +/** + * 离线评测：对一组固定测试用例跑一遍对话模型，用「规则断言 + LLM-as-a-judge」两类 + * evaluator 打分，结果作为一次 experiment run 上报 Langfuse，可在 UI 里跨 run 对比 + * （换模型/改提示词前后各跑一次即可看到回归）。 + * + * 用法： + * pnpm eval # 用下方内置样例集 + * LANGFUSE_EVAL_DATASET= pnpm eval # 用 Langfuse 上的同名 dataset + * + * 依赖 .env.local 里的 MINIMAX_* 与 LANGFUSE_* 配置。 + */ +import { config } from "dotenv" + +// minimax provider 在模块加载时就读环境变量，.env.local 必须先注入，业务模块一律动态导入 +config({ path: ".env.local" }) + +const { generateText } = await import("ai") +const { minimaxChatModel, isMinimaxConfigured } = + await import("../lib/ai/minimax") +const { registerObservability } = await import("../lib/observability/register") +const { flushLangfuseSpans, getLangfuseClient, isLangfuseConfigured } = + await import("../lib/observability/langfuse") +const { TELEMETRY_FUNCTION_IDS } = await import("../constants/observability") + +// 内置样例集：input 喂给模型，expectedOutput 是给 judge 参考的「评分要点」而非精确答案 +const LOCAL_EVAL_ITEMS = [ + { + input: "用一句话解释什么是 OpenTelemetry。", + expectedOutput: + "点出它是遥测数据（trace/metrics/log）采集的开源标准/框架即可", + }, + { + input: "PostgreSQL 里如何给 JSONB 字段建 GIN 索引？给出一条示例 SQL。", + expectedOutput: "包含 CREATE INDEX ... USING GIN (jsonb 列) 形式的正确 SQL", + }, + { + input: "我最近工作压力很大，晚上总睡不好，有什么建议吗？", + expectedOutput: "有共情、给出若干可操作的缓解建议、不做医疗诊断", + }, +] + +/** 容错抽取模型输出里的 JSON 对象（judge 可能包一层 ```json 或加说明文字） */ +function extractJson(raw: string): { score?: number; reason?: string } | null { + const start = raw.indexOf("{") + const end = raw.lastIndexOf("}") + if (start === -1 || end <= start) return null + try { + return JSON.parse(raw.slice(start, end + 1)) + } catch { + return null + } +} + +async function main() { + if (!isMinimaxConfigured()) { + console.error("缺少 MINIMAX_API_KEY（.env.local），无法运行评测") + process.exit(1) + } + if (!isLangfuseConfigured()) { + console.error( + "缺少 LANGFUSE_PUBLIC_KEY / LANGFUSE_SECRET_KEY（.env.local），无法上报评测结果" + ) + process.exit(1) + } + + // 与服务端同一套注册逻辑：OTel provider + AI SDK 遥测集成，experiment 的每个 item 产生一条 trace + registerObservability() + const langfuse = getLangfuseClient() + + const datasetName = process.env.LANGFUSE_EVAL_DATASET + const data = datasetName + ? (await langfuse.dataset.get(datasetName)).items + : LOCAL_EVAL_ITEMS + console.log( + datasetName + ? `使用 Langfuse dataset「${datasetName}」，共 ${data.length} 条` + : `使用内置样例集，共 ${data.length} 条（设 LANGFUSE_EVAL_DATASET 可改用远端 dataset）` + ) + + const result = await langfuse.experiment.run({ + name: "chat-quality", + description: "对话质量基线：规则断言 + LLM-as-a-judge", + metadata: { model: process.env.LLM_MODEL_ID ?? "MiniMax-M2" }, + data, + task: async (item) => { + const { text } = await generateText({ + model: minimaxChatModel(), + prompt: String(item.input), + telemetry: { functionId: TELEMETRY_FUNCTION_IDS.evalChat }, + }) + return text + }, + evaluators: [ + // 规则断言：非空，且 reasoning 抽取干净（没把 泄漏进正文） + async ({ output }) => ({ + name: "output-well-formed", + value: + typeof output === "string" && + output.trim().length > 0 && + !output.includes("") + ? 1 + : 0, + dataType: "BOOLEAN" as const, + }), + // LLM-as-a-judge：以 expectedOutput 为评分要点给 0~1 分 + async ({ input, output, expectedOutput }) => { + const { text } = await generateText({ + model: minimaxChatModel(), + prompt: [ + "你是严格的评审员。根据「评分要点」评估「模型回答」对「用户问题」的质量。", + '只输出一个 JSON 对象，不要任何其他文字：{"score": 0到1的小数, "reason": "一句话理由"}', + "", + `用户问题：${String(input)}`, + `评分要点：${String(expectedOutput ?? "（无，凭常识判断有用性与正确性）")}`, + `模型回答：${String(output)}`, + ].join("\n"), + // 评审自身的调用不进遥测，保持 experiment trace 只含被测任务 + telemetry: { isEnabled: false }, + }) + const parsed = extractJson(text) + const score = + typeof parsed?.score === "number" + ? Math.min(1, Math.max(0, parsed.score)) + : 0 + return { + name: "llm-judge-quality", + value: score, + comment: + parsed?.reason ?? `judge 输出无法解析：${text.slice(0, 120)}`, + } + }, + ], + maxConcurrency: 2, + }) + + console.log(await result.format()) + + // 短生命周期进程：退出前把 score 队列与 span 批次全部发出去 + await langfuse.score.flush() + await flushLangfuseSpans() +} + +await main()