diff --git a/api/ops/intent_router.py b/api/ops/intent_router.py new file mode 100644 index 00000000..2649d350 --- /dev/null +++ b/api/ops/intent_router.py @@ -0,0 +1,174 @@ +"""Ops Chat LLM intent router(P1-4)。 + +基于轻量 LLM JSON 输出的混合意图分类器: +- 环境变量 `OPS_CHAT_LLM_ROUTER=1` 时优先调用 LLM router。 +- LLM 输出必须含 `intent`、`slots`、`confidence`(0~1)。 +- 低置信度或非法 JSON / LLM 异常时降级为规则分类器,并记录 + `intent_router.fallback` 事件到 `ops_run_events`。 +- 默认未开启时行为与原有规则分类器完全一致。 +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import time +from collections.abc import Callable +from typing import Any + +from api.ops.llm import chat_completion + +logger = logging.getLogger(__name__) + +# 与 api.ops.orchestrator.core.Intent 保持一致(避免循环导入) +_ALLOWED_INTENTS = { + "metrics_trend", + "issue_list", + "pr_list", + "issue_contribution", + "graph_module", + "scan_status", + "demo", + "fallback", +} + +_MIN_CONFIDENCE = float(os.getenv("OPS_CHAT_LLM_ROUTER_MIN_CONFIDENCE") or "0.7") + + +def _is_enabled() -> bool: + return os.getenv("OPS_CHAT_LLM_ROUTER", "") == "1" + + +def _build_prompt(message: str) -> str: + return ( + "你是 Ops Chat 意图分类助手。根据用户输入,输出严格 JSON(不要 markdown 代码块,不要解释):\n" + "{\n" + ' "intent": "以下之一:metrics_trend, issue_list, pr_list, issue_contribution, ' + "graph_module, scan_status, demo, fallback\",\n" + ' "slots": {},\n' + ' "confidence": 0.0\n' + "}\n" + "slots 可包含 issue_number、days、metric 等键;没有时填 {}。\n" + "confidence 为 0~1 的浮点数,表示分类置信度。\n\n" + f"用户输入:{message}" + ) + + +def _extract_json_obj(content: str) -> dict[str, Any]: + """从 LLM 输出中提取 JSON 对象;失败时抛出异常。""" + try: + data = json.loads(content) + except json.JSONDecodeError: + match = re.search(r"\{.*\}", content, re.S) + if not match: + raise ValueError("No JSON object found in LLM response") from None + data = json.loads(match.group(0)) + if not isinstance(data, dict): + raise ValueError("LLM response JSON is not an object") + return data + + +def _normalize_intent(value: Any) -> str: + """规范化 intent 为允许值之一,未知则 fallback。""" + if not isinstance(value, str): + return "fallback" + intent = value.strip().lower() + if intent in _ALLOWED_INTENTS: + return intent + return "fallback" + + +def _clamp_confidence(value: Any) -> float: + """将 confidence 限制在 [0, 1]。""" + try: + confidence = float(value) if value is not None else 0.0 + except (TypeError, ValueError): + confidence = 0.0 + if not 0.0 <= confidence <= 1.0: + confidence = 0.0 + return confidence + + +def _record_fallback_event( + run_id: str | None, + store: Any, + reason: str, + detail: str, +) -> None: + """记录 intent_router.fallback 事件;store 未提供时仅记录日志。""" + if not run_id: + return + try: + from api.ops.store.runs import append_event + + append_event( + run_id, + "intent_router.fallback", + {"reason": reason, "detail": detail}, + store=store, + ) + except Exception: # pragma: no cover - 防御性降级 + logger.exception("intent_router.fallback event write failed") + + +def llm_classify_intent(message: str) -> tuple[str, dict[str, Any], float]: + """直接调用 LLM 分类意图。 + + 返回 (intent, slots, confidence)。失败时抛出异常,由调用方降级。 + """ + prompt = _build_prompt(message) + start = time.perf_counter() + try: + result = chat_completion( + [{"role": "user", "content": prompt}], + step="intent_router", + temperature=0.1, + ) + except Exception as exc: + latency_ms = (time.perf_counter() - start) * 1000 + logger.warning("router.latency: %.2f ms; LLM failed: %s", latency_ms, exc) + raise + + latency_ms = (time.perf_counter() - start) * 1000 + logger.info("router.latency: %.2f ms", latency_ms) + + data = _extract_json_obj(result.content) + intent = _normalize_intent(data.get("intent")) + slots = data.get("slots") or {} + if not isinstance(slots, dict): + slots = {} + confidence = _clamp_confidence(data.get("confidence")) + return intent, slots, confidence + + +def classify_intent_with_llm( + message: str, + fallback_fn: Callable[[str], tuple[str, dict[str, Any]]], + *, + run_id: str | None = None, + store: Any = None, +) -> tuple[str, dict[str, Any]]: + """混合意图分类器。 + + - 当 `OPS_CHAT_LLM_ROUTER=1` 时优先调用 LLM router。 + - LLM 低置信度或异常时调用 `fallback_fn` 并记录事件。 + - 默认关闭时直接返回 `fallback_fn(message)`,保持向后兼容。 + """ + if not _is_enabled(): + return fallback_fn(message) + + try: + intent, slots, confidence = llm_classify_intent(message) + except Exception as exc: + result = fallback_fn(message) + _record_fallback_event(run_id, store, "llm_error", str(exc)) + return result + + if confidence < _MIN_CONFIDENCE: + result = fallback_fn(message) + _record_fallback_event(run_id, store, "low_confidence", f"confidence={confidence:.3f}") + return result + + return intent, slots diff --git a/api/ops/orchestrator/core.py b/api/ops/orchestrator/core.py index 803c9ea9..b647f4af 100644 --- a/api/ops/orchestrator/core.py +++ b/api/ops/orchestrator/core.py @@ -5,6 +5,7 @@ import re from typing import Any +from api.ops import intent_router as _intent_router from api.ops.agents.graph_analyst import analyze_graph from api.ops.agents.issue_analyst import analyze_issue from api.ops.agents.scan_analyst import analyze_scan @@ -31,7 +32,7 @@ class Intent: FALLBACK = "fallback" -def classify_intent(message: str) -> tuple[str, dict[str, Any]]: +def _rule_classify_intent(message: str) -> tuple[str, dict[str, Any]]: """基于规则快速分类;返回 (intent, slots)。""" msg = message.lower().strip() slots: dict[str, Any] = {} @@ -114,6 +115,24 @@ def classify_intent(message: str) -> tuple[str, dict[str, Any]]: return Intent.FALLBACK, {} +def classify_intent( + message: str, + run_id: str | None = None, + store: Any | None = None, +) -> tuple[str, dict[str, Any]]: + """混合意图分类:默认规则;`OPS_CHAT_LLM_ROUTER=1` 时优先 LLM router。 + + 当 LLM router 低置信度或异常时,自动降级为 `_rule_classify_intent`, + 并通过 `append_event` 记录 `intent_router.fallback` 事件(提供 run_id/store 时)。 + """ + return _intent_router.classify_intent_with_llm( + message, + _rule_classify_intent, + run_id=run_id, + store=store, + ) + + def is_fast_intent(intent: str) -> bool: return intent in (Intent.METRICS_TREND, Intent.ISSUE_LIST, Intent.PR_LIST, Intent.DEMO) diff --git a/docs/_tech_graph/02_version.md b/docs/_tech_graph/02_version.md index c9474185..75479a4e 100644 --- a/docs/_tech_graph/02_version.md +++ b/docs/_tech_graph/02_version.md @@ -56,5 +56,6 @@ timeline 2026-07-07 : dd89b870 auto: api/agently_lab/__init__.py 2026-07-08 : 4bf5782c auto: api/ops/orchestrator/__init__.py 2026-07-09 : db09fd40 auto: api/ops/events_schema.py + 2026-07-10 : 13553deb auto: api/ops/intent_router.py ``` diff --git a/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0929_30_ops_chat_session_sink_p0_p1_P1-4.md b/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0929_30_ops_chat_session_sink_p0_p1_P1-4.md new file mode 100644 index 00000000..729e936f --- /dev/null +++ b/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0929_30_ops_chat_session_sink_p0_p1_P1-4.md @@ -0,0 +1,79 @@ +# Invoke Snapshot · 30-execute-code · ops-chat-session-sink-p0-p1 · P1-4 + +| 字段 | 值 | +| --- | --- | +| **hat** | 30-execute-code | +| **task** | docs/harness/tasks/active/task_ops_chat_session_sink_p0_p1_v1.md | +| **subproject** | ai-ink-brain-api-python | +| **branch** | task/ops-chat-session-sink-p0-p1 | +| **scope** | P1-4 LLM Router:`OPS_CHAT_LLM_ROUTER` · JSON intent · 规则 fallback | +| **timestamp** | 2026-07-10 09:29 | +| **verify_command** | `pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops` | +| **human_gate** | `HG-TASK-DRAFT`: approved (task §行为变更 · human_gate) · `HG-AUDIT-R1`: approved (task §行为变更 · human_gate) | + +## 用户消息快照 + +```text +你正在扮演工作区 Harness「30-execute-code · 执行编码帽」,严格遵循 docs/harness/prompts/30-execute-code.md。 + +**输入(已替换占位符)** +- 主 task 路径(相对 Projects/):`docs/harness/tasks/active/task_ops_chat_session_sink_p0_p1_v1.md` +- 逻辑子仓(相对 Projects/):`ai-ink-brain-api-python` +- Worktree 研发目录(所有 git/pytest/ruff 默认 cwd):`ai-ink-brain-api-python` +- 当前分支:`task/ops-chat-session-sink-p0-p1`(已基于 main fast-forward,包含 P1-3 merge) +- 合并前须跑通的验证命令: + ```bash + pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops + ``` +- 关联任务审核书面结论路径:`ai-ink-brain-api-python/docs/harness/reviews/task_ops_chat_session_sink_p0_p1_v1_audit_R2_20260708.md` +- 关联 PLAN / 总规:`docs/harness/guides/PLAN_ops_chat_session_sink_p0_p1_v1_zh.md` +- 关联结构化输出缺口矩阵:`docs/harness/guides/RUNTIME_structured_output_gap_matrix_v0_zh.md` + +**本棒目标:P1-4 LLM Router:`OPS_CHAT_LLM_ROUTER` · JSON intent · 规则 fallback** + +P1-4 具体要求(来自 PLAN §2、§3.1 D9、task §失败路径、§实现备忘): +- 新增 `api/ops/intent_router.py`:实现基于 LLM 的轻量 JSON intent 路由器。 +- 当环境变量 `OPS_CHAT_LLM_ROUTER=1` 时,`api/ops/orchestrator/core.py` 中的 `classify_intent` 优先调用 LLM router。 +- LLM router 输出 JSON:至少含 `intent`(字符串)、`slots`(对象)、`confidence`(float,0~1)。 +- 低置信度或非法 JSON 时降级为原有规则 `classify_intent`(即 task §失败路径的 `intent_router.fallback` event / `router.latency` 日志)。 +- 默认 `OPS_CHAT_LLM_ROUTER` 未开启或未设置时,行为与之前完全一致(向后兼容)。 +- 记录 `intent_router.fallback` event 到 `ops_run_events`(可复用 P0-2 `append_event`)。 + +**范围限制** +- 只做 P1-4;不改 P1-1 artifact、P1-2 checkpoint、P1-3 clarify 已交付行为 +- 不改 `harness_runtime` 生产图 +- 不改 Agently lab +- 不改前端代码 + +**test_strategy: required** +- 先写/调整可失败的自动化测试,再改实现 +- 新增 `tests/ops/test_intent_router.py` 覆盖: + - `OPS_CHAT_LLM_ROUTER=1` 时 LLM router 返回合法 JSON intent + - 低置信度时降级规则 fallback + - LLM 超时/非法 JSON 时降级规则 fallback(对应 task §失败路径) + - 默认未开启时走原有规则 + - 1 个集成测:通过 `classify_intent` 走 LLM router +- 最终验证命令必须绿 + +**失败路径硬性检查** +- task §失败路径已列 `LLM router 超时/非法 JSON`:行为 = 降级 `classify_intent` 规则;可观测 = `intent_router.fallback` event / `router.latency` 日志;可重试 = 否;验证命令 = `pytest tests/ops/test_intent_router.py -k fallback` + +**你必须完成** +0. **Invoke 快照(开帽起点)**:将本用户消息全文落盘到 `ai-ink-brain-api-python/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_YYYYMMDD_HHMM_30_ops_chat_session_sink_p0_p1_P1-4.md`(含元数据表 + 快照 fenced code)。同一会话内追问 **不** 再新增快照文件。 +0b. **人工闸**:扫描 task / 关联 reviews 的 human_gate。若任一对本帽(30)为 pending → 仅输出须人改的 gate_id 与路径,拒开工;禁止代填 approved。 +1. 通读 task 全文:头部 gates_before_code、audit_profile、orchestration、chain_prompt、test_strategy / test_strategy_note、failure_paths、验收标准、必读列表、非范围。 +2. 阅读 PLAN §2、§3.1、`RUNTIME_structured_output_gap_matrix_v0_zh.md` 与关联 SNAPSHOT。 +3. 先读现有代码:`api/ops/orchestrator/core.py`(`classify_intent` 与 `Intent` 枚举)、`api/ops/events_schema.py`、`api/ops/chat_service.py`。 +4. 先写失败可复现的测试(`tests/ops/test_intent_router.py`),再实现 `api/ops/intent_router.py` 与 `classify_intent` 改造。 +5. 执行验证命令,保留可核对输出要点;修复直至通过。 +6. 按 40-self-check.md 将结论与命令摘要回填至 task 正文「### 自检结论(执行者)· P1-4」小节(不要覆盖 P0、P1-1、P1-2、P1-3 已有结论)。 +7. 对话回复:生成可以完整复制的 Prompt,用于直接交给下一棒 40 自检执行。 +8. **自动 commit**:在输出下一棒 Prompt 且本轮代码/测试/task 自检回填已落盘后,按 HANDOFF_AUTO_COMMIT.md 在 ai-ink-brain-api-python/ commit(仅本轮路径;禁止 git add -A;对话报 short-hash)。 +9. **禁止**自行 push;由 Lead 合并。 + +**输出要求** +- 若拒开工:仅 Markdown 阻塞清单 +- 若执行:diff 摘要、验证命令输出、commit short-hash、下一棒 40 Prompt + +**Judgment(本帽 · 对话末尾必填)**:experience_capture / gate/risk / hat_self +``` diff --git a/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0930_40_ops_chat_session_sink_p0_p1_P1-4.md b/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0930_40_ops_chat_session_sink_p0_p1_P1-4.md new file mode 100644 index 00000000..ecde18aa --- /dev/null +++ b/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0930_40_ops_chat_session_sink_p0_p1_P1-4.md @@ -0,0 +1,75 @@ +--- +hat: "40-self-check" +task: "ops-chat-session-sink-p0-p1" +phase: "P1-4" +subproject: "ai-ink-brain-api-python" +branch: "task/ops-chat-session-sink-p0-p1" +worktree_root: "ai-ink-brain-api-python/" +date: "2026-07-10" +time: "09:30" +--- + +| 字段 | 值 | +| --- | --- | +| **hat** | 40-self-check | +| **task** | ops-chat-session-sink-p0-p1 | +| **phase** | P1-4 LLM Router | +| **subproject** | ai-ink-brain-api-python | +| **branch** | task/ops-chat-session-sink-p0-p1 | +| **worktree_root** | ai-ink-brain-api-python/ | +| **date** | 2026-07-10 | +| **time** | 09:30 | + +## 用户消息快照 + +```text +你正在扮演工作区 Harness「40-self-check · 执行者自检帽」,严格遵循 docs/harness/prompts/40-self-check.md。 + +**输入(已替换占位符)** +- 主 task 路径(相对 Projects/):`docs/harness/tasks/active/task_ops_chat_session_sink_p0_p1_v1.md` +- 逻辑子仓(相对 Projects/):`ai-ink-brain-api-python` +- Worktree 研发目录(所有 git/pytest/ruff 默认 cwd):`ai-ink-brain-api-python` +- 当前分支:`task/ops-chat-session-sink-p0-p1`(已基于 main fast-forward,包含 P1-3 merge) +- 合并前须跑通的验证命令: + ```bash + pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops + ``` +- 失败路径验证命令: + ```bash + pytest tests/ops/test_intent_router.py -k fallback -q + ``` +- 上一棒 30 commit:`ai-ink-brain-api-python@760179a5` +- 关联任务审核书面结论路径:`ai-ink-brain-api-python/docs/harness/reviews/task_ops_chat_session_sink_p0_p1_v1_audit_R2_20260708.md` +- 关联 PLAN / 总规:`docs/harness/guides/PLAN_ops_chat_session_sink_p0_p1_v1_zh.md` +- 关联结构化输出缺口矩阵:`docs/harness/guides/RUNTIME_structured_output_gap_matrix_v0_zh.md` + +**本棒目标:P1-4 自检复核** + +你必须完成: +0. **Invoke 快照(开帽起点)**:将本用户消息全文落盘到 `ai-ink-brain-api-python/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_YYYYMMDD_HHMM_40_ops_chat_session_sink_p0_p1_P1-4.md`(含元数据表 + 快照 fenced code)。同一会话内追问 **不** 再新增快照文件。 +0b. **人工闸**:扫描 task / 关联 reviews 的 human_gate。若任一对本帽(40)为 pending → 仅输出须人改的 gate_id 与路径,拒开工;禁止代填 approved。 +1. 独立阅读 task 正文「### 自检结论(执行者)· P1-4」小节与上一棒 30 invoke 快照 `ai-ink-brain-api-python/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0929_30_ops_chat_session_sink_p0_p1_P1-4.md`。 +2. 独立阅读本轮 P1-4 改动代码: + - `api/ops/intent_router.py` + - `api/ops/orchestrator/core.py`(`classify_intent` / `_rule_classify_intent`) + - `tests/ops/test_intent_router.py` +3. 在 `ai-ink-brain-api-python/` 内完整执行 30 声明的验证命令: + ```bash + pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops + ``` + 并单独执行失败路径验证命令: + ```bash + pytest tests/ops/test_intent_router.py -k fallback -q + ``` +4. 通过 `git diff origin/main...HEAD --stat`(在 `ai-ink-brain-api-python` 内)核对全量变更路径,确认未扩 scope 到 P1-1 artifact、P1-2 checkpoint、P1-3 clarify、`harness_runtime` 生产图、Agently lab、前端代码。 +5. 按 40-self-check.md 将结论与命令摘要回填至 task 正文「### 自检结论(40 复核)· P1-4」小节(不要覆盖 P0、P1-1、P1-2、P1-3 或 30 已有结论)。 +6. 对话回复:生成可以完整复制的 Prompt,用于直接交给下一棒 50 独立复检执行。 +7. 自动 commit:在输出下一棒 Prompt 且本轮 task 自检回填已落盘后,按 HANDOFF_AUTO_COMMIT.md 在 `ai-ink-brain-api-python/` commit(仅本轮路径;禁止 git add -A;对话报 short-hash)。 +8. **禁止**自行 push;由 Lead 合并。 + +**输出要求** +- 若拒开工:仅 Markdown 阻塞清单 +- 若执行:diff 摘要、验证命令输出、commit short-hash、下一棒 50 Prompt + +**Judgment(本帽 · 对话末尾必填)**:experience_capture / gate/risk / hat_self +``` diff --git a/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0947_40_ops_chat_session_sink_p0_p1_P1-4.md b/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0947_40_ops_chat_session_sink_p0_p1_P1-4.md new file mode 100644 index 00000000..61e3aa02 --- /dev/null +++ b/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0947_40_ops_chat_session_sink_p0_p1_P1-4.md @@ -0,0 +1,67 @@ +# Invoke Snapshot · 40-self-check · ops-chat-session-sink-p0-p1 · P1-4 + +| 字段 | 值 | +| --- | --- | +| **hat** | 40-self-check | +| **task** | docs/harness/tasks/active/task_ops_chat_session_sink_p0_p1_v1.md | +| **subproject** | ai-ink-brain-api-python | +| **branch** | task/ops-chat-session-sink-p0-p1 | +| **scope** | P1-4 LLM Router:`OPS_CHAT_LLM_ROUTER` · JSON intent · 规则 fallback | +| **timestamp** | 2026-07-10 09:47 | +| **verify_command** | `pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops` | +| **failure_path_verify_command** | `pytest tests/ops/test_intent_router.py -k fallback -q` | +| **human_gate** | `HG-TASK-DRAFT`: approved (task §行为变更 · human_gate) · `HG-AUDIT-R1`: approved (task §行为变更 · human_gate) | + +## 用户消息快照 + +```text +你正在扮演工作区 Harness「40-self-check · 执行者自检帽」,严格遵循 docs/harness/prompts/40-self-check.md。 + +**输入(已替换占位符)** +- 主 task 路径(相对 Projects/):`docs/harness/tasks/active/task_ops_chat_session_sink_p0_p1_v1.md` +- 逻辑子仓(相对 Projects/):`ai-ink-brain-api-python` +- Worktree 研发目录(所有 git/pytest/ruff 默认 cwd):`ai-ink-brain-api-python` +- 当前分支:`task/ops-chat-session-sink-p0-p1`(已基于 main fast-forward,包含 P1-3 merge) +- 合并前须跑通的验证命令: + ```bash + pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops + ``` +- 失败路径验证命令: + ```bash + pytest tests/ops/test_intent_router.py -k fallback -q + ``` +- 上一棒 30 commit:`ai-ink-brain-api-python@760179a5` +- 关联任务审核书面结论路径:`ai-ink-brain-api-python/docs/harness/reviews/task_ops_chat_session_sink_p0_p1_v1_audit_R2_20260708.md` +- 关联 PLAN / 总规:`docs/harness/guides/PLAN_ops_chat_session_sink_p0_p1_v1_zh.md` +- 关联结构化输出缺口矩阵:`docs/harness/guides/RUNTIME_structured_output_gap_matrix_v0_zh.md` + +**本棒目标:P1-4 自检复核** + +你必须完成: +0. **Invoke 快照(开帽起点)**:将本用户消息全文落盘到 `ai-ink-brain-api-python/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_YYYYMMDD_HHMM_40_ops_chat_session_sink_p0_p1_P1-4.md`(含元数据表 + 快照 fenced code)。同一会话内追问 **不** 再新增快照文件。 +0b. **人工闸**:扫描 task / 关联 reviews 的 human_gate。若任一对本帽(40)为 pending → 仅输出须人改的 gate_id 与路径,拒开工;禁止代填 approved。 +1. 独立阅读 task 正文「### 自检结论(执行者)· P1-4」小节与上一棒 30 invoke 快照 `ai-ink-brain-api-python/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0929_30_ops_chat_session_sink_p0_p1_P1-4.md`。 +2. 独立阅读本轮 P1-4 改动代码: + - `api/ops/intent_router.py` + - `api/ops/orchestrator/core.py`(`classify_intent` / `_rule_classify_intent`) + - `tests/ops/test_intent_router.py` +3. 在 `ai-ink-brain-api-python/` 内完整执行 30 声明的验证命令: + ```bash + pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops + ``` + 并单独执行失败路径验证命令: + ```bash + pytest tests/ops/test_intent_router.py -k fallback -q + ``` +4. 通过 `git diff origin/main...HEAD --stat`(在 `ai-ink-brain-api-python` 内)核对全量变更路径,确认未扩 scope 到 P1-1 artifact、P1-2 checkpoint、P1-3 clarify、`harness_runtime` 生产图、Agently lab、前端代码。 +5. 按 40-self-check.md 将结论与命令摘要回填至 task 正文「### 自检结论(40 复核)· P1-4」小节(不要覆盖 P0、P1-1、P1-2、P1-3 或 30 已有结论)。 +6. 对话回复:生成可以完整复制的 Prompt,用于直接交给下一棒 50 独立复检执行。 +7. 自动 commit:在输出下一棒 Prompt 且本轮 task 自检回填已落盘后,按 HANDOFF_AUTO_COMMIT.md 在 `ai-ink-brain-api-python/` commit(仅本轮路径;禁止 git add -A;对话报 short-hash)。 +8. **禁止**自行 push;由 Lead 合并。 + +**输出要求** +- 若拒开工:仅 Markdown 阻塞清单 +- 若执行:diff 摘要、验证命令输出、commit short-hash、下一棒 50 Prompt + +**Judgment(本帽 · 对话末尾必填)**:experience_capture / gate/risk / hat_self +``` diff --git a/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0947_50_ops_chat_session_sink_p0_p1_P1-4.md b/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0947_50_ops_chat_session_sink_p0_p1_P1-4.md new file mode 100644 index 00000000..858d5dda --- /dev/null +++ b/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_20260710_0947_50_ops_chat_session_sink_p0_p1_P1-4.md @@ -0,0 +1,69 @@ +# 50-self-check Invoke Snapshot · P1-4 + +| 项 | 内容 | +| --- | --- | +| **hat** | 50-self-check | +| **task** | `docs/harness/tasks/active/task_ops_chat_session_sink_p0_p1_v1.md` | +| **subproject** | `ai-ink-brain-api-python` | +| **phase** | P1-4 LLM Router | +| **branch** | `task/ops-chat-session-sink-p0-p1` | +| **timestamp** | 2026-07-10 09:47 | +| **verify_command** | `pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops` | +| **failure_path_verify_command** | `pytest tests/ops/test_intent_router.py -k fallback -q` | + +## Input Snapshot + +```text +你正在扮演工作区 Harness「50-self-check · 独立复检 + 全局验收帽」,严格遵循 docs/harness/prompts/50-self-check.md。 + +**输入(已替换占位符)** +- 主 task 路径(相对 Projects/):`docs/harness/tasks/active/task_ops_chat_session_sink_p0_p1_v1.md` +- 逻辑子仓(相对 Projects/):`ai-ink-brain-api-python` +- Worktree 研发目录(所有 git/pytest/ruff 默认 cwd):`ai-ink-brain-api-python` +- 当前分支:`task/ops-chat-session-sink-p0-p1`(已基于 main fast-forward,包含 P1-3 merge) +- 合并前须跑通的验证命令: + ```bash + pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops + ``` +- 失败路径验证命令: + ```bash + pytest tests/ops/test_intent_router.py -k fallback -q + ``` +- 上一棒 30 commit:`ai-ink-brain-api-python @ 760179a5` +- 上一棒 40 commit:`ai-ink-brain-api-python @ 5c4f05c6` +- 关联任务审核书面结论路径:`ai-ink-brain-api-python/docs/harness/reviews/task_ops_chat_session_sink_p0_p1_v1_audit_R2_20260708.md` +- 关联 PLAN / 总规:`docs/harness/guides/PLAN_ops_chat_session_sink_p0_p1_v1_zh.md` +- 关联结构化输出缺口矩阵:`docs/harness/guides/RUNTIME_structured_output_gap_matrix_v0_zh.md` + +**本棒目标:P1-4 独立复检 + 全局验收** + +你必须完成: +0. **Invoke 快照(开帽起点)**:将本用户消息全文落盘到 `ai-ink-brain-api-python/docs/harness/invokes/by-task/ops-chat-session-sink-p0-p1/invoke_YYYYMMDD_HHMM_50_ops_chat_session_sink_p0_p1_P1-4.md`(含元数据表 + 快照 fenced code)。同一会话内追问 **不** 再新增快照文件。 +0b. **人工闸**:扫描 task / 关联 reviews 的 human_gate。若任一对本帽(50)为 pending → 仅输出须人改的 gate_id 与路径,拒开工;禁止代填 approved。 +1. 独立阅读 task 正文「### 自检结论(执行者)· P1-4」与「### 自检结论(40 复核)· P1-4」小节。 +2. 独立阅读本轮 P1-4 改动代码: + - `api/ops/intent_router.py` + - `api/ops/orchestrator/core.py`(`classify_intent` / `_rule_classify_intent`) + - `tests/ops/test_intent_router.py` +3. 在 `ai-ink-brain-api-python/` 内完整执行验证命令: + ```bash + pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops + ``` + 并单独执行失败路径验证命令: + ```bash + pytest tests/ops/test_intent_router.py -k fallback -q + ``` +4. 执行 `git diff origin/main...HEAD --stat`(在 `ai-ink-brain-api-python` 内)核对全量变更路径,确认未扩 scope 到 P1-1 artifact、P1-2 checkpoint、P1-3 clarify、`harness_runtime` 生产图、Agently lab、前端代码。 +5. 与 30 commit `760179a5`、40 commit(待回填)、R2 任务审核书面结论逐条核对。 +6. 50 复检书面结论落盘:`ai-ink-brain-api-python/docs/harness/reviews/task_ops_chat_session_sink_p0_p1_v1_audit_R1_20260710_50_P1-4.md`。 +7. 按 50-self-check.md 将结论与命令摘要回填至 task 正文「### 自检结论(50 复检)· P1-4」小节(不要覆盖 P0、P1-1、P1-2、P1-3、30、40 已有结论)。 +8. 对话回复:给出是否建议合并、阻塞项清单、全局验收结论。 +9. 自动 commit:在输出最终结论且本轮 reviews/task 自检回填已落盘后,按 HANDOFF_AUTO_COMMIT.md 在 `ai-ink-brain-api-python/` commit(仅本轮路径;禁止 git add -A;对话报 short-hash)。 +10. **禁止**自行 push;由 Lead 合并。 + +**输出要求** +- 若拒开工:仅 Markdown 阻塞清单 +- 若执行:diff 摘要、验证命令输出、commit short-hash、合并建议、全局验收表 + +**Judgment(本帽 · 对话末尾必填)**:experience_capture / gate/risk / hat_self +``` diff --git a/docs/harness/reviews/task_ops_chat_session_sink_p0_p1_v1_audit_R1_20260710_50_P1-4.md b/docs/harness/reviews/task_ops_chat_session_sink_p0_p1_v1_audit_R1_20260710_50_P1-4.md new file mode 100644 index 00000000..06c25c51 --- /dev/null +++ b/docs/harness/reviews/task_ops_chat_session_sink_p0_p1_v1_audit_R1_20260710_50_P1-4.md @@ -0,0 +1,164 @@ +# 50 独立复检 + 全局验收报告 · Ops Chat Session Sink P0+P1 · P1-4 LLM Router + +| 项 | 内容 | +| --- | --- | +| **task** | `docs/harness/tasks/active/task_ops_chat_session_sink_p0_p1_v1.md` | +| **subproject** | `ai-ink-brain-api-python` | +| **scope** | P1-4 LLM Router:`OPS_CHAT_LLM_ROUTER` · JSON intent · 规则 fallback | +| **hat** | 50-independent-reinspect | +| **date** | 2026-07-10 | +| **30 commit** | `ai-ink-brain-api-python@760179a5` | +| **40 commit** | `ai-ink-brain-api-python@5c4f05c6` · `Projects@9fef12f` | + +--- + +## 复核方法 + +1. 独立阅读 task 正文「### 自检结论(执行者)· P1-4」与「### 自检结论(40 复核)· P1-4」小节。 +2. 独立阅读本轮 P1-4 改动代码: + - `api/ops/intent_router.py` + - `api/ops/orchestrator/core.py`(`classify_intent` / `_rule_classify_intent`) + - `tests/ops/test_intent_router.py` +3. 在 `ai-ink-brain-api-python/` 内完整执行 30 声明的验证命令: + ```bash + pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q && ruff check api/ops + ``` +4. 单独执行失败路径验证命令: + ```bash + pytest tests/ops/test_intent_router.py -k fallback -q + ``` +5. 执行 `git diff origin/main...HEAD --stat` 核对全量变更路径,确认未扩 scope 到 P1-1 artifact、P1-2 checkpoint、P1-3 clarify、`harness_runtime` 生产图、Agently lab、前端代码。 +6. 与 30 commit `760179a5`、40 commit `5c4f05c6` / `Projects@9fef12f`、R2 任务审核书面结论逐条核对。 + +--- + +## 命令输出 + +### 完整验证命令 + +```text +pytest tests/ops tests/ops_desk -m "not intent_eval and not intent_benchmark" -q +............................................s........................... [ 22%] +........................................................................ [ 44%] +........................................................................ [ 66%] +.........................................ss......................sssssss [ 88%] +...................................... [100%] +=============================== warnings summary ================================ +../../../miniconda3/lib/python3.13/site-packages/fastapi/testclient.py:1 + /Users/cyning/miniconda3/lib/python3.13/site-packages/fastapi/testclient.py:1: StarletteDeprecationWarning: Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead. + from starlette.testclient import TestClient as TestClient # noqa + +-- Docs: https://docs.pytest.org/en/stable/howto/capture-warnings.html +=========================== short test summary info ============================ +SKIPPED [1] tests/ops/test_events_schema.py:181: 需要真实 Supabase 连接;本地/CI 环境缺失时跳过 +SKIPPED [2] tests/ops_desk/test_run_schema_p1.py:102: public 中表已存在,跳过写测试以避免破坏数据 +SKIPPED [7] tests/ops_desk/test_schema_p0.py:102: public 中表已存在,跳过写测试以避免破坏数据 +316 passed, 10 skipped, 1 warning in 57.58s + +ruff check api/ops +All checks passed! +``` + +- pytest 退出码:`0` +- ruff 退出码:`0` +- 10 skipped 中:1 个为 `tests/ops/test_events_schema.py::test_append_event_integration_with_real_store`(显式 skip,需真实 Supabase 连接);其余 9 个为 `tests/ops_desk/test_run_schema_p1.py` / `tests/ops_desk/test_schema_p0.py` 中环境感知跳过(表已存在),与 P1-4 改动无关。 + +### 失败路径额外验证 + +```text +pytest tests/ops/test_intent_router.py -k fallback -q +... [100%] +3 passed, 3 deselected in 0.38s +``` + +- pytest 退出码:`0` +- 覆盖:低置信度 fallback、非法 JSON fallback、LLM 超时 fallback;均记录 `intent_router.fallback` event。 + +--- + +## 全量变更路径核对 + +```text + api/ops/intent_router.py | 174 ++++++++++++++++ + api/ops/orchestrator/core.py | 21 +- + docs/_tech_graph/02_version.md | 1 + + ...710_0929_30_ops_chat_session_sink_p0_p1_P1-4.md | 79 +++++++ + ...710_0930_40_ops_chat_session_sink_p0_p1_P1-4.md | 75 +++++++ + ...710_0947_40_ops_chat_session_sink_p0_p1_P1-4.md | 67 ++++++ + ...710_0947_50_ops_chat_session_sink_p0_p1_P1-4.md | 69 ++++++ + tests/ops/test_intent_router.py | 231 +++++++++++++++++++++ + 8 files changed, 716 insertions(+), 1 deletion(-) +``` + +- 代码变更仅涉及 `api/ops/intent_router.py`、`api/ops/orchestrator/core.py`、`tests/ops/test_intent_router.py`。 +- `docs/_tech_graph/02_version.md` 追加版本时间点,属技术图谱例行更新,未改生产代码。 +- 文档变更为本轮 invoke 快照、task 自检结论与本 50 复检报告,属 harness 落盘工件。 +- 未涉及 P1-1 artifact、P1-2 checkpoint、P1-3 clarify、`harness_runtime` 生产图、Agently lab、前端代码。 + +--- + +## 与 30 / 40 结论差异核对 + +| 30 / 40 声称项 | 50 独立复核 | 结果 | +| --- | --- | --- | +| 新增 `api/ops/intent_router.py` 轻量 JSON LLM router | 文件存在;`_build_prompt` / `llm_classify_intent` / `classify_intent_with_llm` 实现完整;输出含 `intent` / `slots` / `confidence` | 一致 | +| `OPS_CHAT_LLM_ROUTER=1` 时 `classify_intent` 优先调用 LLM router | `core.py:118-133` 委托 `_intent_router.classify_intent_with_llm`;集成测 `test_classify_intent_uses_llm_router` 通过 | 一致 | +| 低置信度降级规则 fallback | `intent_router.py:169-172` 在 `confidence < _MIN_CONFIDENCE` 时调用 fallback;`test_llm_router_low_confidence_fallback` 通过 | 一致 | +| 非法 JSON / LLM 异常降级规则 fallback | `_extract_json_obj` 抛异常与 `chat_completion` 异常均在 `classify_intent_with_llm` 捕获并 fallback;对应测例通过 | 一致 | +| 默认未开启时行为与之前一致 | `_is_enabled()` 仅在 `OPS_CHAT_LLM_ROUTER=1` 时启用;`test_llm_router_disabled_uses_rule` 通过 | 一致 | +| 记录 `intent_router.fallback` event | `_record_fallback_event` 复用 P0-2 `append_event`;fallback 测例断言 store 中存在 `event_type=intent_router.fallback` | 一致 | +| `router.latency` 日志 | `llm_classify_intent` 在成功/失败路径均记录 `router.latency` | 一致 | +| 最终验证命令绿 | 本轮独立跑通 `316 passed, 10 skipped` + ruff 全绿 | 一致 | +| 未扩 scope | 全量 diff 仅 P1-4 router 相关文件 + docs 版本时间点 + invoke;未涉及 P1-1/2/3、Session 生产图、Agently lab、前端 | 一致 | +| P0 ~ P1-3 已有实现未被破坏 | `review/rules.py`、`events_schema.py`、`store/runs.py`、`chat_context.py`、`chat_service.py`、`orchestrator/clarify.py` 未改动;相关测例全绿 | 一致 | + +**差异项**:无。 + +--- + +## 验收项复核表 + +| 验收项 | 状态 | 证据 | 备注 | +| --- | --- | --- | --- | +| `api/ops/intent_router.py` 存在且实现 LLM JSON intent router | pass | 文件新增;`_build_prompt` / `llm_classify_intent` / `classify_intent_with_llm` 完整;返回含 `intent` / `slots` / `confidence` | — | +| `OPS_CHAT_LLM_ROUTER=1` 时 `classify_intent` 优先调用 LLM router | pass | `core.py:118-133`;集成测 `test_classify_intent_uses_llm_router` 通过 | — | +| 低置信度降级规则 fallback | pass | `intent_router.py:169-172`;`test_llm_router_low_confidence_fallback` 通过 | — | +| 非法 JSON / LLM 异常降级规则 fallback | pass | `classify_intent_with_llm` 异常捕获分支;`test_llm_router_invalid_json_fallback` / `test_llm_router_timeout_fallback` 通过 | — | +| 默认未开启时行为与之前一致 | pass | `_is_enabled()` 仅在 `OPS_CHAT_LLM_ROUTER=1` 时启用;`test_llm_router_disabled_uses_rule` 通过 | — | +| 记录 `intent_router.fallback` event | pass | `_record_fallback_event` 复用 P0-2 `append_event`;fallback 测例断言 event 存在 | — | +| `router.latency` 日志 | pass | `llm_classify_intent` 成功/失败路径均记录 `router.latency` | — | +| `tests/ops/test_intent_router.py` 覆盖目标场景 | pass | 6 测例全绿:启用返回合法 intent、低置信度 fallback、非法 JSON fallback、超时 fallback、默认规则、集成路径 | — | +| task §失败路径验证命令绿 | pass | `pytest tests/ops/test_intent_router.py -k fallback -q` → `3 passed, 3 deselected` | — | +| 最终验证命令绿 | pass | pytest `316 passed, 10 skipped` + ruff `All checks passed!`;退出码均为 `0` | — | +| 未静默扩大 scope | pass | 全量 diff 路径清单见上文 | 未改 P1-1/2/3、Session 生产图、Agently lab、前端 | +| P0 ~ P1-3 已有实现未被破坏 | pass | `review/rules.py`、`events_schema.py`、`store/runs.py`、`chat_context.py`、`chat_service.py`、`orchestrator/clarify.py` 未改动;相关测例全绿 | — | + +--- + +## 阻塞项清单 + +无。 + +--- + +## 是否建议合并 + +**建议合并**。P1-4 LLM Router 实现、测试、30 执行、40 自检、50 独立复检均通过,无 scope creep,人工闸 `HG-TASK-DRAFT` / `HG-AUDIT-R1` 已 approved。 + +--- + +## 执行路线与 Commit 回溯 + +| 阶段 | 帽子 | 关键动作 | 落盘工件 | 对应 commit | +|------|------|----------|----------|-------------| +| P1-4 | 30 execute | LLM intent router + `classify_intent` 混合分类器 + 测试 | `api/ops/intent_router.py`, `api/ops/orchestrator/core.py`, `tests/ops/test_intent_router.py` | `ai-ink-brain-api-python@760179a5` | +| P1-4 | 40 self-check | 复核 P1-4 验收 | task 内 P1-4 30/40 自检结论 | `ai-ink-brain-api-python@5c4f05c6` · `Projects@9fef12f` | +| P1-4 | 50 reinspect R1 | 独立复检 + 全局验收 | `ai-ink-brain-api-python/docs/harness/reviews/task_ops_chat_session_sink_p0_p1_v1_audit_R1_20260710_50_P1-4.md` | 待本审查落盘后 commit | + +--- + +## Judgment(50) + +- **experience_capture**: `required` — LLM router JSON intent、confidence 阈值、规则 fallback、事件记录模式可复用到后续 ChatBI / Session 路由。 +- **gate/risk**: 无 — `HG-TASK-DRAFT` / `HG-AUDIT-R1` 均为 `approved`;50 未遇 pending 人工闸。 +- **hat_self**: `pass` — 独立复检与 30/40 结论一致,验证命令绿,失败路径额外验证绿,输出 pass/fail 表、阻塞项清单、合并建议与执行路线回溯。 diff --git a/tests/ops/test_intent_router.py b/tests/ops/test_intent_router.py new file mode 100644 index 00000000..0349466f --- /dev/null +++ b/tests/ops/test_intent_router.py @@ -0,0 +1,231 @@ +"""P1-4: Ops Chat LLM intent router 单测。 + +覆盖: +- OPS_CHAT_LLM_ROUTER=1 时 LLM router 返回合法 JSON intent。 +- 低置信度时降级为规则 fallback。 +- LLM 超时 / 非法 JSON 时降级为规则 fallback。 +- 默认未开启时走原有规则。 +- 通过 classify_intent 走 LLM router 的集成路径。 +""" + +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from api.ops.llm.types import LlmCompletionResult, LlmUsage + + +def _make_llm_result( + content: str, + step: str = "intent_router", + latency_ms: float = 80.0, +) -> LlmCompletionResult: + return LlmCompletionResult( + content=content, + usage=LlmUsage( + provider="siliconflow", + model="Qwen/Qwen2.5-72B-Instruct", + prompt_tokens=20, + completion_tokens=10, + total_tokens=30, + latency_ms=latency_ms, + step=step, + ), + ) + + +class FakeStore: + """内存版 OpsRunStore,仅记录 append_event。""" + + def __init__(self) -> None: + self.events: list[dict[str, Any]] = [] + + def append_event( + self, + run_id: str, + agent_role: str, + event_type: str, + payload: dict[str, Any] | None = None, + node_id: str | None = None, + seq: int | None = None, + ) -> dict[str, Any]: + evt: dict[str, Any] = { + "run_id": run_id, + "agent_role": agent_role, + "event_type": event_type, + "payload": payload or {}, + "node_id": node_id, + "seq": seq or len(self.events) + 1, + } + self.events.append(evt) + return evt + + +def test_llm_router_enabled_returns_valid_intent(monkeypatch: pytest.MonkeyPatch) -> None: + """OPS_CHAT_LLM_ROUTER=1 且 LLM 返回合法 JSON 时,优先采用 LLM 结果。""" + monkeypatch.setenv("OPS_CHAT_LLM_ROUTER", "1") + from api.ops import intent_router + + def fake_chat_completion(messages: list[dict[str, str]], **kwargs: Any) -> LlmCompletionResult: + return _make_llm_result( + json.dumps( + {"intent": "issue_contribution", "slots": {"issue_number": 123}, "confidence": 0.95}, + ensure_ascii=False, + ) + ) + + monkeypatch.setattr(intent_router, "chat_completion", fake_chat_completion) + + intent, slots = intent_router.classify_intent_with_llm("分析 #123", lambda _msg: ("fallback", {})) + + assert intent == "issue_contribution" + assert slots == {"issue_number": 123} + + +def test_llm_router_low_confidence_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + """置信度低于阈值时降级为规则 fallback,并记录 intent_router.fallback 事件。""" + monkeypatch.setenv("OPS_CHAT_LLM_ROUTER", "1") + from api.ops import intent_router + + def fake_chat_completion(messages: list[dict[str, str]], **kwargs: Any) -> LlmCompletionResult: + return _make_llm_result( + json.dumps( + {"intent": "issue_contribution", "slots": {"issue_number": 123}, "confidence": 0.2}, + ensure_ascii=False, + ) + ) + + monkeypatch.setattr(intent_router, "chat_completion", fake_chat_completion) + + fallback_calls: list[str] = [] + + def rule_fallback(msg: str) -> tuple[str, dict[str, Any]]: + fallback_calls.append(msg) + return "metrics_trend", {"days": 7} + + store = FakeStore() + intent, slots = intent_router.classify_intent_with_llm( + "比较含糊的问题", + rule_fallback, + run_id="run-low-conf", + store=store, + ) + + assert intent == "metrics_trend" + assert slots == {"days": 7} + assert fallback_calls == ["比较含糊的问题"] + + fallback_events = [e for e in store.events if e["event_type"] == "intent_router.fallback"] + assert len(fallback_events) == 1 + assert fallback_events[0]["payload"]["reason"] == "low_confidence" + assert "0.2" in fallback_events[0]["payload"]["detail"] + + +def test_llm_router_invalid_json_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + """LLM 返回非法 JSON 时降级为规则 fallback,并记录 intent_router.fallback 事件。""" + monkeypatch.setenv("OPS_CHAT_LLM_ROUTER", "1") + from api.ops import intent_router + + def fake_chat_completion(messages: list[dict[str, str]], **kwargs: Any) -> LlmCompletionResult: + return _make_llm_result("这不是 JSON") + + monkeypatch.setattr(intent_router, "chat_completion", fake_chat_completion) + + def rule_fallback(_msg: str) -> tuple[str, dict[str, Any]]: + return "fallback", {} + + store = FakeStore() + intent, slots = intent_router.classify_intent_with_llm( + "模糊问题", + rule_fallback, + run_id="run-bad-json", + store=store, + ) + + assert intent == "fallback" + assert slots == {} + + fallback_events = [e for e in store.events if e["event_type"] == "intent_router.fallback"] + assert len(fallback_events) == 1 + assert fallback_events[0]["payload"]["reason"] == "llm_error" + + +def test_llm_router_timeout_fallback(monkeypatch: pytest.MonkeyPatch) -> None: + """LLM 调用超时/抛异常时降级为规则 fallback,并记录 intent_router.fallback 事件。""" + monkeypatch.setenv("OPS_CHAT_LLM_ROUTER", "1") + from api.ops import intent_router + + def fake_chat_completion(messages: list[dict[str, str]], **kwargs: Any) -> LlmCompletionResult: + raise TimeoutError("LLM timeout") + + monkeypatch.setattr(intent_router, "chat_completion", fake_chat_completion) + + def rule_fallback(_msg: str) -> tuple[str, dict[str, Any]]: + return "metrics_trend", {"metric": "cycle-time"} + + store = FakeStore() + intent, slots = intent_router.classify_intent_with_llm( + "最近 PR 周期多少", + rule_fallback, + run_id="run-timeout", + store=store, + ) + + assert intent == "metrics_trend" + assert slots == {"metric": "cycle-time"} + + fallback_events = [e for e in store.events if e["event_type"] == "intent_router.fallback"] + assert len(fallback_events) == 1 + assert fallback_events[0]["payload"]["reason"] == "llm_error" + assert "LLM timeout" in fallback_events[0]["payload"]["detail"] + + +def test_llm_router_disabled_uses_rule(monkeypatch: pytest.MonkeyPatch) -> None: + """默认未开启 OPS_CHAT_LLM_ROUTER 时,走原有规则,不调用 LLM。""" + monkeypatch.delenv("OPS_CHAT_LLM_ROUTER", raising=False) + from api.ops import intent_router + + llm_calls: list[Any] = [] + + def fake_chat_completion(messages: list[dict[str, str]], **kwargs: Any) -> LlmCompletionResult: + llm_calls.append(messages) + return _make_llm_result(json.dumps({"intent": "issue_list", "slots": {}, "confidence": 0.9})) + + monkeypatch.setattr(intent_router, "chat_completion", fake_chat_completion) + + def rule_fallback(_msg: str) -> tuple[str, dict[str, Any]]: + return "pr_list", {} + + intent, slots = intent_router.classify_intent_with_llm("列出 PR", rule_fallback) + + assert intent == "pr_list" + assert slots == {} + assert llm_calls == [] + + +def test_classify_intent_uses_llm_router(monkeypatch: pytest.MonkeyPatch) -> None: + """集成测:通过 api.ops.orchestrator.classify_intent 走 LLM router。""" + monkeypatch.setenv("OPS_CHAT_LLM_ROUTER", "1") + from api.ops import intent_router + from api.ops.orchestrator import classify_intent + + def fake_chat_completion(messages: list[dict[str, str]], **kwargs: Any) -> LlmCompletionResult: + return _make_llm_result( + json.dumps( + {"intent": "issue_list", "slots": {}, "confidence": 0.88}, + ensure_ascii=False, + ) + ) + + monkeypatch.setattr(intent_router, "chat_completion", fake_chat_completion) + + store = FakeStore() + intent, slots = classify_intent("列出所有 issues", run_id="run-integration", store=store) + + assert intent == "issue_list" + assert slots == {} + fallback_events = [e for e in store.events if e["event_type"] == "intent_router.fallback"] + assert fallback_events == []