From 2d7f76a011f902d22a43200ffcfd04d78722d75c Mon Sep 17 00:00:00 2001 From: ACE Engineering Date: Fri, 28 Aug 2026 11:32:10 -0700 Subject: [PATCH] feat(quality): exclude models and agents with fewer than 20 turns from evaluation --- src/ace/sidecar/insights.py | 25 ++++++-- tests/test_quality_metrics.py | 104 +++++++++++++++++++++++----------- 2 files changed, 89 insertions(+), 40 deletions(-) diff --git a/src/ace/sidecar/insights.py b/src/ace/sidecar/insights.py index 0dea4db..fe8ebcf 100644 --- a/src/ace/sidecar/insights.py +++ b/src/ace/sidecar/insights.py @@ -2436,12 +2436,15 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: } +MIN_QUALITY_EVAL_TURNS = 20 + + def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: """Calculates unified code quality, verification hygiene, and reliability metrics. Includes top-line metrics along with breakdowns: - - by_agent: Quality scores partitioned per agent engine (Claude Code, Antigravity, Codex). - - by_model: Quality scores partitioned per LLM model. + - by_agent: Quality scores partitioned per agent engine (Claude Code, Antigravity, Codex) with >= 20 turns. + - by_model: Quality scores partitioned per LLM model with >= 20 turns. - by_category: Quality and capability metrics partitioned per coding task domain (UI, Backend, Testing, Docs, Research). """ overall = _calc_quality_block(sess) @@ -2451,7 +2454,7 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: overall["by_category"] = {} return overall - # Group by agent + # Group by agent (exclude agents with < 20 turns) by_agent: Dict[str, Any] = {} agent_groups: Dict[str, List[Dict[str, Any]]] = {} for s in sess: @@ -2459,15 +2462,19 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: agent_groups.setdefault(ak, []).append(s) for ak, a_sess in agent_groups.items(): + a_turns = sum(len(s.get("turns", [])) for s in a_sess) + if a_turns < MIN_QUALITY_EVAL_TURNS: + continue block = _calc_quality_block(a_sess) by_agent[ak] = { "agent": ak, "label": AGENTS.get(ak, ak.capitalize()), "sessions": len(a_sess), + "total_turns": a_turns, **block, } - # Group by model + # Group by model (exclude models with < 20 turns or synthetic tags) model_sessions: Dict[str, List[Dict[str, Any]]] = {} for s in sess: models_in_s = set(t.get("model") for t in s.get("turns", []) if t.get("model")) @@ -2485,11 +2492,15 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: by_model: List[Dict[str, Any]] = [] for m_name, m_sess in sorted(model_sessions.items(), key=lambda kv: -len(kv[1])): + m_turns = sum(len(s.get("turns", [])) for s in m_sess) + if m_turns < MIN_QUALITY_EVAL_TURNS or str(m_name).startswith("<"): + continue block = _calc_quality_block(m_sess) by_model.append( { "model": m_name, "sessions": len(m_sess), + "total_turns": m_turns, **block, } ) @@ -2509,7 +2520,8 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: best_agent_score = -1 for ak in (AGENT_CLAUDE, AGENT_ANTIGRAVITY, AGENT_CODEX): sub_ak = [s for s in c_sess if (s.get("agent_type") or AGENT_CLAUDE) == ak] - if sub_ak: + sub_ak_turns = sum(len(s.get("turns", [])) for s in sub_ak) + if sub_ak and sub_ak_turns >= MIN_QUALITY_EVAL_TURNS: sc = _calc_quality_block(sub_ak)["quality_score"] if sc > best_agent_score: best_agent_score = sc @@ -2529,7 +2541,8 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: proj = [t for t in s.get("turns", []) if t.get("model") == m] if proj: sub_m.append({"turns": proj, "events": s.get("events", [])}) - if sub_m: + sub_m_turns = sum(len(s.get("turns", [])) for s in sub_m) + if sub_m and sub_m_turns >= MIN_QUALITY_EVAL_TURNS: sc = _calc_quality_block(sub_m)["quality_score"] if sc > best_model_score: best_model_score = sc diff --git a/tests/test_quality_metrics.py b/tests/test_quality_metrics.py index 44e74d0..9021b5f 100644 --- a/tests/test_quality_metrics.py +++ b/tests/test_quality_metrics.py @@ -277,48 +277,53 @@ def test_quality_in_payload_and_prometheus() -> None: def test_quality_metrics_by_agent_and_model() -> None: + # 20 turns per session to satisfy MIN_QUALITY_EVAL_TURNS + claude_turns = [ + { + "model": "claude-sonnet-4-6", + "input_tokens": 100, + "output_tokens": 20, + "cache_read_input_tokens": 80, + "cache_creation_input_tokens": 0, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "calls": [ + {"name": "Edit", "raw_target": "src/a.py", "is_edit": True, "is_src_file": True}, + {"name": "Bash", "command": "pytest", "is_test_run": True, "is_error": False}, + ] if i == 0 else [], + } + for i in range(20) + ] + + agy_turns = [ + { + "model": "gemini-3.6-flash", + "input_tokens": 50, + "output_tokens": 10, + "cache_read_input_tokens": 30, + "cache_creation_input_tokens": 0, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "calls": [ + {"name": "write_to_file", "raw_target": "src/b.py", "is_edit": True, "is_src_file": True, "is_error": True}, + ] if i == 0 else [], + } + for i in range(20) + ] + sess: List[Dict[str, Any]] = [ - # Session 1: Claude using Sonnet - verified, high quality { "session": "s1", "agent_type": "claude", "cwds": ["/test"], - "turns": [ - { - "model": "claude-sonnet-4-6", - "input_tokens": 1000, - "output_tokens": 200, - "cache_read_input_tokens": 800, - "cache_creation_input_tokens": 0, - "ephemeral_5m_input_tokens": 0, - "ephemeral_1h_input_tokens": 0, - "calls": [ - {"name": "Edit", "raw_target": "src/a.py", "is_edit": True, "is_src_file": True}, - {"name": "Bash", "command": "pytest", "is_test_run": True, "is_error": False}, - ], - } - ], + "turns": claude_turns, "events": [], }, - # Session 2: Antigravity using Gemini Flash - unverified, error { "session": "s2", "agent_type": "antigravity", "cwds": ["/test"], - "turns": [ - { - "model": "gemini-3.6-flash", - "input_tokens": 500, - "output_tokens": 100, - "cache_read_input_tokens": 300, - "cache_creation_input_tokens": 0, - "ephemeral_5m_input_tokens": 0, - "ephemeral_1h_input_tokens": 0, - "calls": [ - {"name": "write_to_file", "raw_target": "src/b.py", "is_edit": True, "is_src_file": True, "is_error": True}, - ], - } - ], + "turns": agy_turns, "events": [], }, ] @@ -347,8 +352,36 @@ def test_quality_metrics_by_agent_and_model() -> None: assert "Claude Code" in html or "claude" in html assert "claude-sonnet-4-6" in html assert "gemini-3.6-flash" in html - assert "Engine / model" in html - assert "Engine & model reliability" in html + assert "ENGINE / MODEL" in html or "Engine / model" in html + + +def test_quality_metrics_turn_threshold_filter() -> None: + # Session with only 5 turns should NOT be evaluated in by_agent or by_model + short_sess = [ + { + "session": "short_s1", + "agent_type": "codex", + "cwds": ["/test"], + "turns": [ + { + "model": "gpt-5.3-codex", + "input_tokens": 100, + "output_tokens": 50, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "calls": [], + } + for _ in range(5) # only 5 turns < 20 + ], + "events": [], + } + ] + + qm = quality_metrics(short_sess) + assert "codex" not in qm["by_agent"] + assert len(qm["by_model"]) == 0 def test_quality_metrics_by_task_category() -> None: @@ -492,7 +525,10 @@ def test_quality_metrics_by_task_category() -> None: # Render dashboard html = render(payload) - assert "Capability by coding task domain" in html + assert ( + "CAPABILITY & PERFORMANCE BY CODING TASK DOMAIN" in html + or "Capability by coding task domain" in html + ) assert "UI & Frontend" in html or "UI & Frontend" in html