Skip to content

feat(quality): exclude models and agents with fewer than 20 turns fro… #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
contactacefleetdev merged 1 commit into from
Aug 28, 2026
+89 −40
Merged

feat(quality): exclude models and agents with fewer than 20 turns fro… #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 19 additions & 6 deletions src/ace/sidecar/insights.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2436,12 +2436,15 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]:
}


MIN_QUALITY_EVAL_TURNS = 20


def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]:
"""Calculates unified code quality, verification hygiene, and reliability metrics.

Includes top-line metrics along with breakdowns:
- by_agent: Quality scores partitioned per agent engine (Claude Code, Antigravity, Codex).
- by_model: Quality scores partitioned per LLM model.
- by_agent: Quality scores partitioned per agent engine (Claude Code, Antigravity, Codex) with >= 20 turns.
- by_model: Quality scores partitioned per LLM model with >= 20 turns.
- by_category: Quality and capability metrics partitioned per coding task domain (UI, Backend, Testing, Docs, Research).
"""
overall = _calc_quality_block(sess)
Expand All @@ -2451,23 +2454,27 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]:
overall["by_category"] = {}
return overall

# Group by agent
# Group by agent (exclude agents with < 20 turns)
by_agent: Dict[str, Any] = {}
agent_groups: Dict[str, List[Dict[str, Any]]] = {}
for s in sess:
ak = s.get("agent_type") or AGENT_CLAUDE
agent_groups.setdefault(ak, []).append(s)

for ak, a_sess in agent_groups.items():
a_turns = sum(len(s.get("turns", [])) for s in a_sess)
if a_turns < MIN_QUALITY_EVAL_TURNS:
continue
block = _calc_quality_block(a_sess)
by_agent[ak] = {
"agent": ak,
"label": AGENTS.get(ak, ak.capitalize()),
"sessions": len(a_sess),
"total_turns": a_turns,
**block,
}

# Group by model
# Group by model (exclude models with < 20 turns or synthetic tags)
model_sessions: Dict[str, List[Dict[str, Any]]] = {}
for s in sess:
models_in_s = set(t.get("model") for t in s.get("turns", []) if t.get("model"))
Expand All @@ -2485,11 +2492,15 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]:

by_model: List[Dict[str, Any]] = []
for m_name, m_sess in sorted(model_sessions.items(), key=lambda kv: -len(kv[1])):
m_turns = sum(len(s.get("turns", [])) for s in m_sess)
if m_turns < MIN_QUALITY_EVAL_TURNS or str(m_name).startswith("<"):
continue
block = _calc_quality_block(m_sess)
by_model.append(
{
"model": m_name,
"sessions": len(m_sess),
"total_turns": m_turns,
**block,
}
)
Expand All @@ -2509,7 +2520,8 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]:
best_agent_score = -1
for ak in (AGENT_CLAUDE, AGENT_ANTIGRAVITY, AGENT_CODEX):
sub_ak = [s for s in c_sess if (s.get("agent_type") or AGENT_CLAUDE) == ak]
if sub_ak:
sub_ak_turns = sum(len(s.get("turns", [])) for s in sub_ak)
if sub_ak and sub_ak_turns >= MIN_QUALITY_EVAL_TURNS:
sc = _calc_quality_block(sub_ak)["quality_score"]
if sc > best_agent_score:
best_agent_score = sc
Expand All @@ -2529,7 +2541,8 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]:
proj = [t for t in s.get("turns", []) if t.get("model") == m]
if proj:
sub_m.append({"turns": proj, "events": s.get("events", [])})
if sub_m:
sub_m_turns = sum(len(s.get("turns", [])) for s in sub_m)
if sub_m and sub_m_turns >= MIN_QUALITY_EVAL_TURNS:
sc = _calc_quality_block(sub_m)["quality_score"]
if sc > best_model_score:
best_model_score = sc
Expand Down
104 changes: 70 additions & 34 deletions tests/test_quality_metrics.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -277,48 +277,53 @@ def test_quality_in_payload_and_prometheus() -> None:


def test_quality_metrics_by_agent_and_model() -> None:
# 20 turns per session to satisfy MIN_QUALITY_EVAL_TURNS
claude_turns = [
{
"model": "claude-sonnet-4-6",
"input_tokens": 100,
"output_tokens": 20,
"cache_read_input_tokens": 80,
"cache_creation_input_tokens": 0,
"ephemeral_5m_input_tokens": 0,
"ephemeral_1h_input_tokens": 0,
"calls": [
{"name": "Edit", "raw_target": "src/a.py", "is_edit": True, "is_src_file": True},
{"name": "Bash", "command": "pytest", "is_test_run": True, "is_error": False},
] if i == 0 else [],
}
for i in range(20)
]

agy_turns = [
{
"model": "gemini-3.6-flash",
"input_tokens": 50,
"output_tokens": 10,
"cache_read_input_tokens": 30,
"cache_creation_input_tokens": 0,
"ephemeral_5m_input_tokens": 0,
"ephemeral_1h_input_tokens": 0,
"calls": [
{"name": "write_to_file", "raw_target": "src/b.py", "is_edit": True, "is_src_file": True, "is_error": True},
] if i == 0 else [],
}
for i in range(20)
]

sess: List[Dict[str, Any]] = [
# Session 1: Claude using Sonnet - verified, high quality
{
"session": "s1",
"agent_type": "claude",
"cwds": ["/test"],
"turns": [
{
"model": "claude-sonnet-4-6",
"input_tokens": 1000,
"output_tokens": 200,
"cache_read_input_tokens": 800,
"cache_creation_input_tokens": 0,
"ephemeral_5m_input_tokens": 0,
"ephemeral_1h_input_tokens": 0,
"calls": [
{"name": "Edit", "raw_target": "src/a.py", "is_edit": True, "is_src_file": True},
{"name": "Bash", "command": "pytest", "is_test_run": True, "is_error": False},
],
}
],
"turns": claude_turns,
"events": [],
},
# Session 2: Antigravity using Gemini Flash - unverified, error
{
"session": "s2",
"agent_type": "antigravity",
"cwds": ["/test"],
"turns": [
{
"model": "gemini-3.6-flash",
"input_tokens": 500,
"output_tokens": 100,
"cache_read_input_tokens": 300,
"cache_creation_input_tokens": 0,
"ephemeral_5m_input_tokens": 0,
"ephemeral_1h_input_tokens": 0,
"calls": [
{"name": "write_to_file", "raw_target": "src/b.py", "is_edit": True, "is_src_file": True, "is_error": True},
],
}
],
"turns": agy_turns,
"events": [],
},
]
Expand Down Expand Up @@ -347,8 +352,36 @@ def test_quality_metrics_by_agent_and_model() -> None:
assert "Claude Code" in html or "claude" in html
assert "claude-sonnet-4-6" in html
assert "gemini-3.6-flash" in html
assert "Engine / model" in html
assert "Engine &amp; model reliability" in html
assert "ENGINE / MODEL" in html or "Engine / model" in html


def test_quality_metrics_turn_threshold_filter() -> None:
# Session with only 5 turns should NOT be evaluated in by_agent or by_model
short_sess = [
{
"session": "short_s1",
"agent_type": "codex",
"cwds": ["/test"],
"turns": [
{
"model": "gpt-5.3-codex",
"input_tokens": 100,
"output_tokens": 50,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
"ephemeral_5m_input_tokens": 0,
"ephemeral_1h_input_tokens": 0,
"calls": [],
}
for _ in range(5) # only 5 turns < 20
],
"events": [],
}
]

qm = quality_metrics(short_sess)
assert "codex" not in qm["by_agent"]
assert len(qm["by_model"]) == 0


def test_quality_metrics_by_task_category() -> None:
Expand Down Expand Up @@ -492,7 +525,10 @@ def test_quality_metrics_by_task_category() -> None:

# Render dashboard
html = render(payload)
assert "Capability by coding task domain" in html
assert (
"CAPABILITY &amp; PERFORMANCE BY CODING TASK DOMAIN" in html
or "Capability by coding task domain" in html
)
assert "UI &amp; Frontend" in html or "UI & Frontend" in html


Expand Down
Loading