diff --git a/src/ace/sidecar/dashboard_render.py b/src/ace/sidecar/dashboard_render.py index 7b9eb96..0caf082 100644 --- a/src/ace/sidecar/dashboard_render.py +++ b/src/ace/sidecar/dashboard_render.py @@ -714,6 +714,355 @@ def _activity_svg(daily: List[Dict[str, Any]], commits: bool) -> str: ) +def _quality(qm: Optional[Dict[str, Any]]) -> str: + """§ 02 — Code quality, verification hygiene, and agent execution reliability.""" + if not qm or not qm.get("available"): + return ( + "
~/ace/code_quality" + "NO SESSIONS
" + "
No session data available in this scope to compute code quality metrics. " + "Metrics will populate as coding agent sessions run and edit workspace files.
" + ) + + score = qm.get("quality_score", 100) + grade = qm.get("grade", "A") + c_rate = qm.get("task_completion_rate_pct", 100.0) + v_rate = qm.get("verification_rate_pct", 100.0) + fsr = qm.get("first_pass_success_rate_pct", 100.0) + err_rate = qm.get("tool_error_rate_pct", 0.0) + thrash_cnt = qm.get("thrashed_files_count", 0) + recovery_turns = qm.get("avg_error_recovery_turns", 1.0) + redundant_reads = qm.get("redundant_reads_count", 0) + sessions_edits = qm.get("sessions_with_edits", 0) + sessions_tests = qm.get("sessions_with_tests", 0) + clean_completed = qm.get("clean_completed_sessions", 0) + + turns_task = qm.get("turns_per_completion_avg", 1.0) + time_task_min = qm.get("duration_minutes_per_completion_avg", 0.0) + followup_fixes = qm.get("followup_code_fixes_count", 0) + followup_rate = qm.get("followup_code_fix_rate_pct", 0.0) + comment_ratio = qm.get("comment_to_code_ratio", 0.0) + comment_density = qm.get("comment_density_pct", 0.0) + verbosity_tok = qm.get("verbosity_tokens_per_turn", 0.0) + verbosity_lvl = qm.get("verbosity_level", "Concise") + + score_color = ( + "var(--mint)" + if score >= 80 + else ("var(--gold)" if score >= 60 else "var(--crit)") + ) + c_cls = "" if c_rate >= 80 else ("warn" if c_rate >= 60 else "crit") + v_cls = "" if v_rate >= 75 else ("warn" if v_rate >= 50 else "crit") + fsr_cls = "" if fsr >= 85 else ("warn" if fsr >= 70 else "crit") + thrash_cls = "" if thrash_cnt == 0 else ("warn" if thrash_cnt <= 5 else "crit") + + tiles = [ + _st( + "quality_score", + f"{score}/ 100", + f"Grade {grade}", + delta="COMPOSITE", + title="Weighted reliability index across task completion (35%), verification hygiene (30%), first-pass tool success (20%), and edit stability (15%).", + ), + _st( + "task_completion", + f"{c_rate}%", + f"{clean_completed} verified sessions", + delta="TASK RESOLUTION", + dcls=c_cls, + title="Percentage of sessions that resolved cleanly without trailing tool errors or unverified changes.", + ), + _st( + "turns_per_task", + f"{turns_task} turns", + "avg turns / completion", + delta="CONVERSATION EFFICIENCY", + title="Average number of conversation turns required to achieve a verified clean task completion.", + ), + _st( + "time_per_task", + f"{time_task_min} min", + "avg elapsed / completion", + delta="DELIVERY SPEED", + title="Average wall-clock duration in minutes from session start to verified resolution.", + ), + _st( + "first_pass_success", + f"{fsr}%", + f"{err_rate}% error rate", + delta="TOOL RELIABILITY", + dcls=fsr_cls, + title="Share of tool executions that succeeded on their first attempt without returning execution errors or non-zero exit codes.", + ), + _st( + "verification_rate", + f"{v_rate}%", + f"{sessions_tests} of {sessions_edits} edit sessions", + delta="TEST HYGIENE", + dcls=v_cls, + title="Percentage of sessions containing file modifications that executed an automated test runner or linter (pytest, npm test, ruff, etc.).", + ), + _st( + "followup_fixes", + f"{followup_fixes}", + f"{followup_rate}% rework rate", + delta="FOLLOW-UP FIXES", + title="Follow-up code modifications and bugfixes applied to the same files in subsequent turns.", + ), + _st( + "comment_ratio", + f"{comment_ratio}x", + f"{comment_density}% comment density", + delta="CODE COMMENT DENSITY", + title="Ratio of inline comments to executable code lines in modifications.", + ), + _st( + "verbosity", + f"{verbosity_tok} tok", + f"{verbosity_lvl} explanation", + delta="VERBOSITY LEVEL", + title="Average generated output tokens per conversational turn.", + ), + _st( + "edit_thrash_files", + f"{thrash_cnt}", + f"{qm.get('total_edits', 0)} total file edits", + delta="REWORK CHURN", + dcls=thrash_cls, + title="Files edited 3 or more times within the same session, indicating thrashing or lack of convergence.", + ), + ] + + thrashed_files_list = qm.get("thrashed_files_list") or [] + thrash_html = "" + if thrashed_files_list: + thrashed_items = "".join( + f"
  • {escape(_mask_home(f))}
    • " + for f in thrashed_files_list + ) + thrash_html = ( + f"
    " + f"⚠️ Repeatedly Modified Files (Thrashing Detected):" + f"" + f"
    " + ) + + breakdown_rows = [] + # Agent breakdown rows + by_agent = qm.get("by_agent") or {} + for ak, a_info in by_agent.items(): + a_score = a_info.get("quality_score", 100) + a_grade = a_info.get("grade", "A") + a_comp = a_info.get("task_completion_rate_pct", 100.0) + a_turns = a_info.get("turns_per_completion_avg", 1.0) + a_time = a_info.get("duration_minutes_per_completion_avg", 0.0) + a_v_rate = a_info.get("verification_rate_pct", 100.0) + a_fsr = a_info.get("first_pass_success_rate_pct", 100.0) + a_fixes = a_info.get("followup_code_fixes_count", 0) + a_comm = a_info.get("comment_to_code_ratio", 0.0) + a_verb = a_info.get("verbosity_tokens_per_turn", 0.0) + a_sess = a_info.get("sessions", 0) + badge_style = ( + "color:var(--mint);border-color:#1d3b2e;background:#0F231A" + if ak == "antigravity" + else ( + "color:var(--blue);border-color:#1e355b;background:#0d1c33" + if ak == "claude" + else "color:var(--purple, #c084fc);border-color:#3b1e5b;background:#1a0d33" + ) + ) + score_badge = ( + "color:var(--mint);border-color:#1d3b2e;background:#0F231A" + if a_score >= 80 + else ( + "color:var(--gold);border-color:#3d3014;background:#241D0E" + if a_score >= 60 + else "color:var(--crit);border-color:#4a1e17;background:#2a110e" + ) + ) + breakdown_rows.append( + f"" + f"{escape(a_info.get('label', ak))}" + f"{a_score} ({a_grade})" + f"{a_comp}%" + f"{a_turns}" + f"{a_time}m" + f"{a_v_rate}%" + f"{a_fsr}%" + f"{a_fixes}" + f"{a_comm}x" + f"{a_verb} tok" + f"{a_sess}" + f"" + ) + + # Model breakdown rows + by_model = qm.get("by_model") or [] + for m_info in by_model: + m_name = m_info.get("model", "unknown") + m_score = m_info.get("quality_score", 100) + m_grade = m_info.get("grade", "A") + m_comp = m_info.get("task_completion_rate_pct", 100.0) + m_turns = m_info.get("turns_per_completion_avg", 1.0) + m_time = m_info.get("duration_minutes_per_completion_avg", 0.0) + m_v_rate = m_info.get("verification_rate_pct", 100.0) + m_fsr = m_info.get("first_pass_success_rate_pct", 100.0) + m_fixes = m_info.get("followup_code_fixes_count", 0) + m_comm = m_info.get("comment_to_code_ratio", 0.0) + m_verb = m_info.get("verbosity_tokens_per_turn", 0.0) + m_sess = m_info.get("sessions", 0) + score_badge = ( + "color:var(--mint);border-color:#1d3b2e;background:#0F231A" + if m_score >= 80 + else ( + "color:var(--gold);border-color:#3d3014;background:#241D0E" + if m_score >= 60 + else "color:var(--crit);border-color:#4a1e17;background:#2a110e" + ) + ) + breakdown_rows.append( + f"" + f"{escape(m_name)}" + f"{m_score} ({m_grade})" + f"{m_comp}%" + f"{m_turns}" + f"{m_time}m" + f"{m_v_rate}%" + f"{m_fsr}%" + f"{m_fixes}" + f"{m_comm}x" + f"{m_verb} tok" + f"{m_sess}" + f"" + ) + + matrix_table = "" + if breakdown_rows: + matrix_table = ( + f"
    " + f"
    " + f"ENGINE & MODEL RELIABILITY COMPARISON" + f"
    " + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"{''.join(breakdown_rows)}" + f"
    ENGINE / MODELSCORECOMPLETIONTURNS/TASKTIME/TASKVERIFICATIONFIRST-PASS SUCCESSFOLLOW-UP FIXESCOMMENT RATIOVERBOSITYSESSIONS
    " + f"
    " + ) + + category_table = "" + by_category = qm.get("by_category") or {} + category_rows = [] + for ck, c_info in by_category.items(): + c_sessions = c_info.get("sessions", 0) + if c_sessions == 0: + continue + c_label = c_info.get("label", ck.capitalize()) + c_icon = c_info.get("icon", "💻") + c_desc = c_info.get("desc", "") + c_score = c_info.get("quality_score", 100) + c_grade = c_info.get("grade", "A") + c_comp = c_info.get("task_completion_rate_pct", 100.0) + c_turns = c_info.get("turns_per_completion_avg", 1.0) + c_time = c_info.get("duration_minutes_per_completion_avg", 0.0) + c_verif = c_info.get("verification_rate_pct", 100.0) + c_fsr = c_info.get("first_pass_success_rate_pct", 100.0) + c_fixes = c_info.get("followup_code_fixes_count", 0) + c_comm = c_info.get("comment_to_code_ratio", 0.0) + c_share = c_info.get("share_pct", 0.0) + best_agent = c_info.get("best_agent", "—") + best_model = c_info.get("best_model", "—") + + score_badge = ( + "color:var(--mint);border-color:#1d3b2e;background:#0F231A" + if c_score >= 80 + else ( + "color:var(--gold);border-color:#3d3014;background:#241D0E" + if c_score >= 60 + else "color:var(--crit);border-color:#4a1e17;background:#2a110e" + ) + ) + category_rows.append( + f"" + f"" + f"
    " + f"{c_icon}{escape(c_label)}" + f"({c_share}%)" + f"
    " + f"
    {escape(c_desc)}
    " + f"" + f"{c_score} ({c_grade})" + f"{c_comp}%" + f"{c_turns}" + f"{c_time}m" + f"{c_verif}%" + f"{c_fsr}%" + f"{c_fixes}" + f"{c_comm}x" + f"{c_sessions}" + f"" + f"
    {escape(best_agent)}
    " + f"
    {escape(best_model)}
    " + f"" + f"" + ) + + if category_rows: + category_table = ( + f"
    " + f"
    " + f"CAPABILITY & PERFORMANCE BY CODING TASK DOMAIN" + f"
    " + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"{''.join(category_rows)}" + f"
    TASK CATEGORYSCORECOMPLETIONTURNS/TASKTIME/TASKVERIFICATIONFIRST-PASS SUCCESSFOLLOW-UP FIXESCOMMENT RATIOSESSIONSBEST FIT ENGINE / MODEL
    " + f"
    " + ) + + return ( + f"
    {''.join(tiles)}
    " + f"
    " + f"
    ~/ace/quality_breakdownLOCAL VERIFIED
    " + f"
    " + f"
    " + f"
    {sessions_tests} test-verified sessions
    " + f"
    {sessions_edits} editing sessions
    " + f"
    {qm.get('total_tool_calls', 0)} total tool executions
    " + f"
    {redundant_reads} redundant duplicate file reads
    " + f"
    " + f"{thrash_html}" + f"{matrix_table}" + f"{category_table}" + f"
    Measures how safely and stably coding agents operate in your repository. Correlates cost against first-pass tool correctness, test diligence, and domain task fit.
    " + f"
    " + ) + + def _fleet(f: Optional[Dict[str, Any]]) -> str: """§ 01 — the eleven fleet metrics from docs/22 §0, on this machine's transcripts. @@ -1597,6 +1946,7 @@ def _prometheus_section(d: Dict[str, Any]) -> str: # the same reasoning that keeps _sec's id keyed off the section number. _NAV = ( ("◫", "Overview", "01"), + ("🎯", "Code Quality", "02"), ("⇄", "Strategies", "04"), ("✦", "Recommendations", "06"), ("⚡", "Workflow Skills", "07"), @@ -1822,7 +2172,7 @@ def _lever_note(d: Dict[str, Any]) -> str: def _rail(d: Dict[str, Any]) -> str: - live = d["live"] + live = d.get("live") or {"turns": 0} # Each entry jumps to a section already on the page -- one document, not five views. # Anchors rather than divs: clickable without JS. nav = "".join( @@ -1981,11 +2331,23 @@ def render(d: Dict[str, Any]) -> str: # on to "what it cost" and "what to do about it". b.append(_fleet(d.get("fleet"))) - peak = h.get("peak_context") or 0 - # § 02 — spend + # § 02 — code quality & reliability b.append( _sec( "02", + "CODE QUALITY & RELIABILITY", + "Agent execution stability & test hygiene.", + "Verification rate, rework thrash, and error recovery.", + "LOCAL", + ) + ) + b.append(_quality(d.get("quality"))) + + peak = h.get("peak_context") or 0 + # § 03 — spend + b.append( + _sec( + "03", "SPEND", "Where the money goes.", "List price on your transcripts.", diff --git a/src/ace/sidecar/insights.py b/src/ace/sidecar/insights.py index 96e5063..36f53c0 100644 --- a/src/ace/sidecar/insights.py +++ b/src/ace/sidecar/insights.py @@ -148,6 +148,256 @@ def _sig(name: str, tool_input: Dict[str, Any]) -> str: return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16] +_TEST_CMD_RE = re.compile( + r"\b(pytest|npm\s+(?:run\s+)?test|vitest|jest|cargo\s+test|go\s+test|dotnet\s+test|ctest|ruff|eslint|mypy|flake8|pylint|black\s+--check|tsc\s+--noEmit|bundle\s+exec\s+rspec)\b", + re.IGNORECASE, +) + +_ERR_MSG_RE = re.compile( + r"(encountered error in tool execution|the command exited with code [1-9]|exit code [1-9]|operation not permitted|command failed|fatal:|traceback \(most recent call last\)|syntaxerror|typeerror|keyerror|assertionerror|modulenotfounderror|permission denied|no such file or directory)", + re.IGNORECASE, +) + +_TEST_FILE_RE = re.compile( + r"(^|[/\\])(tests?|spec|__tests__)[/\\]|(\.|_)(test|spec)\.[a-zA-Z0-9]+$", + re.IGNORECASE, +) + +_SOURCE_FILE_EXTS = ( + ".py", + ".js", + ".jsx", + ".ts", + ".tsx", + ".go", + ".rs", + ".java", + ".c", + ".cpp", + ".h", + ".hpp", + ".cs", + ".rb", + ".php", + ".swift", + ".kt", + ".scala", + ".sh", + ".html", + ".css", + ".vue", +) + +TASK_CAT_UI = "ui" +TASK_CAT_BACKEND = "backend" +TASK_CAT_TESTING = "testing" +TASK_CAT_DOCS = "docs" +TASK_CAT_RESEARCH = "research" + +TASK_CATEGORIES: Dict[str, Dict[str, str]] = { + TASK_CAT_UI: { + "label": "UI & Frontend", + "icon": "🎨", + "desc": "User interfaces, web layouts, components, templates, and styling.", + }, + TASK_CAT_BACKEND: { + "label": "Backend & Systems", + "icon": "⚙️", + "desc": "Server logic, database models, APIs, pipelines, and algorithms.", + }, + TASK_CAT_TESTING: { + "label": "Testing & QA", + "icon": "🧪", + "desc": "Unit/integration tests, test harnesses, assertions, and linter auto-fixes.", + }, + TASK_CAT_DOCS: { + "label": "Docs & Specs", + "icon": "📝", + "desc": "Architecture documentation, implementation plans, and walkthroughs.", + }, + TASK_CAT_RESEARCH: { + "label": "Codebase Research", + "icon": "🔍", + "desc": "Codebase navigation, symbol search, architecture analysis, and comprehension.", + }, +} + +_UI_EXTS = ( + ".tsx", + ".jsx", + ".vue", + ".svelte", + ".css", + ".scss", + ".sass", + ".html", + ".svg", +) +_DOC_EXTS = (".md", ".mdx", ".rst", ".txt", ".adoc") + + +def _classify_session_task_category(session: Dict[str, Any]) -> str: + """Classifies a coding session into a primary task domain based on tool calls and target files.""" + turns = session.get("turns") or [] + edits: List[str] = [] + has_test_runs = False + + for t in turns: + for c in t.get("calls") or []: + if c.get("is_edit"): + tgt = str(c.get("raw_target") or c.get("target") or "").lower() + edits.append(tgt) + if c.get("is_test_run"): + has_test_runs = True + + if not edits: + return TASK_CAT_RESEARCH + + ui_count = sum( + 1 + for e in edits + if any(e.endswith(ext) for ext in _UI_EXTS) + or any( + k in e + for k in ( + "/ui/", + "/frontend/", + "/components/", + "/views/", + "/styles/", + "/web/", + "/templates/", + "dashboard_render", + ) + ) + ) + test_count = sum(1 for e in edits if _TEST_FILE_RE.search(e)) + doc_count = sum( + 1 + for e in edits + if any(e.endswith(ext) for ext in _DOC_EXTS) + or any(k in e for k in ("doc", "readme", "walkthrough", "plan")) + ) + backend_count = len(edits) - ui_count - test_count - doc_count + + counts = { + TASK_CAT_UI: ui_count, + TASK_CAT_TESTING: test_count, + TASK_CAT_DOCS: doc_count, + TASK_CAT_BACKEND: max(0, backend_count), + } + top_cat = max(counts.items(), key=lambda kv: kv[1]) + if top_cat[1] > 0: + return top_cat[0] + return TASK_CAT_BACKEND + + +_COMMENT_LINE_RE = re.compile(r"^\s*(#|//|/\*|\*|\*/|\"\"\"|\'\'\'|--|