From ff863a470e640128318b4df7c863624dda76800a Mon Sep 17 00:00:00 2001 From: ACE Engineering Date: Thu, 27 Aug 2026 19:19:27 -0700 Subject: [PATCH 1/8] feat(quality): add code quality, verification hygiene, and reliability metrics --- src/ace/gateway/local_store.py | 173 ++++++- src/ace/gateway/messages.py | 120 ++++- src/ace/sidecar/app.py | 21 + src/ace/sidecar/dashboard_render.py | 129 ++++- src/ace/sidecar/insights.py | 491 ++++++++++++++++++- src/ace/sidecar/levers/__init__.py | 115 +++++ src/ace/sidecar/levers/counter.py | 240 +++++++++ src/ace/sidecar/levers/ledger.py | 450 +++++++++++++++++ src/ace/sidecar/levers/protocol.py | 205 ++++++++ src/ace/sidecar/levers/rail.py | 276 +++++++++++ src/ace/sidecar/levers/registry.py | 232 +++++++++ src/ace/sidecar/levers/shadow.py | 736 ++++++++++++++++++++++++++++ src/ace/sidecar/levers/types.py | 278 +++++++++++ src/ace/sidecar/strategies.py | 68 ++- tests/test_lever_ledger.py | 160 ++++++ tests/test_lever_shadow.py | 470 ++++++++++++++++++ tests/test_levers.py | 205 ++++++++ tests/test_quality_metrics.py | 276 +++++++++++ 18 files changed, 4626 insertions(+), 19 deletions(-) create mode 100644 src/ace/sidecar/levers/__init__.py create mode 100644 src/ace/sidecar/levers/counter.py create mode 100644 src/ace/sidecar/levers/ledger.py create mode 100644 src/ace/sidecar/levers/protocol.py create mode 100644 src/ace/sidecar/levers/rail.py create mode 100644 src/ace/sidecar/levers/registry.py create mode 100644 src/ace/sidecar/levers/shadow.py create mode 100644 src/ace/sidecar/levers/types.py create mode 100644 tests/test_lever_ledger.py create mode 100644 tests/test_lever_shadow.py create mode 100644 tests/test_levers.py create mode 100644 tests/test_quality_metrics.py diff --git a/src/ace/gateway/local_store.py b/src/ace/gateway/local_store.py index d088812..808d344 100644 --- a/src/ace/gateway/local_store.py +++ b/src/ace/gateway/local_store.py @@ -23,12 +23,13 @@ from __future__ import annotations +import json import logging import os import sqlite3 import threading import time -from typing import Any, Dict, List, Optional +from typing import Any, Dict, Iterable, List, Mapping, Optional log = logging.getLogger("ace.gateway.local_store") @@ -53,6 +54,51 @@ ); CREATE INDEX IF NOT EXISTS idx_turns_ts ON turns(ts); CREATE INDEX IF NOT EXISTS idx_turns_session ON turns(session_id); + +-- One row per lever per proxied turn: what an installed lever, run for real against the +-- actual request body, measured. Sibling of `turns` rather than columns on it, because a +-- turn has N of these (one per enabled lever) and because every row here is a +-- COUNTERFACTUAL -- a prompt that was never sent -- while every row in `turns` is what the +-- provider actually billed. Merging the two would put a real charge and a hypothetical +-- saving in one record with nothing to tell them apart. +-- +-- Why this table has to exist at all: a measured result is produced once, in a background +-- task, moments after a response is served. Without a row here it is logged and lost, and +-- the dashboard is back to simulating headroom over transcripts. +CREATE TABLE IF NOT EXISTS lever_turns ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts REAL NOT NULL, + request_id TEXT, + session_id TEXT, + lever TEXT NOT NULL, + mode TEXT, + model TEXT, + -- Both sides of the counterfactual, kept so the delta can be re-derived rather than + -- trusted. Counted the same way through the provider's own counter; only their + -- difference is exact. + baseline_tokens INTEGER DEFAULT 0, + counterfactual_tokens INTEGER DEFAULT 0, + removed_tokens INTEGER DEFAULT 0, + -- How the removed tokens were allocated against this turn's real usage buckets. Kept + -- because it is the entire pricing argument: the same token delta is worth ~12x more + -- coming out of a cache write than out of a cache read. + from_cache_write INTEGER DEFAULT 0, + from_input INTEGER DEFAULT 0, + from_cache_read INTEGER DEFAULT 0, + usd REAL DEFAULT 0.0, + -- 0 means the model had no catalog entry: tokens are real, dollars are absent. Must + -- never render as $0.00 of saving -- a silent zero looks like a measured result. + priced INTEGER DEFAULT 1, + -- 0 where an edit touched already-cached history, whose cache-write penalty lands on + -- the NEXT turn and is therefore not netted into `usd`. + prefix_safe INTEGER DEFAULT 1, + edits_applied INTEGER DEFAULT 0, + -- Lever-authored counters, numeric values only -- see LocalStore._numeric_diagnostics. + diagnostics TEXT, + note TEXT +); +CREATE INDEX IF NOT EXISTS idx_lever_turns_ts ON lever_turns(ts); +CREATE INDEX IF NOT EXISTS idx_lever_turns_lever ON lever_turns(lever); """ @@ -106,8 +152,133 @@ def record_log(self, row: Any) -> None: except Exception: # pragma: no cover - defensive log.debug("[local_store] failed to record a turn", exc_info=True) + @staticmethod + def _numeric_diagnostics(diagnostics: Any) -> Optional[str]: + """A lever's diagnostics, numbers only, as JSON — or ``None``. + + This store's one invariant is that it holds numbers and never text from a developer's + session. Diagnostics are authored by a third-party lever package, so they are the one + field here that could carry arbitrary strings — a lever logging the command it + matched would quietly put a shell line into the database. Numeric values survive, + everything else is dropped, and the invariant stays a property of the code rather + than a promise about third-party behaviour. + """ + if not isinstance(diagnostics, Mapping): + return None + clean = { + str(k): v + for k, v in diagnostics.items() + if isinstance(v, (int, float)) and not isinstance(v, bool) + } + return json.dumps(clean, sort_keys=True) if clean else None + + def record_lever_turns(self, rows: Iterable[Any]) -> int: + """Persist measured lever results for one turn. Never raises. + + Takes the whole batch for a turn in one transaction: the rows describe a single + request, and half of them landing would leave the rail ranking levers against + different denominators. + """ + prepared = [] + for m in rows or (): + try: + edits = getattr(m, "edits", ()) or () + prepared.append(( + getattr(m, "ts", None) or time.time(), + getattr(m, "request_id", ""), + getattr(m, "session_id", None), + getattr(m, "lever", ""), + getattr(m, "mode", ""), + getattr(m, "model", ""), + int(getattr(m, "baseline_tokens", 0)), + int(getattr(m, "counterfactual_tokens", 0)), + int(getattr(m, "removed_tokens", 0)), + int(getattr(m, "from_cache_write", 0)), + int(getattr(m, "from_input", 0)), + int(getattr(m, "from_cache_read", 0)), + float(getattr(m, "usd", 0.0)), + 1 if getattr(m, "priced", True) else 0, + 0 if any(e.applied and not e.prefix_safe for e in edits) else 1, + sum(1 for e in edits if e.applied), + self._numeric_diagnostics(getattr(m, "diagnostics", None)), + getattr(m, "note", ""), + )) + except Exception: + log.debug("[local_store] skipped a malformed lever row", exc_info=True) + if not prepared: + return 0 + try: + with self._lock: + self._db.executemany( + "INSERT INTO lever_turns (ts, request_id, session_id, lever, mode, model," + " baseline_tokens, counterfactual_tokens, removed_tokens," + " from_cache_write, from_input, from_cache_read, usd, priced," + " prefix_safe, edits_applied, diagnostics, note)" + " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + prepared, + ) + self._db.commit() + return len(prepared) + except Exception: # pragma: no cover - defensive + log.debug("[local_store] failed to record lever turns", exc_info=True) + return 0 + # -- read -------------------------------------------------------------------------- + def lever_summary(self, since: Optional[float] = None) -> Dict[str, Any]: + """Measured lever results, aggregated per lever. What the rail's live half renders. + + Aggregated **per lever and never totalled**, the same discipline + ``levers.ledger.LedgerReport`` documents: two levers can claim the same bytes, so + adding their savings produces a number larger than anything they could jointly + deliver. Ranking answers the question actually being asked. + + Only ``priced`` rows contribute dollars. Unpriced rows still contribute their token + counts and are surfaced separately — a model with no catalog entry saved real tokens, + and rendering that as $0.00 would read as "this lever does nothing". + """ + where, args = ("WHERE ts >= ?", (since,)) if since else ("", ()) + with self._lock: + cur = self._db.execute( + f"""SELECT lever, + COUNT(*) AS turns, + COALESCE(SUM(removed_tokens), 0) AS removed_tokens, + COALESCE(SUM(CASE WHEN priced=1 THEN usd END), 0.0) AS usd, + COALESCE(SUM(from_cache_write), 0) AS from_cache_write, + COALESCE(SUM(from_input), 0) AS from_input, + COALESCE(SUM(from_cache_read), 0) AS from_cache_read, + SUM(CASE WHEN priced=0 THEN 1 ELSE 0 END) AS unpriced_turns, + SUM(CASE WHEN prefix_safe=0 THEN 1 ELSE 0 END) AS unsafe_turns, + COALESCE(SUM(edits_applied), 0) AS edits_applied, + MAX(ts) AS last_ts + FROM lever_turns {where} + GROUP BY lever + ORDER BY usd DESC""", + args, + ) + cols = [c[0] for c in cur.description] + by_lever = [dict(zip(cols, r)) for r in cur.fetchall()] + cur = self._db.execute( + f"SELECT COUNT(*), COUNT(DISTINCT request_id) FROM lever_turns {where}", args + ) + n_rows, n_turns = cur.fetchone() + return { + "by_lever": by_lever, + "rows": n_rows, + "turns_observed": n_turns, + # Deliberately absent: a `total_usd`. See the docstring. + } + + def recent_lever_turns(self, limit: int = 50) -> List[Dict[str, Any]]: + with self._lock: + cur = self._db.execute( + "SELECT ts, lever, mode, model, removed_tokens, usd, priced, prefix_safe," + " edits_applied, note FROM lever_turns ORDER BY ts DESC LIMIT ?", + (limit,), + ) + cols = [c[0] for c in cur.description] + return [dict(zip(cols, r)) for r in cur.fetchall()] + def summary(self, since: Optional[float] = None) -> Dict[str, Any]: """Aggregates for the dashboard.""" where, args = ("WHERE ts >= ?", (since,)) if since else ("", ()) diff --git a/src/ace/gateway/messages.py b/src/ace/gateway/messages.py index 5daac83..6d7a0ff 100644 --- a/src/ace/gateway/messages.py +++ b/src/ace/gateway/messages.py @@ -538,6 +538,46 @@ async def _body() -> AsyncIterator[bytes]: ) +class _LocalRequestLog: + """The telemetry row shape, for a sidecar that has no telemetry package. + + ``ace.observability.telemetry.RequestLog`` is the cloud gateway's row and does not exist + in this distribution — the sidecar was extracted from that tree without it. The import + was unconditional, it raised ``ModuleNotFoundError`` on **every** turn, and the caller + catches ``Exception`` around accounting so that nothing must ever cost a developer their + response. The result was silent: ``~/.ace/telemetry.db`` stayed empty, ``store.summary()`` + returned zeros, and the dashboard's live panel reported no spend on a sidecar that was + relaying traffic correctly the whole time. + + A permissive attribute bag rather than a fixed dataclass, because the consumer + (``LocalStore.record_log``) reads by ``getattr`` with defaults, and pinning a field list + here would create a second definition to keep in step with the cloud one. + """ + + __slots__ = ("__dict__",) + + def __init__(self, **fields: Any) -> None: + self.__dict__.update(fields) + + def __repr__(self) -> str: # pragma: no cover - debugging aid + return f"_LocalRequestLog({self.__dict__!r})" + + +def _request_log_class(): + """The cloud gateway's ``RequestLog`` when this tree has one, else the local stand-in. + + Resolved per call and not cached: the import is cheap once Python has it in + ``sys.modules``, and caching a negative result would defeat a deployment that adds the + telemetry package later. + """ + try: + from ace.observability.telemetry import RequestLog + + return RequestLog + except Exception: + return _LocalRequestLog + + def usage_to_request_log( usage: StreamUsage, *, @@ -557,7 +597,7 @@ def usage_to_request_log( cache, not ACE's semantic cache (``cache_hit`` / ``cache_served``, left False here — no ACE lever ran on this path in Phase 0). See the RequestLog field comments. """ - from ace.observability.telemetry import RequestLog + RequestLog = _request_log_class() cost = usage.cost() write_5m, write_1h = usage.split_cache_writes() @@ -597,6 +637,28 @@ def usage_to_request_log( ) +def _levers_usage(usage: StreamUsage): + """Project this route's usage record onto the provider-neutral one levers read. + + A deliberate narrowing, not a copy. ``levers.types.Usage`` carries a TTL *breakdown* + rather than Anthropic's ``ephemeral_5m``/``ephemeral_1h`` field names, because the + cache-write premium is a provider property and a lever tuned against one provider's cache + economics gives wrong answers on another. Doing the translation here keeps the wire + vocabulary on this side of the seam. + """ + from ace.sidecar.levers.types import Usage + + write_5m, write_1h = usage.split_cache_writes() + by_ttl = {k: v for k, v in (("5m", write_5m), ("1h", write_1h)) if v} + return Usage( + input_tokens=usage.input_tokens or 0, + output_tokens=usage.output_tokens or 0, + cache_read_tokens=usage.cache_read_input_tokens or 0, + cache_write_tokens=write_5m + write_1h, + cache_write_by_ttl=by_ttl, + ) + + def install_messages_route( app, *, @@ -607,16 +669,32 @@ def install_messages_route( capture=None, auth_config: Optional["AuthConfig"] = None, byok=None, + shadow=None, ) -> None: """Mount ``POST /v1/messages`` on ``app``. Self-contained on purpose: it takes an app and a config rather than threading through - ``proxy.create_app``'s ~40-parameter factory. Phase 0 runs no levers, so it needs none - of that wiring — and keeping the seam this thin is what lets the P0-5 local sidecar - mount this route alone, without the cloud gateway's cache/router/telemetry stack. + ``proxy.create_app``'s ~40-parameter factory — keeping the seam this thin is what lets + the P0-5 local sidecar mount this route alone, without the cloud gateway's + cache/router/telemetry stack. ``client`` injects an ``httpx.AsyncClient`` so the P0-4 conformance suite can drive this exact production branch through ``MockTransport`` with no live call. + + ``shadow`` is an optional :class:`ace.sidecar.levers.shadow.ShadowRunner`. It is the one + place this module does anything a Phase 0 relay did not, and it was designed to be + unable to violate the fidelity invariant: + + * it never sees ``raw`` — only ``parsed``, the throwaway copy this route already makes to + decide streaming and model, so there is no object shared with what goes upstream; + * it runs **after** the response has been served, on a worker thread, so a counting round + trip cannot land in the developer's turn latency; + * it is skipped entirely — one cached entry-point lookup — when no lever package is + installed, which is the ordinary state. + + It also supplies the credential problem's only solution: under ``no_key: true`` the + relayed token is the sole credential that can reach the counting endpoint, and it exists + only for the life of this request. """ cfg = config or MessagesConfig.from_env() auth_cfg = auth_config or AuthConfig.from_env() @@ -716,6 +794,40 @@ def _sink(usage: StreamUsage) -> None: log.debug("[messages] accountant.record_log failed", exc_info=True) if on_usage is not None: on_usage(usage) + _shadow(usage) + + def _shadow(usage: StreamUsage) -> None: + """Hand this turn to the levers, detached. Never touches the served response. + + Ordered last in `_sink` on purpose: accounting is the thing that must not be lost, + and a lever package is third-party code. Anything that goes wrong past this point + costs a measurement, never a turn. + """ + if shadow is None or not shadow.enabled: + return + try: + from ace.sidecar.levers.counter import resolve_counter + + # The in-flight credential, adopted once. `set_counter` keeps the first one + # for the life of the process so a refusal is remembered instead of re-asked + # on every turn. + if shadow.counter is None and api_key: + counter, _ = resolve_counter(api_key, auth.scheme) + shadow.set_counter(counter) + + import asyncio + + asyncio.get_running_loop().create_task( + shadow.observe_async( + parsed, + _levers_usage(usage), + model=usage.model or parsed.get("model") or "", + request_id=req_id, + session_id=session_id, + ) + ) + except Exception: # pragma: no cover - a shadow run never surfaces + log.debug("[messages] shadow lever run could not start", exc_info=True) url = cfg.base_url.rstrip("/") + MESSAGES_PATH diff --git a/src/ace/sidecar/app.py b/src/ace/sidecar/app.py index 96cf808..b3b8b79 100644 --- a/src/ace/sidecar/app.py +++ b/src/ace/sidecar/app.py @@ -75,6 +75,26 @@ def build_sidecar_app( mode=MODE_LOOPBACK, local_api_key=api_key or auth_env.local_api_key ) + # The measured half of the lever rail. Constructed unconditionally and cheap when nothing + # is installed — `ShadowRunner.enabled` is one cached entry-point lookup — so the + # open-source sidecar on its own pays nothing for a feature it does not have. + # + # Its sink is the telemetry store, which is the whole point of item 4: a lever result is + # produced once, in a background task moments after a response is served, and without a + # row in `lever_turns` it is logged and lost. + shadow = None + try: + from ace.sidecar.levers.shadow import ShadowRunner + + sink = ( + accountant.record_lever_turns + if hasattr(accountant, "record_lever_turns") + else None + ) + shadow = ShadowRunner(sink=sink) + except Exception: # pragma: no cover - levers never block the sidecar starting + log.debug("[sidecar] lever shadow runner unavailable", exc_info=True) + install_messages_route( app, config=cfg, @@ -82,6 +102,7 @@ def build_sidecar_app( accountant=accountant, capture=capture, client=client, + shadow=shadow, ) @app.get("/dashboard", response_class=HTMLResponse) diff --git a/src/ace/sidecar/dashboard_render.py b/src/ace/sidecar/dashboard_render.py index 7b9eb96..2595a12 100644 --- a/src/ace/sidecar/dashboard_render.py +++ b/src/ace/sidecar/dashboard_render.py @@ -714,6 +714,116 @@ def _activity_svg(daily: List[Dict[str, Any]], commits: bool) -> str: ) +def _quality(qm: Optional[Dict[str, Any]]) -> str: + """§ 02 — Code quality, verification hygiene, and agent execution reliability.""" + if not qm or not qm.get("available"): + return ( + "
~/ace/code_quality" + "NO SESSIONS
" + "
No session data available in this scope to compute code quality metrics. " + "Metrics will populate as coding agent sessions run and edit workspace files.
" + ) + + score = qm.get("quality_score", 100) + grade = qm.get("grade", "A") + v_rate = qm.get("verification_rate_pct", 100.0) + fsr = qm.get("first_pass_success_rate_pct", 100.0) + err_rate = qm.get("tool_error_rate_pct", 0.0) + thrash_cnt = qm.get("thrashed_files_count", 0) + recovery_turns = qm.get("avg_error_recovery_turns", 1.0) + redundant_reads = qm.get("redundant_reads_count", 0) + test_code_ratio = qm.get("test_to_code_ratio", 1.0) + sessions_edits = qm.get("sessions_with_edits", 0) + sessions_tests = qm.get("sessions_with_tests", 0) + + score_color = ( + "var(--mint)" + if score >= 80 + else ("var(--gold)" if score >= 60 else "var(--crit)") + ) + v_cls = "" if v_rate >= 75 else ("warn" if v_rate >= 50 else "crit") + fsr_cls = "" if fsr >= 85 else ("warn" if fsr >= 70 else "crit") + thrash_cls = "" if thrash_cnt == 0 else ("warn" if thrash_cnt <= 2 else "crit") + + tiles = [ + _st( + "quality_score", + f"{score}/ 100", + f"Grade {grade}", + delta="COMPOSITE", + title="Weighted reliability index across verification hygiene (35%), first-pass tool success (35%), edit stability (15%), and test balance (15%).", + ), + _st( + "verification_rate", + f"{v_rate}%", + f"{sessions_tests} of {sessions_edits} edit sessions", + delta="TEST HYGIENE", + dcls=v_cls, + title="Percentage of sessions containing file modifications that executed an automated test runner or linter (pytest, npm test, ruff, etc.).", + ), + _st( + "first_pass_success", + f"{fsr}%", + f"{err_rate}% error rate", + delta="TOOL RELIABILITY", + dcls=fsr_cls, + title="Share of tool executions that succeeded on their first attempt without returning execution errors or non-zero exit codes.", + ), + _st( + "edit_thrash_files", + f"{thrash_cnt}", + f"{qm.get('total_edits', 0)} total file edits", + delta="REWORK CHURN", + dcls=thrash_cls, + title="Files edited 3 or more times within the same session, indicating thrashing or lack of convergence.", + ), + _st( + "healing_latency", + f"{recovery_turns} turns", + "avg turns to recover", + delta="ERROR HEALING", + title="Average number of conversation turns required for the agent to resolve a failed tool execution and resume forward progress.", + ), + _st( + "context_waste", + f"{redundant_reads} reads", + f"test/code ratio: {test_code_ratio}x", + delta="REDUNDANCY", + title="Consecutive duplicate reads of identical files without intervening edits.", + ), + ] + + thrashed_files_list = qm.get("thrashed_files_list") or [] + thrash_html = "" + if thrashed_files_list: + thrashed_items = "".join( + f"
  • {escape(_mask_home(f))}
    • " + for f in thrashed_files_list + ) + thrash_html = ( + f"
    " + f"⚠️ Repeatedly Modified Files (Thrashing Detected):" + f"" + f"
    " + ) + + return ( + f"
    {''.join(tiles)}
    " + f"
    " + f"
    ~/ace/quality_breakdownLOCAL VERIFIED
    " + f"
    " + f"
    " + f"
    {sessions_tests} test-verified sessions
    " + f"
    {sessions_edits} editing sessions
    " + f"
    {qm.get('total_tool_calls', 0)} total tool executions
    " + f"
    {redundant_reads} redundant duplicate file reads
    " + f"
    " + f"{thrash_html}" + f"
    Measures how safely and stably coding agents operate in your repository. High verification rates and low thrash indicate clean first-pass execution without prompt churn.
    " + f"
    " + ) + + def _fleet(f: Optional[Dict[str, Any]]) -> str: """§ 01 — the eleven fleet metrics from docs/22 §0, on this machine's transcripts. @@ -1597,6 +1707,7 @@ def _prometheus_section(d: Dict[str, Any]) -> str: # the same reasoning that keeps _sec's id keyed off the section number. _NAV = ( ("◫", "Overview", "01"), + ("🎯", "Code Quality", "02"), ("⇄", "Strategies", "04"), ("✦", "Recommendations", "06"), ("⚡", "Workflow Skills", "07"), @@ -1822,7 +1933,7 @@ def _lever_note(d: Dict[str, Any]) -> str: def _rail(d: Dict[str, Any]) -> str: - live = d["live"] + live = d.get("live") or {"turns": 0} # Each entry jumps to a section already on the page -- one document, not five views. # Anchors rather than divs: clickable without JS. nav = "".join( @@ -1981,11 +2092,23 @@ def render(d: Dict[str, Any]) -> str: # on to "what it cost" and "what to do about it". b.append(_fleet(d.get("fleet"))) - peak = h.get("peak_context") or 0 - # § 02 — spend + # § 02 — code quality & reliability b.append( _sec( "02", + "CODE QUALITY & RELIABILITY", + "Agent execution stability & test hygiene.", + "Verification rate, rework thrash, and error recovery.", + "LOCAL", + ) + ) + b.append(_quality(d.get("quality"))) + + peak = h.get("peak_context") or 0 + # § 03 — spend + b.append( + _sec( + "03", "SPEND", "Where the money goes.", "List price on your transcripts.", diff --git a/src/ace/sidecar/insights.py b/src/ace/sidecar/insights.py index 96e5063..9439ee9 100644 --- a/src/ace/sidecar/insights.py +++ b/src/ace/sidecar/insights.py @@ -148,6 +148,111 @@ def _sig(name: str, tool_input: Dict[str, Any]) -> str: return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16] +_TEST_CMD_RE = re.compile( + r"\b(pytest|npm\s+(?:run\s+)?test|vitest|jest|cargo\s+test|go\s+test|dotnet\s+test|ctest|ruff|eslint|mypy|flake8|pylint|black\s+--check|tsc\s+--noEmit|bundle\s+exec\s+rspec)\b", + re.IGNORECASE, +) + +_TEST_FILE_RE = re.compile( + r"(^|[/\\])(tests?|spec|__tests__)[/\\]|(\.|_)(test|spec)\.[a-zA-Z0-9]+$", + re.IGNORECASE, +) + +_SOURCE_FILE_EXTS = ( + ".py", + ".js", + ".jsx", + ".ts", + ".tsx", + ".go", + ".rs", + ".java", + ".c", + ".cpp", + ".h", + ".hpp", + ".cs", + ".rb", + ".php", + ".swift", + ".kt", + ".scala", + ".sh", + ".html", + ".css", + ".vue", +) + + +def _classify_call(name: str, tool_input: Dict[str, Any]) -> Dict[str, Any]: + target_raw = None + for k in ( + "file_path", + "path", + "TargetFile", + "target_file", + "filename", + "file", + "notebook_path", + "AbsolutePath", + ): + v = tool_input.get(k) + if isinstance(v, str) and v: + target_raw = v + break + + cmd = None + for k in ("command", "CommandLine", "cmd", "input"): + v = tool_input.get(k) + if isinstance(v, str) and v: + cmd = v + break + + name_lower = name.lower() + is_test_run = False + if cmd and _TEST_CMD_RE.search(cmd): + is_test_run = True + + is_edit = name_lower in ( + "edit", + "str_replace_editor", + "write_to_file", + "replace_file_content", + "create_file", + "modify_file_content", + "save_file", + "patch", + ) or ( + name_lower.startswith("edit") + or name_lower.startswith("write") + or name_lower.startswith("replace") + ) + is_view = name_lower in ( + "view", + "view_file", + "read_file", + "cat", + "open_file", + "get_file_contents", + ) or (name_lower.startswith("view") or name_lower.startswith("read")) + + is_test_file = bool(target_raw and _TEST_FILE_RE.search(target_raw)) + is_src_file = bool( + target_raw + and any(target_raw.lower().endswith(ext) for ext in _SOURCE_FILE_EXTS) + and not is_test_file + ) + + return { + "raw_target": target_raw, + "is_test_run": is_test_run, + "is_edit": is_edit, + "is_view": is_view, + "is_test_file": is_test_file, + "is_src_file": is_src_file, + } + + def _measure(body: Any) -> int: """A tool_result body -> its size in byte-equivalents, with images priced as images. @@ -256,6 +361,7 @@ def _scan(root: str) -> List[Dict[str, Any]]: # tool_use_id -> short hash of the result content, so the de-dup lever can prove # "already in context" instead of inferring it from a matching path. result_digests: Dict[str, str] = {} + result_errors: Dict[str, bool] = {} seen_ids: Dict[str, int] = {} # message id -> index of its turn in `turns` cwds: List[str] = [] # The timeline `time_budget` and `parked` read: (start, kind, tool names, end). Only @@ -304,10 +410,16 @@ def _scan(root: str) -> List[Dict[str, Any]]: ): saw_result = True body = b.get("content") - result_bytes[b.get("tool_use_id")] = _measure(body) - dg = _digest(body) - if dg: - result_digests[b.get("tool_use_id")] = dg + tid = b.get("tool_use_id") + if tid: + result_bytes[tid] = _measure(body) + dg = _digest(body) + if dg: + result_digests[tid] = dg + if b.get("is_error") or str( + b.get("status", "") + ).lower() in ("error", "failed"): + result_errors[tid] = True # A tool finishing and a human typing are both "user" records and # they mean opposite things about who the session is waiting on. at = _epoch(rec.get("timestamp")) @@ -330,12 +442,19 @@ def _scan(root: str) -> List[Dict[str, Any]]: if bt == "tool_use": ti = b.get("input") or {} nm = b.get("name") or "?" + cl = _classify_call(nm, ti) calls.append( { "id": b.get("id"), "name": nm, "target": _target(ti), "sig": _sig(nm, ti), + "raw_target": cl["raw_target"], + "is_test_run": cl["is_test_run"], + "is_edit": cl["is_edit"], + "is_view": cl["is_view"], + "is_test_file": cl["is_test_file"], + "is_src_file": cl["is_src_file"], } ) turn = { @@ -382,12 +501,15 @@ def _scan(root: str) -> List[Dict[str, Any]]: for t in turns: for c in t["calls"]: cid = c.pop("id", None) - n = result_bytes.get(cid) - if n is not None: - c["result_bytes"] = n - dg = result_digests.get(cid) - if dg: - c["digest"] = dg + if cid is not None: + n = result_bytes.get(cid) + if n is not None: + c["result_bytes"] = n + dg = result_digests.get(cid) + if dg: + c["digest"] = dg + if cid in result_errors: + c["is_error"] = True # Assistant events are derived here rather than inside the loop above: a turn's # ``tool_use`` blocks can be spread across several records sharing one message id (see # the docstring), so the complete call list only exists once the join is done. Reading @@ -464,6 +586,7 @@ def _scan_antigravity(root: str) -> List[Dict[str, Any]]: result_bytes: Dict[str, int] = {} result_digests: Dict[str, str] = {} + result_errors: Dict[str, bool] = {} first_snippet: str = "" try: @@ -504,6 +627,13 @@ def _scan_antigravity(root: str) -> List[Dict[str, Any]]: dg = _digest(body) if dg: result_digests[tid] = dg + is_err = ( + str(rec.get("status", "")).upper() in ("ERROR", "FAILED") + or bool(rec.get("error")) + or bool(rec.get("is_error")) + ) + if is_err: + result_errors[tid] = True events.append((at, "tool_result", (), at)) continue @@ -525,12 +655,19 @@ def _scan_antigravity(root: str) -> List[Dict[str, Any]]: nm = tc.get("name") or "tool" args = tc.get("args") or tc.get("input") or {} call_id = tc.get("id") or f"call_{len(turns)}_{idx_c}" + cl = _classify_call(nm, args) calls.append( { "id": call_id, "name": nm, "target": _target(args), "sig": _sig(nm, args), + "raw_target": cl["raw_target"], + "is_test_run": cl["is_test_run"], + "is_edit": cl["is_edit"], + "is_view": cl["is_view"], + "is_test_file": cl["is_test_file"], + "is_src_file": cl["is_src_file"], } ) @@ -594,6 +731,8 @@ def _scan_antigravity(root: str) -> List[Dict[str, Any]]: c["result_bytes"] = result_bytes[cid_tag] if cid_tag and cid_tag in result_digests: c["digest"] = result_digests[cid_tag] + if cid_tag and cid_tag in result_errors: + c["is_error"] = True events.sort(key=lambda e: e[0]) name = f"agy_{cid[:8]}" if cid else f"agy_{len(sessions)+1}" @@ -634,6 +773,7 @@ def _scan_codex(root: str) -> List[Dict[str, Any]]: result_bytes: Dict[str, int] = {} result_digests: Dict[str, str] = {} + result_errors: Dict[str, bool] = {} first_snippet: str = "" try: @@ -732,12 +872,19 @@ def _scan_codex(root: str) -> List[Dict[str, Any]]: or payload.get("id") or f"call_{len(turns)}_{len(current_calls)}" ) + cl = _classify_call(nm, args) current_calls.append( { "id": cid, "name": nm, "target": _target(args), "sig": _sig(nm, args), + "raw_target": cl["raw_target"], + "is_test_run": cl["is_test_run"], + "is_edit": cl["is_edit"], + "is_view": cl["is_view"], + "is_test_file": cl["is_test_file"], + "is_src_file": cl["is_src_file"], } ) continue @@ -750,6 +897,13 @@ def _scan_codex(root: str) -> List[Dict[str, Any]]: dg = _digest(out_body) if dg: result_digests[cid] = dg + if ( + payload.get("exit_code") not in (None, 0) + or bool(payload.get("is_error")) + or str(payload.get("status", "")).lower() + in ("error", "failed") + ): + result_errors[cid] = True events.append((at, "tool_result", (), at)) continue @@ -842,6 +996,12 @@ def _scan_codex(root: str) -> List[Dict[str, Any]]: dg = _digest(body) if dg: result_digests[tid] = dg + if ( + rec.get("exit_code") not in (None, 0) + or bool(rec.get("is_error")) + or str(rec.get("status", "")).lower() in ("error", "failed") + ): + result_errors[tid] = True events.append((at, "tool_result", (), at)) continue @@ -883,12 +1043,19 @@ def _scan_codex(root: str) -> List[Dict[str, Any]]: except Exception: args = {"raw": args} call_id = tc.get("id") or f"call_{len(turns)}_{idx_c}" + cl = _classify_call(nm, args) calls.append( { "id": call_id, "name": nm, "target": _target(args), "sig": _sig(nm, args), + "raw_target": cl["raw_target"], + "is_test_run": cl["is_test_run"], + "is_edit": cl["is_edit"], + "is_view": cl["is_view"], + "is_test_file": cl["is_test_file"], + "is_src_file": cl["is_src_file"], } ) @@ -965,6 +1132,8 @@ def _scan_codex(root: str) -> List[Dict[str, Any]]: c["result_bytes"] = result_bytes[cid_tag] if cid_tag and cid_tag in result_digests: c["digest"] = result_digests[cid_tag] + if cid_tag and cid_tag in result_errors: + c["is_error"] = True if not first_snippet: try: @@ -1810,6 +1979,203 @@ def totals(sess: List[Dict[str, Any]]) -> Dict[str, Any]: return agg +# ------------------------------------------------- code quality & reliability metrics + + +def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: + """Calculates unified code quality, verification hygiene, and reliability metrics. + + Evaluates across all agent sessions: + - Verification Hygiene: share of editing sessions running test suites / linters. + - Edit Thrash / Rework: files modified 3+ times within a single session. + - First-Pass Success Rate (FSR): share of tool executions with zero errors on initial run. + - Error Healing Latency: average conversation turns to resolve tool execution errors. + - Redundant File Reads: consecutive duplicate view/reads of unchanged files. + - Test-to-Code Ratio: ratio of test file edits vs source file edits. + - Composite Score: 0-100 overall quality and reliability index. + """ + total_sessions = len(sess) + if not total_sessions: + return { + "available": False, + "quality_score": 100, + "grade": "A", + "verification_rate": 1.0, + "verification_rate_pct": 100.0, + "first_pass_success_rate": 1.0, + "first_pass_success_rate_pct": 100.0, + "tool_error_rate": 0.0, + "tool_error_rate_pct": 0.0, + "total_edits": 0, + "total_tests": 0, + "total_tool_calls": 0, + "thrashed_files_count": 0, + "thrashed_files_list": [], + "rework_thrash_rate": 0.0, + "rework_thrash_rate_pct": 0.0, + "redundant_reads_count": 0, + "avg_error_recovery_turns": 1.0, + "test_to_code_ratio": 1.0, + "sessions_with_edits": 0, + "sessions_with_tests": 0, + } + + sessions_with_edits = 0 + sessions_with_tests = 0 + total_edits = 0 + total_tests = 0 + total_tool_calls = 0 + failed_tool_calls = 0 + redundant_reads_count = 0 + test_edits_count = 0 + src_edits_count = 0 + + all_thrashed_files = set() + recovery_turns_list = [] + + for s in sess: + turns = s.get("turns") or [] + session_has_edit = False + session_has_test = False + session_file_edits: Dict[str, int] = {} + last_view_sig: Optional[str] = None + pending_error_turn: Optional[int] = None + + for turn_idx, t in enumerate(turns): + turn_has_error = False + for c in t.get("calls") or []: + total_tool_calls += 1 + is_err = bool(c.get("is_error")) + if is_err: + failed_tool_calls += 1 + turn_has_error = True + + if c.get("is_test_run"): + session_has_test = True + total_tests += 1 + + if c.get("is_edit"): + session_has_edit = True + total_edits += 1 + raw_t = c.get("raw_target") or c.get("target") or "unknown" + session_file_edits[raw_t] = session_file_edits.get(raw_t, 0) + 1 + if c.get("is_test_file"): + test_edits_count += 1 + elif c.get("is_src_file"): + src_edits_count += 1 + # A file edit invalidates previous view cache + last_view_sig = None + + if c.get("is_view"): + view_sig = ( + c.get("sig") + or c.get("digest") + or c.get("raw_target") + or c.get("target") + ) + if view_sig and view_sig == last_view_sig: + redundant_reads_count += 1 + last_view_sig = view_sig + + if turn_has_error: + if pending_error_turn is None: + pending_error_turn = turn_idx + elif pending_error_turn is not None: + recovery_turns_list.append(max(1, turn_idx - pending_error_turn)) + pending_error_turn = None + + if session_has_edit: + sessions_with_edits += 1 + if session_has_test: + sessions_with_tests += 1 + + for fpath, count in session_file_edits.items(): + if count >= 3: + all_thrashed_files.add(fpath) + + thrashed_files_count = len(all_thrashed_files) + verification_rate = ( + (sessions_with_tests / sessions_with_edits) + if sessions_with_edits > 0 + else (1.0 if not total_edits else 0.0) + ) + + first_pass_success_rate = ( + ((total_tool_calls - failed_tool_calls) / total_tool_calls) + if total_tool_calls > 0 + else 1.0 + ) + + tool_error_rate = ( + (failed_tool_calls / total_tool_calls) + if total_tool_calls > 0 + else 0.0 + ) + + rework_thrash_rate = ( + (thrashed_files_count / max(1, len(all_thrashed_files) + total_edits)) + if total_edits > 0 + else 0.0 + ) + + test_to_code_ratio = ( + (test_edits_count / src_edits_count) + if src_edits_count > 0 + else (1.0 if test_edits_count > 0 else 0.5) + ) + + avg_error_recovery_turns = ( + (sum(recovery_turns_list) / len(recovery_turns_list)) + if recovery_turns_list + else 1.0 + ) + + # Score calculation (0-100) + # Verification: 35%, FSR: 35%, Thrash Freedom: 15%, Test/Code balance: 15% + raw_score = ( + 0.35 * verification_rate + + 0.35 * first_pass_success_rate + + 0.15 * max(0.0, 1.0 - (rework_thrash_rate * 2)) + + 0.15 * min(1.0, test_to_code_ratio) + ) * 100.0 + + quality_score = max(0, min(100, int(round(raw_score)))) + if quality_score >= 90: + grade = "A" + elif quality_score >= 80: + grade = "B" + elif quality_score >= 70: + grade = "C" + elif quality_score >= 60: + grade = "D" + else: + grade = "F" + + return { + "available": True, + "quality_score": quality_score, + "grade": grade, + "verification_rate": round(verification_rate, 4), + "verification_rate_pct": round(verification_rate * 100.0, 1), + "first_pass_success_rate": round(first_pass_success_rate, 4), + "first_pass_success_rate_pct": round(first_pass_success_rate * 100.0, 1), + "tool_error_rate": round(tool_error_rate, 4), + "tool_error_rate_pct": round(tool_error_rate * 100.0, 1), + "total_edits": total_edits, + "total_tests": total_tests, + "total_tool_calls": total_tool_calls, + "thrashed_files_count": thrashed_files_count, + "thrashed_files_list": sorted(list(all_thrashed_files))[:10], + "rework_thrash_rate": round(rework_thrash_rate, 4), + "rework_thrash_rate_pct": round(rework_thrash_rate * 100.0, 1), + "redundant_reads_count": redundant_reads_count, + "avg_error_recovery_turns": round(avg_error_recovery_turns, 1), + "test_to_code_ratio": round(test_to_code_ratio, 2), + "sessions_with_edits": sessions_with_edits, + "sessions_with_tests": sessions_with_tests, + } + + # ------------------------------------------------- time budget (docs/analysis_docs §1 and §2) # Declared order is presentation order for ties; the payload sorts by size. @@ -2140,6 +2506,37 @@ def recommendations( ) ) + # 4. Code Quality & Test Hygiene Recommendations + if sess is not None: + qm = quality_metrics(sess) + if ( + qm.get("sessions_with_edits", 0) > 0 + and qm.get("verification_rate", 1.0) < 0.5 + ): + recs.append( + _rec( + "Low test verification hygiene in agent sessions", + f"Only {qm.get('verification_rate_pct', 0)}% of sessions with code edits ran automated test suites. " + f"Running test/lint passes before finishing turns reduces runtime bugs and catches regressions early.", + f"{qm.get('sessions_with_tests', 0)} of {qm.get('sessions_with_edits', 0)} editing sessions verified", + "LOW", + f"{round((1.0 - qm.get('verification_rate', 0.0)) * 100, 1)}% unverified", + "of editing sessions", + ) + ) + if qm.get("thrashed_files_count", 0) >= 3: + recs.append( + _rec( + f"File edit thrashing detected on {qm.get('thrashed_files_count')} files", + "Agent modified the same files 3+ times in single sessions. Providing more explicit prompt instructions, " + "specifying test fixtures, or decomposing tasks into smaller subagents reduces edit churn.", + f"{qm.get('thrashed_files_count')} thrashed files across scope", + "MED", + f"{qm.get('rework_thrash_rate_pct')}% thrash", + "churn rate", + ) + ) + if not recs: recs.append( _rec( @@ -2444,6 +2841,35 @@ def span( _BUILD_CACHE_MAX = 32 +def _lever_rail_payload(scoped: List[Dict[str, Any]]) -> Dict[str, Any]: + """The live half of the lever rail, or a payload saying why there isn't one. + + Imported lazily and wrapped: ``ace.sidecar.levers`` discovers third-party packages, and + nothing a stranger's distribution does at import time may take this dashboard down. A + failure here costs the live column and leaves every measured figure on the page intact. + """ + try: + from ace.sidecar.levers.rail import rail_payload + + return rail_payload(scoped) + except Exception: + log.warning("[levers] rail payload failed; rendering headroom only", exc_info=True) + return {"status": "no_package", "note": "lever rail unavailable", "installed": []} + + +def _refresh_lever_rail(payload: Dict[str, Any], store: Any) -> Dict[str, Any]: + """Live half of the lever rail, re-read from the telemetry store. Never raises.""" + if store is None or not payload: + return payload + try: + from ace.sidecar.levers.rail import refresh_measured + + return refresh_measured(payload, store) + except Exception: + log.warning("[levers] measured rail refresh failed", exc_info=True) + return payload + + def _build_payload( all_sessions: List[Dict[str, Any]], capture: Optional[Dict[str, Any]], @@ -2471,6 +2897,7 @@ def _build_payload( "agent_breakdown": ab, "span": span(range_key, window, agg), "historical": agg, + "quality": quality_metrics(scoped), "time": time_budget(scoped), "parked": parked(scoped), "fleet": _fleet, @@ -2478,6 +2905,11 @@ def _build_payload( "scorecards": ( scorecards(scoped, agg.get("cost_usd") or 0.0) if agg["available"] else None ), + # The *measured* half of the lever rail, beside `scorecards`' simulated headroom. + # The two are different claims and the renderer must not merge them: headroom is a + # byte-turn estimate of what a lever would be worth, `levers` is what an installed + # one actually measured. Its `status` says which of the two exists. + "levers": _lever_rail_payload(scoped), "files": session_files(agent=agent, all_sessions=all_sessions), "capture": capture or {}, "recommendations": recommendations(agg, capture, sess=scoped), @@ -2541,6 +2973,11 @@ def build( # live keys below are per-request and must not be written into the shared cached dict. out = dict(payload) out["live"] = store.summary() if store is not None else {"turns": 0} + # Re-read for the same reason `live` is: the measured lever rail moves on every proxied + # turn, while the payload around it is memoised on a transcript fingerprint that a + # proxied turn does not change. Cached with the rest, the one live number on the rail + # would be frozen at whatever it read when the transcripts last changed. + out["levers"] = _refresh_lever_rail(out.get("levers") or {}, store) out["recent"] = store.recent(30) if store is not None else [] return out @@ -2686,6 +3123,40 @@ def format_prometheus_metrics(d: Dict[str, Any]) -> str: lines.append("# TYPE ace_installed_skills_total gauge") lines.append(f'ace_installed_skills_total {len(skills)}') + # Code Quality & Reliability Metrics + qm = d.get("quality") or {} + lines.append("# HELP ace_quality_score Composite code quality and verification score (0-100).") + lines.append("# TYPE ace_quality_score gauge") + lines.append(f'ace_quality_score {qm.get("quality_score", 100)}') + + lines.append("# HELP ace_quality_verification_rate Share of edited sessions that ran automated tests or linters.") + lines.append("# TYPE ace_quality_verification_rate gauge") + lines.append(f'ace_quality_verification_rate {qm.get("verification_rate", 1.0)}') + + lines.append("# HELP ace_quality_first_pass_success_rate Share of tool calls that succeeded on first pass.") + lines.append("# TYPE ace_quality_first_pass_success_rate gauge") + lines.append(f'ace_quality_first_pass_success_rate {qm.get("first_pass_success_rate", 1.0)}') + + lines.append("# HELP ace_quality_tool_error_rate Share of tool executions that returned errors.") + lines.append("# TYPE ace_quality_tool_error_rate gauge") + lines.append(f'ace_quality_tool_error_rate {qm.get("tool_error_rate", 0.0)}') + + lines.append("# HELP ace_quality_thrashed_files_total Number of files edited 3 or more times in a single session.") + lines.append("# TYPE ace_quality_thrashed_files_total counter") + lines.append(f'ace_quality_thrashed_files_total {qm.get("thrashed_files_count", 0)}') + + lines.append("# HELP ace_quality_redundant_reads_total Count of consecutive duplicate file reads.") + lines.append("# TYPE ace_quality_redundant_reads_total counter") + lines.append(f'ace_quality_redundant_reads_total {qm.get("redundant_reads_count", 0)}') + + lines.append("# HELP ace_quality_error_recovery_turns_avg Average turns to recover from an execution error.") + lines.append("# TYPE ace_quality_error_recovery_turns_avg gauge") + lines.append(f'ace_quality_error_recovery_turns_avg {qm.get("avg_error_recovery_turns", 1.0)}') + + lines.append("# HELP ace_quality_test_to_code_ratio Ratio of test file edits to source file edits.") + lines.append("# TYPE ace_quality_test_to_code_ratio gauge") + lines.append(f'ace_quality_test_to_code_ratio {qm.get("test_to_code_ratio", 1.0)}') + return "\n".join(lines) + "\n" diff --git a/src/ace/sidecar/levers/__init__.py b/src/ace/sidecar/levers/__init__.py new file mode 100644 index 0000000..bf3ed50 --- /dev/null +++ b/src/ace/sidecar/levers/__init__.py @@ -0,0 +1,115 @@ +"""ace.sidecar.levers — the public contract optimization modules are written against. + +This package contains no optimizations. It defines the normalized session model every lever +reads (:mod:`~ace.sidecar.levers.types`), what a lever is allowed to do +(:mod:`~ace.sidecar.levers.protocol`), and how installed ones are found +(:mod:`~ace.sidecar.levers.registry`). Implementations ship separately and register through +the ``ace.sidecar.levers`` entry-point group. + +Three properties are worth stating once, because everything here follows from them. + +**One lever, every agent.** Levers read the corpus shape that ``insights._scan``, +``_scan_antigravity`` and ``_scan_codex`` already agree on, never a provider's wire format. +Supporting a fourth coding agent is a scanner, not a lever rewrite. + +**Measurement is universal; actuation is not.** Scoring runs off transcripts, so it works +for every agent the sidecar can read, with no proxy and no hooks. Rewriting bytes needs a +write path, and only some agents have one. ``Lever.requires_content`` is where a lever +declares which half it needs, and the registry refuses the mismatch rather than degrading. + +**Levers propose; the ledger prices.** No lever returns a dollar figure. Pricing happens +once, in :mod:`~ace.sidecar.levers.ledger`, where the tokenizer and the rate catalog live — +which is what keeps a saving auditable and keeps provider-specific cache economics out of a +lever that is meant to be provider-neutral. The ledger prices nothing it cannot count +exactly, and it ranks levers rather than totalling them. +""" + +from ace.sidecar.levers.ledger import ( + FIDELITY_MEASURED, + FIDELITY_UNMEASURABLE, + FIDELITY_UNPRICED, + EditCost, + LedgerEntry, + LedgerReport, + price_all, + price_proposal, +) +from ace.sidecar.levers.protocol import ( + MODE_OFF, + MODE_ON, + MODE_SHADOW, + MODES, + RISK_HIGH, + RISK_LOW, + RISK_MEDIUM, + RISK_NONE, + Edit, + EditKind, + Lever, + LeverContext, + Proposal, + TokenCounter, +) +from ace.sidecar.levers.registry import ( + CONFIG_PATH, + ENTRY_POINT_GROUP, + RegisteredLever, + discover, + load_settings, + propose_safely, + resolve_modes, +) +from ace.sidecar.levers.types import ( + ContentRef, + ContentUnavailable, + Session, + ToolCall, + Turn, + Usage, + from_corpus_session, + from_corpus_sessions, +) + +__all__ = [ + # types + "Session", + "Turn", + "ToolCall", + "Usage", + "ContentRef", + "ContentUnavailable", + "from_corpus_session", + "from_corpus_sessions", + # protocol + "Lever", + "LeverContext", + "Proposal", + "Edit", + "EditKind", + "TokenCounter", + "MODE_OFF", + "MODE_SHADOW", + "MODE_ON", + "MODES", + "RISK_NONE", + "RISK_LOW", + "RISK_MEDIUM", + "RISK_HIGH", + # registry + "discover", + "resolve_modes", + "load_settings", + "propose_safely", + "RegisteredLever", + "ENTRY_POINT_GROUP", + "CONFIG_PATH", + # ledger + "price_proposal", + "price_all", + "LedgerEntry", + "LedgerReport", + "EditCost", + "FIDELITY_MEASURED", + "FIDELITY_UNMEASURABLE", + "FIDELITY_UNPRICED", +] diff --git a/src/ace/sidecar/levers/counter.py b/src/ace/sidecar/levers/counter.py new file mode 100644 index 0000000..cd2289a --- /dev/null +++ b/src/ace/sidecar/levers/counter.py @@ -0,0 +1,240 @@ +"""ace.sidecar.levers.counter — an exact token counter, built from the credential in hand. + +Why this module exists at all +----------------------------- +The ledger prices a prompt that was never sent. The baseline side of every counterfactual is +ground truth — the provider's own per-turn counts, read off the transcript — but the proposed +side is text nobody submitted, so its tokens have to be *produced*. Approximating there is +what turns a measured saving back into an estimate, which is the one thing +:mod:`ace.sidecar.levers.ledger` refuses to do. Hence an exact counter, and hence a network +call: for Claude the only exact counter is Anthropic's ``POST /v1/messages/count_tokens``. + +Why it takes the credential instead of reading the environment +-------------------------------------------------------------- +The previous version sniffed ``ANTHROPIC_API_KEY`` out of ``os.environ``. On the deployment +that matters that variable is empty: the sidecar's own default is ``{"no_key": true}``, and a +Claude Code session on a **subscription** never has an API key at all — it authenticates with +an OAuth token that exists only for the life of a request, in the ``Authorization`` header the +proxy is already relaying. + +So the credential is passed in, from whoever has one: + +* the proxy turn path hands over the in-flight credential it is about to relay upstream + (:mod:`ace.gateway.messages`), which is the only path that has one under ``no_key``; +* the dashboard falls back to the environment, for a developer who does export a key. + +There is no preflight probe +--------------------------- +An earlier plan for this module called for one live call to establish whether a subscription +OAuth token is accepted by the counting endpoint. It isn't needed: the first real count +answers the same question as a side effect, and a dedicated ping only adds a round trip and a +second code path that can disagree with the first. + +What *is* needed is that a refusal be remembered and explained. :class:`AnthropicCounter` +latches the first authentication failure, stops calling, and keeps the reason as +:attr:`~AnthropicCounter.note`, so the rail can render ``no_counter — the counting endpoint +rejected this credential (401)`` rather than an unexplained blank. A transport blip is +treated differently and is *not* latched: it costs one edit, not the whole feature. + +Presentation is delegated, never re-derived +-------------------------------------------- +Building the auth headers here would be a second implementation of a rule this repository +already got wrong once: an OAuth token sent as ``x-api-key`` is rejected, and ``/v1/messages`` +additionally requires the ``oauth-2025-04-20`` beta. ``messages_auth.upstream_auth_headers`` +owns that rule for the relay, so it owns it here too. The counting endpoint lives under the +same ``/v1/messages`` prefix and takes the same credentials as the route it belongs to. +""" + +from __future__ import annotations + +import logging +import os +import threading +from typing import Any, Mapping, Optional, Tuple + +import httpx + +from ace.gateway.messages_auth import ( + SCHEME_API_KEY, + SCHEME_BEARER, + upstream_auth_headers, +) + +__all__ = ["COUNT_TOKENS_PATH", "COUNTABLE_FIELDS", "AnthropicCounter", "resolve_counter"] + +log = logging.getLogger(__name__) + +# The only request fields ``/v1/messages/count_tokens`` accepts. Everything else on a real +# turn — ``stream``, ``max_tokens``, ``temperature``, ``metadata`` — is rejected as an unknown +# parameter, so a body cannot be forwarded to the counter as-is. +# +# All four listed here contribute tokens and must be kept: dropping ``system`` or ``tools`` +# would under-count the prompt by the largest stable part of an agent request. That does not +# matter for a *delta* between two counts taken the same way, but it matters enormously for +# the cross-check against the provider's own reported prompt size. +COUNTABLE_FIELDS = ("model", "messages", "system", "tools", "tool_choice", "thinking") + +ANTHROPIC_DEFAULT_BASE_URL = "https://api.anthropic.com" +COUNT_TOKENS_PATH = "/v1/messages/count_tokens" + +# Long enough for a real call, short enough that a hung endpoint cannot stall a dashboard +# render or add itself to a developer's turn latency. +_TIMEOUT_S = 10.0 + +# A model must be named for the endpoint to answer, and the count is model-family specific. +# Only used when the caller supplies nothing — every real call carries the turn's own model. +_FALLBACK_MODEL = "claude-sonnet-5" + +# Statuses that mean "this credential will never work here". Latched. Anything else — a 429, +# a 500, a timeout — is transient and must not disable counting for the whole process. +_FATAL_AUTH_STATUSES = (401, 403) + + +class AnthropicCounter: + """Exact token counts from Anthropic, over one credential. Satisfies ``TokenCounter``. + + Callable, and deliberately stateful: the state is the single fact worth remembering + across calls, which is whether this credential is accepted at all. + + Raises rather than returning a sentinel on failure. The ledger already treats a raising + counter as "this edit is unmeasurable" and prices nothing for it, which is the correct + outcome — a counter that returned ``0`` for an uncountable string would silently report + the entire original as saved. + """ + + __slots__ = ("_credential", "_scheme", "_url", "_client", "_lock", "_dead", "note", "calls") + + def __init__( + self, + credential: str, + scheme: str = SCHEME_API_KEY, + *, + base_url: Optional[str] = None, + client: Optional[httpx.Client] = None, + ) -> None: + self._credential = credential + self._scheme = scheme + self._url = (base_url or ANTHROPIC_DEFAULT_BASE_URL).rstrip("/") + COUNT_TOKENS_PATH + # Injectable so the suite can drive this exact branch through MockTransport with no + # live call — the same discipline `install_messages_route` uses for its relay client. + self._client = client + self._lock = threading.Lock() + self._dead: Optional[str] = None + self.note = "Anthropic /v1/messages/count_tokens" + self.calls = 0 + + @property + def usable(self) -> bool: + """False once the credential has been definitively refused.""" + return self._dead is None + + def _http(self) -> httpx.Client: + if self._client is None: + self._client = httpx.Client(timeout=_TIMEOUT_S) + return self._client + + def __call__(self, text: str, *, model: str) -> int: + """Tokens in one standalone string. The ``TokenCounter`` protocol's shape.""" + return self.count_body( + {"model": model or _FALLBACK_MODEL, + "messages": [{"role": "user", "content": text}]} + ) + + def count_body(self, body: Mapping[str, Any]) -> int: + """Tokens in a whole ``/v1/messages`` request — system prompt and tools included. + + This is what the live shadow path needs. A lever's edit lands inside one tool result + buried in a long ``messages`` array, and the quantity that matters is what the whole + prompt would have cost, not what the edited fragment costs on its own: an edit can + change block boundaries and therefore tokenize differently in place than in isolation. + + Only a *delta* between two bodies counted this way is exact. Comparing one of these + against the provider's reported ``prompt_tokens`` is a cross-check, not a measurement + — the two include slightly different scaffolding. + """ + if self._dead is not None: + raise RuntimeError(self._dead) + + payload = {k: body[k] for k in COUNTABLE_FIELDS if k in body} + payload.setdefault("model", _FALLBACK_MODEL) + + headers = { + "anthropic-version": "2023-06-01", + "content-type": "application/json", + } + # The OAuth beta is merged in here exactly as the relay does it; presenting a + # subscription token without it is the failure this indirection exists to avoid. + headers.update(upstream_auth_headers(self._credential, self._scheme)) + + resp = self._http().post( + self._url, + json=payload, + headers=headers, + timeout=_TIMEOUT_S, + ) + + if resp.status_code in _FATAL_AUTH_STATUSES: + # Latch, and say which credential shape was refused. This is the line that turns + # "the dashboard shows nothing" into an answerable question, and it is the one + # place the OAuth-vs-API-key outcome is actually established. + kind = "OAuth token" if self._scheme == SCHEME_BEARER else "API key" + with self._lock: + self._dead = ( + f"the counting endpoint rejected this {kind} ({resp.status_code}) — " + f"exact counts need a credential it accepts" + ) + self.note = self._dead + log.warning("[levers] %s", self._dead) + raise RuntimeError(self._dead) + + # Not latched: a rate limit or a 5xx says nothing about the credential, and disabling + # measurement for the process because one call was throttled would be a bug that + # looks exactly like the feature not working. + resp.raise_for_status() + + n = int((resp.json() or {}).get("input_tokens", -1)) + if n < 0: + raise RuntimeError("counting endpoint returned no input_tokens") + with self._lock: + self.calls += 1 + return n + + +def resolve_counter( + credential: Optional[str] = None, + scheme: str = SCHEME_API_KEY, + *, + base_url: Optional[str] = None, + client: Optional[httpx.Client] = None, +) -> Tuple[Optional[AnthropicCounter], str]: + """An exact counter and where its credential came from, or ``(None, reason)``. + + Exactness is the whole requirement, so only the model vendor's own counter is offered. + ``tiktoken`` is deliberately not a fallback: it is OpenAI's BPE and merely a *proxy* for + anything else, and a proxy here turns a measured saving into an estimate wearing a dollar + sign. Neither is ``bytes / 4`` — see ``strategies.BYTES_PER_TOKEN``, where the real + measured ratio on agent tool output is closer to 2.8 and the 4.0 everyone reaches for sits + at the 99th percentile of the distribution. + + Returning ``None`` is an ordinary outcome and not a failure. It costs the live column and + leaves the simulated headroom rail exactly as it is. + """ + if not credential: + # No in-flight credential: the dashboard path, rendered outside any request. An + # exported key is the only thing that can serve it. + env_key = (os.getenv("ANTHROPIC_API_KEY") or "").strip() + env_tok = (os.getenv("ANTHROPIC_AUTH_TOKEN") or "").strip() + if env_key: + credential, scheme = env_key, SCHEME_API_KEY + elif env_tok: + credential, scheme = env_tok, SCHEME_BEARER + else: + return None, ( + "no credential available — this sidecar runs on `no_key: true`, so exact " + "counts come from the token a proxied turn relays, or from an exported " + "ANTHROPIC_API_KEY" + ) + + counter = AnthropicCounter(credential, scheme, base_url=base_url, client=client) + kind = "relayed OAuth token" if scheme == SCHEME_BEARER else "API key" + return counter, f"Anthropic /v1/messages/count_tokens via {kind}" diff --git a/src/ace/sidecar/levers/ledger.py b/src/ace/sidecar/levers/ledger.py new file mode 100644 index 0000000..677a2e9 --- /dev/null +++ b/src/ace/sidecar/levers/ledger.py @@ -0,0 +1,450 @@ +"""ace.sidecar.levers.ledger — the one place a lever's proposal becomes money. + +The rule +-------- +**No estimates.** Both sides of every figure here are either measured or absent. + +The baseline side is ground truth: the provider's own per-turn token counts, read off the +transcript. The proposed side is a prompt that was never sent, so its tokens have to be +produced — and the only honest way to produce them is to count the actual text with the +provider's own counter (``ctx.count_tokens``: Anthropic's ``/v1/messages/count_tokens`` for +Claude, tiktoken for OpenAI models where it is that vendor's own BPE, Gemini's counting +endpoint for Gemini). + +Where the text is not in hand, this module returns :data:`FIDELITY_UNMEASURABLE` and prices +nothing. It does **not** fall back to ``result_bytes / 4``. A ratio-derived saving is an +estimate wearing a dollar sign, and one number here that a developer can contradict against +their own invoice discredits the measured half of the dashboard along with it. Ranking +levers without content is a real and useful job — it is what ``strategies.py`` does, in +byte-turns, explicitly labelled a simulation — but it is not this module's job. + +The three arithmetic traps this module exists to avoid +----------------------------------------------------- +**1. Ignoring the cache-write penalty.** Every lever removes tokens from a prompt that the +provider was mostly serving from cache at ~0.1x. Removing them saves that cheap rate, not +the fresh-input rate. And an edit that changes content the cache has *already* seen +invalidates the prefix from that point, so the next turn re-writes it at a premium (1.25x at +Anthropic's 5-minute TTL, 2x at one hour). A lever that reports gross saving and omits the +penalty can report a win on an edit that cost money. :class:`EditCost` carries both legs and +``net_usd`` is the only figure meant to be quoted. + +**2. Summing levers.** Two levers can target the same bytes; scored alone their figures +overlap. :class:`LedgerReport` therefore ranks and never totals — the same discipline +``strategies.STANDALONE`` already documents. + +**3. Calling an unpriced model free.** A model with no catalog entry yields +``priced=False`` and zeroes, which must render as "unpriced", never as $0.00 of spend. A +silent zero looks like a cost win. + +What is out of scope +-------------------- +Only *volume* is priced here: edits that put fewer tokens in a later prompt. Accounting +levers — buying a longer cache TTL, normalising a mutating field so a prefix stops +breaking — convert price without sending less, produce no :class:`Edit`, and are worth +exactly nothing against a token cap. Conflating the two is the easiest way to overstate this +product, so they do not share a number with it. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple + +from ace.gateway.pricing import Rates, rates_for +from ace.sidecar.levers.protocol import Edit, LeverContext, Proposal +from ace.sidecar.levers.types import Session, ToolCall, Turn + +__all__ = [ + "FIDELITY_MEASURED", + "FIDELITY_UNMEASURABLE", + "FIDELITY_UNPRICED", + "EditCost", + "LedgerEntry", + "LedgerReport", + "price_proposal", + "price_all", +] + +# Counted exactly, with a catalog rate behind every dollar. The only tier that may be quoted. +FIDELITY_MEASURED = "measured" +# The text was not in hand, so the token delta could not be counted. No dollars, by design. +FIDELITY_UNMEASURABLE = "unmeasurable" +# Counted exactly, but the model has no catalog entry. Tokens are real; dollars are absent. +FIDELITY_UNPRICED = "unpriced" + +_MTOK = 1_000_000.0 + +# What a dropped result is replaced by in context — a short pointer, not nothing. Matches +# ``strategies.POINTER_BYTES``; at ~4 bytes/token it is a rounding error, but counting it as +# zero would claim a saving the applied lever does not actually deliver. +_POINTER_TOKENS = 30 + + +@dataclass(frozen=True, slots=True) +class EditCost: + """One edit, priced against the turns that actually carried its bytes. + + ``prefix_safe`` is the distinction that decides whether this edit is nearly free or has + to earn back a penalty first. A tool result created at turn *i* first enters the prompt + at turn *i+1* and is written to the cache there for the first time. Editing it before + that write costs nothing — the same single write happens, just smaller. Editing content + the cache has already stored invalidates the prefix from that point on, and the next turn + re-writes the remainder at the write premium. + + That is why tail-acting levers (truncate a fresh dump, strip a screenshot on the way in) + can ship far earlier than history-rewriting ones: they are prefix-safe by construction + and carry ``cache_write_penalty_usd == 0``. + """ + + lever: str + turn_index: int + call_index: int + kind: str + reason: str + model: str + + removed_tokens: int = 0 + # First turn whose prompt is smaller because of this edit. + apply_at: int = 0 + # How many turns carried the removed tokens and now do not. + turns_carried: int = 0 + + prefix_safe: bool = True + invalidated_tokens: int = 0 + cache_write_ttl: str = "5m" + + gross_saving_usd: float = 0.0 + cache_write_penalty_usd: float = 0.0 + priced: bool = True + + @property + def net_usd(self) -> float: + """The only figure meant to be quoted. May be negative — that is the point.""" + return self.gross_saving_usd - self.cache_write_penalty_usd + + @property + def per_turn_saving_usd(self) -> float: + return self.gross_saving_usd / self.turns_carried if self.turns_carried else 0.0 + + @property + def break_even_turn(self) -> Optional[int]: + """The turn at which this edit stops costing money and starts saving it. + + ``None`` when there is no penalty to earn back (a prefix-safe edit is in profit + immediately) or when the edit saves nothing per turn. This is the number worth + putting in front of a developer: "compacting here costs $0.04 now and saves + $0.011/turn — you break even at turn 7, and this session ran 60." + """ + per_turn = self.per_turn_saving_usd + if self.cache_write_penalty_usd <= 0.0 or per_turn <= 0.0: + return None + return self.apply_at + math.ceil(self.cache_write_penalty_usd / per_turn) + + +@dataclass(frozen=True, slots=True) +class LedgerEntry: + """One lever, on one session, priced. + + ``fidelity`` qualifies every number above it and must be carried into the UI. A + :data:`FIDELITY_UNMEASURABLE` entry has real ``diagnostics`` and no dollars; rendering it + beside a measured entry without the label is how an estimate ends up quoted as a + measurement. + """ + + lever: str + session_id: str + agent: str + fidelity: str = FIDELITY_MEASURED + edits: Tuple[EditCost, ...] = () + diagnostics: Mapping[str, Any] = field(default_factory=dict) + # Why nothing was priced, when fidelity is not MEASURED. + note: str = "" + # Provenance for every rate used, so a figure can cite the price list that produced it. + rate_sources: Tuple[Tuple[str, str, str], ...] = () # (model, source, as_of) + + @property + def removed_tokens(self) -> int: + return sum(e.removed_tokens for e in self.edits) + + @property + def gross_saving_usd(self) -> float: + return sum(e.gross_saving_usd for e in self.edits) + + @property + def cache_write_penalty_usd(self) -> float: + return sum(e.cache_write_penalty_usd for e in self.edits) + + @property + def net_usd(self) -> float: + return self.gross_saving_usd - self.cache_write_penalty_usd + + @property + def priced(self) -> bool: + return self.fidelity == FIDELITY_MEASURED and bool(self.edits) + + +@dataclass(frozen=True, slots=True) +class LedgerReport: + """Every lever's entries, ranked. Deliberately without a total. + + There is no ``total_usd`` here and there must not be one. Levers are scored alone, so two + of them can claim the same bytes and their figures overlap; adding them up produces a + number that is larger than anything the levers could jointly deliver. Ranking answers the + question that is actually being asked — which lever is worth building or enabling first. + """ + + entries: Tuple[LedgerEntry, ...] = () + + def by_lever(self) -> Dict[str, float]: + """``{lever_id: net usd}`` summed across sessions — safe, because it is one lever.""" + out: Dict[str, float] = {} + for e in self.entries: + if e.priced: + out[e.lever] = out.get(e.lever, 0.0) + e.net_usd + return out + + def ranked(self) -> List[Tuple[str, float]]: + """Levers, best first. The ordering the rail exists to show.""" + return sorted(self.by_lever().items(), key=lambda kv: -kv[1]) + + def unmeasured(self) -> Tuple[LedgerEntry, ...]: + """Entries that produced no dollars, with the reason. Surface these, do not drop them.""" + return tuple(e for e in self.entries if e.fidelity != FIDELITY_MEASURED) + + +def _call_at(session: Session, edit: Edit) -> Optional[ToolCall]: + if not (0 <= edit.turn_index < len(session.turns)): + return None + calls = session.turns[edit.turn_index].calls + if not (0 <= edit.call_index < len(calls)): + return None + return calls[edit.call_index] + + +def _count(text: Any, model: str, ctx: LeverContext) -> Optional[int]: + """Exact token count, or ``None`` when the text is not countable text. + + A non-string body (a list of content parts) is serialized the way the provider would + render it only if the lever already reduced it to text. Anything else returns ``None`` + and the edit goes unmeasured — guessing at a multimodal part's token cost is precisely + the estimate this module refuses to make. + """ + if text is None: + return None + if not isinstance(text, str): + return None + try: + n = ctx.count_tokens(text, model=model) + except Exception: + return None + return int(n) if n is not None and n >= 0 else None + + +def _removed_tokens( + edit: Edit, call: ToolCall, model: str, ctx: LeverContext +) -> Optional[int]: + """Tokens this edit takes out of every later prompt. Exact, or ``None``. + + Requires the original bytes for every kind except ``expire``, which removes the whole + result from later prompts and so still needs the result's own token count. + """ + if call.content is None or not call.content.available: + return None + try: + body = call.content.resolve() + except Exception: + return None + + original = _count(body, model, ctx) + if original is None: + return None + + if edit.kind == "drop": + return max(0, original - _POINTER_TOKENS) + if edit.kind == "expire": + # Nothing is rewritten; the result simply stops being resident after ``live_until``. + return original + if edit.kind == "replace": + kept = _count(edit.replacement or "", model, ctx) + return None if kept is None else max(0, original - kept) + if edit.kind == "truncate": + if edit.keep_bytes is None: + return None + if not isinstance(body, str): + return None + kept = _count(body[: edit.keep_bytes], model, ctx) + return None if kept is None else max(0, original - kept) + return None + + +def _apply_at(edit: Edit) -> int: + """First turn whose prompt this edit changes. + + A tool result created at turn *i* is not in turn *i*'s own prompt — it lands in *i+1*'s. + ``expire`` instead takes effect the turn after the result stops being worth keeping. + """ + if edit.kind == "expire": + return (edit.live_until if edit.live_until is not None else edit.turn_index) + 1 + return edit.turn_index + 1 + + +def _invalidated_tokens(session: Session, edit_turn: int, apply_at: int) -> int: + """Cached tokens the prefix loses when this edit lands after the content was cached. + + Derived from two ground-truth numbers and nothing else: the prompt size at the turn the + content was created and at the turn the edit takes effect. What sits between them is what + the cache holds beyond the edit point and must be re-written. + + Capped by the tokens actually served from cache at ``apply_at`` — a prefix cannot lose + more than it held, and reporting a penalty larger than the cache read would overstate the + cost of every history-rewriting lever. + + When the prompt SHRANK between the two turns the subtraction is meaningless: something + else already rewrote the history (a compaction, a context edit), so the edited content's + position can no longer be derived from sizes. The answer there is the whole cached + prefix, not zero. Both readings are wrong, and they are wrong in opposite directions — + assuming zero understates the penalty, which overstates the saving, which is the one + error this module exists to prevent. + """ + turns = session.turns + if not (0 <= edit_turn < len(turns)) or not (0 <= apply_at < len(turns)): + return 0 + cached = turns[apply_at].usage.cache_read_tokens + grew = turns[apply_at].usage.prompt_tokens - turns[edit_turn].usage.prompt_tokens + if grew < 0: + return max(0, int(cached)) + return max(0, min(int(grew), int(cached))) + + +def _ttl_for(turn: Turn) -> str: + """The TTL this turn's cache writes were actually billed at. + + Read from the turn rather than assumed. ``strategies.TTL_SECONDS`` hard-codes one hour + while the default on a Claude Code session is the 5-minute tier, and the two carry + different write premiums (2x vs 1.25x) — an assumed TTL prices the penalty wrong in + whichever direction the assumption is off. + """ + by_ttl = turn.usage.cache_write_by_ttl + if by_ttl: + return max(by_ttl.items(), key=lambda kv: kv[1])[0] + return "5m" + + +def price_proposal( + session: Session, + proposal: Proposal, + ctx: LeverContext, + *, + rates_lookup: Callable[[str], Optional[Rates]] = rates_for, +) -> LedgerEntry: + """Price one lever's proposal against one session. The core of this module. + + Returns an entry rather than raising: an unmeasurable proposal is an ordinary outcome + (the measurement path has no content for any of them) and the caller needs the + diagnostics either way. + """ + n = session.n_turns + costs: List[EditCost] = [] + sources: Dict[str, Tuple[str, str]] = {} + unmeasured = 0 + + for edit in proposal.edits: + call = _call_at(session, edit) + if call is None: + unmeasured += 1 + continue + + apply_at = _apply_at(edit) + turns_carried = n - apply_at + if turns_carried <= 0: + # The result never reached another prompt, so removing it saves nothing. Recorded + # as a zero rather than dropped: "this lever fired on the last turn and therefore + # saved nothing" is a real and useful thing for a rail row to say. + turns_carried = 0 + + model = session.turns[min(apply_at, n - 1)].model if n else "" + removed = _removed_tokens(edit, call, model, ctx) + if removed is None: + unmeasured += 1 + continue + + rates = rates_lookup(model) + prefix_safe = apply_at <= edit.turn_index + 1 + invalidated = ( + 0 if prefix_safe else _invalidated_tokens(session, edit.turn_index, apply_at) + ) + ttl = _ttl_for(session.turns[min(apply_at, n - 1)]) if n else "5m" + + if rates is None: + costs.append( + EditCost( + lever=proposal.lever, turn_index=edit.turn_index, + call_index=edit.call_index, kind=edit.kind, reason=edit.reason, + model=model, removed_tokens=removed, apply_at=apply_at, + turns_carried=turns_carried, prefix_safe=prefix_safe, + invalidated_tokens=invalidated, cache_write_ttl=ttl, priced=False, + ) + ) + continue + + sources[model] = (rates.source, rates.as_of) + # Priced at the CACHE-READ rate, not the fresh-input rate. These tokens were resident + # in a cached prefix and re-read each turn at ~0.1x; valuing them at the input rate + # would inflate every lever tenfold. It is also the conservative direction. + gross = (removed / _MTOK) * rates.cache_read_per_mtok * turns_carried + # The penalty is the DELTA between writing those tokens and reading them, not the + # full write price: they were going to be paid for either way. + penalty = (invalidated / _MTOK) * ( + rates.cache_write_per_mtok(ttl) - rates.cache_read_per_mtok + ) + costs.append( + EditCost( + lever=proposal.lever, turn_index=edit.turn_index, + call_index=edit.call_index, kind=edit.kind, reason=edit.reason, + model=model, removed_tokens=removed, apply_at=apply_at, + turns_carried=turns_carried, prefix_safe=prefix_safe, + invalidated_tokens=invalidated, cache_write_ttl=ttl, + gross_saving_usd=gross, cache_write_penalty_usd=max(0.0, penalty), + priced=True, + ) + ) + + if not costs: + note = ( + "no tool-result content available; token delta cannot be counted exactly" + if proposal.edits + else "lever proposed no edits" + ) + return LedgerEntry( + lever=proposal.lever, session_id=session.id, agent=session.agent, + fidelity=FIDELITY_UNMEASURABLE if proposal.edits else FIDELITY_MEASURED, + diagnostics=dict(proposal.diagnostics), note=note, + ) + + fidelity = ( + FIDELITY_MEASURED if all(c.priced for c in costs) else FIDELITY_UNPRICED + ) + diagnostics = dict(proposal.diagnostics) + if unmeasured: + diagnostics["edits_unmeasured"] = unmeasured + return LedgerEntry( + lever=proposal.lever, session_id=session.id, agent=session.agent, + fidelity=fidelity, edits=tuple(costs), diagnostics=diagnostics, + note="" if fidelity == FIDELITY_MEASURED else "model has no catalog entry — unpriced, not free", + rate_sources=tuple((m, s, a) for m, (s, a) in sorted(sources.items())), + ) + + +def price_all( + pairs: Sequence[Tuple[Session, Proposal]], + ctx: LeverContext, + *, + rates_lookup: Callable[[str], Optional[Rates]] = rates_for, +) -> LedgerReport: + """Price many ``(session, proposal)`` pairs into one ranked report.""" + return LedgerReport( + entries=tuple( + price_proposal(s, p, ctx, rates_lookup=rates_lookup) for s, p in pairs + ) + ) diff --git a/src/ace/sidecar/levers/protocol.py b/src/ace/sidecar/levers/protocol.py new file mode 100644 index 0000000..01d52c8 --- /dev/null +++ b/src/ace/sidecar/levers/protocol.py @@ -0,0 +1,205 @@ +"""ace.sidecar.levers.protocol — what a lever is, and what it is forbidden to do. + +A lever proposes; it never prices +--------------------------------- +:meth:`Lever.propose` returns :class:`Edit` objects and nothing else. It does not return +dollars, tokens saved, or a percentage. Pricing happens once, in the ledger, which owns the +tokenizer and the rate catalog and is the only thing that knows the provider's cache-write +premium. + +That split is not tidiness, it is the credibility of the number. A lever that reports its +own saving is a lever that can overstate it, and the three ways this arithmetic has already +gone wrong in this codebase were all self-reporting: + +* compaction savings denominated in whitespace words and priced per BPE token, which + undersold the leg by the word->BPE ratio (see ``ace.gateway.tokenizer``); +* a de-dup lever keyed on file path rather than full tool input, measuring $36.88 where the + provable version measures $0.33; +* the same avoided call counted twice — once as avoided, once as a counterfactual. + +With pricing centralized, a lever cannot commit any of them. It also means a lever needs no +knowledge of which provider it is running against, which is what lets one implementation +serve Claude Code, Antigravity and Codex. + +Counting must be exact, so the counter is injected +-------------------------------------------------- +The baseline side of every counterfactual is ground truth: the provider's own token counts, +read off the transcript. The proposed side is a prompt that was never sent, so its tokens +have to be produced — and an approximation there turns a measured claim into an estimate. + +:class:`TokenCounter` is therefore a seam with an exact implementation per model family: +Anthropic's ``POST /v1/messages/count_tokens`` for Claude (tiktoken is a *proxy* for +non-OpenAI models, not a truth), tiktoken for OpenAI models where it is that provider's own +BPE, and the provider's counting endpoint for Gemini. A lever calls ``ctx.count_tokens`` +and stays out of that decision. + +Modes +----- +``off`` / ``shadow`` / ``on`` mirror the cloud gateway's vocabulary on purpose, so the two +products' telemetry reads as one thing. ``shadow`` is the default and should stay the +default: this process sits in front of a developer's real coding session, and a lever that +silently rewrites a prompt owns every unexplained agent failure that follows. Shadow costs +nothing and proves the same number. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import ( + Any, + ClassVar, + Literal, + Mapping, + Optional, + Protocol, + Tuple, + runtime_checkable, +) + +from ace.sidecar.levers.types import Session + +__all__ = [ + "MODE_OFF", + "MODE_SHADOW", + "MODE_ON", + "MODES", + "RISK_NONE", + "RISK_LOW", + "RISK_MEDIUM", + "RISK_HIGH", + "EditKind", + "Edit", + "Proposal", + "TokenCounter", + "LeverContext", + "Lever", +] + +MODE_OFF = "off" +MODE_SHADOW = "shadow" +MODE_ON = "on" +MODES = (MODE_OFF, MODE_SHADOW, MODE_ON) + +# Same vocabulary as ``strategies.LEVER_RISK``, so a lever's declared risk and the rail's +# scored risk are comparable without a mapping table. +RISK_NONE, RISK_LOW, RISK_MEDIUM, RISK_HIGH = "NONE", "LOW", "MEDIUM", "HIGH" + +EditKind = Literal["truncate", "drop", "replace", "expire"] + + +@dataclass(frozen=True, slots=True) +class Edit: + """One proposed change to one tool result. + + Addressed positionally by ``(turn_index, call_index)`` rather than by tool-call id: + ids exist in Claude Code transcripts and not reliably elsewhere, and position is + unambiguous in every scanner's output. ``sig`` rides along for debugging and for the + ledger's audit line, never as the key. + + The four kinds split along a line that decides what an edit costs: + + ``truncate`` / ``drop`` / ``replace`` change bytes. Applied at the tail — to a result + that has not yet entered the cached prefix — they are free. Applied to history they + invalidate the cached prefix from that point on and the next turn pays a full cache + write, which is why the ledger nets that penalty before reporting anything. + + ``expire`` changes nothing about the bytes. It asserts the result stops being worth + keeping resident after ``live_until``, which is an accounting claim about residency, not + a smaller prompt. Volume levers help against a token cap; ``expire`` helps only against a + bill. Conflating the two is the easiest way to overstate this product, so the ledger + reports them separately and never sums them. + """ + + turn_index: int + call_index: int + kind: EditKind + reason: str + sig: str = "" + # truncate: bytes retained from the head (and, if the lever keeps a tail, from the end). + keep_bytes: Optional[int] = None + # replace: the substituted text. Only ever populated in actuation mode, where the lever + # was handed the original bytes to begin with. + replacement: Optional[str] = None + # expire: the last turn index at which this result is still worth holding in context. + live_until: Optional[int] = None + + +@dataclass(frozen=True, slots=True) +class Proposal: + """What one lever would do to one session. No savings figure — see the module docstring.""" + + lever: str + edits: Tuple[Edit, ...] = () + # Free-form counters a lever wants surfaced for debugging or for its dashboard row + # ("loops_detected": 3). Never priced, never summed into a saving. + diagnostics: Mapping[str, Any] = field(default_factory=dict) + + # No ``__bool__``. An edit-free proposal is a legitimate and important result: a loop + # guardrail's whole output is "I detected three runaway tool cycles" with nothing to + # rewrite, and the most valuable thing a truncation lever can report on a clean session + # is that it found nothing to do. Defining truthiness as "has edits" makes ``if + # proposal:`` quietly discard both. Callers test ``proposal.edits`` when they mean edits + # and ``proposal is None`` when they mean the lever declined or failed. + + +class TokenCounter(Protocol): + """Exact token count for ``text`` under ``model``. Must not approximate. + + Implementations may be slow and may do I/O — Anthropic's counting endpoint is a network + call. Levers should call it on whole segments rather than per word, and the runtime is + free to batch or sample across turns; that policy lives in the runtime, not here. + """ + + def __call__(self, text: str, *, model: str) -> int: ... + + +@dataclass(frozen=True, slots=True) +class LeverContext: + """Everything a lever is allowed to depend on. + + Deliberately small. It carries no rate catalog (levers do not price), no database, no + HTTP client and no agent identity beyond what ``Session.agent`` already says. A lever + needing something absent here is a lever reaching past its contract — extend this + dataclass rather than importing around it, so the dependency stays visible at the seam. + """ + + count_tokens: TokenCounter + mode: str = MODE_SHADOW + now: float = 0.0 + # Per-lever configuration from ``~/.ace/config.json``, already narrowed to this lever's + # own key. A lever must tolerate an empty mapping: the common case is a user who enabled + # it and tuned nothing. + settings: Mapping[str, Any] = field(default_factory=dict) + + +@runtime_checkable +class Lever(Protocol): + """One optimization, scored or applied against the normalized session model. + + Implementations live outside this repository. This protocol and + :mod:`ace.sidecar.levers.types` are the entire public surface they compile against, and + both are versioned as a contract: adding an optional field is fine, changing the meaning + of one is not. + + ``requires_content`` is the honest declaration of what a lever needs. A lever reading only + ``sig``/``digest``/``result_bytes`` scores from transcripts alone and therefore works for + every agent the sidecar can read, with no proxy and no hooks. A lever that must rewrite + text needs the bytes in hand, so the registry offers it only where an actuator supplied + them — today that is Claude Code's proxy and hook paths. Declaring ``False`` and then + calling ``ContentRef.resolve`` raises rather than silently degrading. + """ + + id: ClassVar[str] + label: ClassVar[str] + risk: ClassVar[str] + requires_content: ClassVar[bool] + + def propose(self, session: Session, ctx: LeverContext) -> Proposal: + """Edits this lever would make to ``session``. Must not mutate ``session``. + + Called on the measurement path for every session in a developer's history, so it is + expected to be cheap in the ``requires_content = False`` case and to raise nothing: + a lever that throws on one malformed session must not take the dashboard down with + it. The registry isolates failures, but a lever should not rely on that. + """ + ... diff --git a/src/ace/sidecar/levers/rail.py b/src/ace/sidecar/levers/rail.py new file mode 100644 index 0000000..b58a012 --- /dev/null +++ b/src/ace/sidecar/levers/rail.py @@ -0,0 +1,276 @@ +"""ace.sidecar.levers.rail — the dashboard's view of installed levers. + +Sits between the scanner and the renderer so neither has to know about levers. +``insights._build_payload`` calls :func:`rail_payload` and hands the result through; the +renderer reads it. Nothing here scans transcripts and nothing here writes HTML. + +What this is honest about +------------------------- +The rail already shows what each lever would be *worth* — ``strategies.standalone_levers``, +a byte-turn simulation over the developer's own sessions. That is a headroom estimate and it +is labelled one. This module adds the other half: what an installed lever, run for real, +actually measured. + +Those two numbers must never be confused, so :func:`rail_payload` reports a ``status`` that +says which of them exists, and the renderer is expected to show it. Four states, and three of +them mean "no live number": + +``no_package`` nothing registers against the ``ace.sidecar.levers`` entry-point group. + The ordinary state for the open-source sidecar on its own. +``all_off`` levers are installed but every one resolves to ``off`` in + ``~/.ace/config.json``. Presence is not consent; this is the default even + after installing a lever package. +``no_counter`` levers ran, but no exact token counter is configured, so the ledger priced + nothing. A byte-ratio fallback would produce a number here — which is why + there is none. +``measured`` real edits, exactly counted, priced from the catalog and net of the + cache-write penalty. + +Only ``measured`` may put a dollar figure on the page. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence + +from ace.sidecar.levers.counter import resolve_counter +from ace.sidecar.levers.ledger import FIDELITY_MEASURED, price_all +from ace.sidecar.levers.protocol import MODE_OFF, LeverContext, TokenCounter +from ace.sidecar.levers.registry import discover, load_settings, propose_safely, resolve_modes +from ace.sidecar.levers.types import from_corpus_sessions + +__all__ = ["STATUS_NO_PACKAGE", "STATUS_ALL_OFF", "STATUS_NO_COUNTER", "STATUS_MEASURED", + "resolve_counter", "rail_payload", "refresh_measured"] + +log = logging.getLogger(__name__) + +STATUS_NO_PACKAGE = "no_package" +STATUS_ALL_OFF = "all_off" +STATUS_NO_COUNTER = "no_counter" +STATUS_MEASURED = "measured" + +_STATUS_NOTE = { + STATUS_NO_PACKAGE: "no lever package installed — this release measures headroom only", + STATUS_ALL_OFF: "levers installed, all off — enable one in ~/.ace/config.json", + STATUS_NO_COUNTER: ( + "no exact token counter configured — the ledger prices nothing it cannot count" + ), + STATUS_MEASURED: "measured on your own sessions, net of the cache-write penalty", +} + +# Discovery walks installed distribution metadata, which is stable for the life of the +# process and is on the cached dashboard path. Re-walking it per request buys nothing. +_DISCOVERED: Optional[Sequence[Any]] = None + + +def _levers() -> Sequence[Any]: + global _DISCOVERED + if _DISCOVERED is None: + _DISCOVERED = discover() + return _DISCOVERED + + +def _measured(store: Any, *, since: Optional[float] = None) -> Dict[str, Any]: + """Aggregated live results from the telemetry store, or ``{}``. + + Tolerant of a store that predates the ``lever_turns`` table, or of no store at all: this + is an optional column on a dashboard that has to render either way, and an old + ``~/.ace/telemetry.db`` is the common case immediately after an upgrade. + """ + if store is None or not hasattr(store, "lever_summary"): + return {} + try: + summary = store.lever_summary(since=since) + except Exception: + log.debug("[levers] lever_summary failed", exc_info=True) + return {} + return summary if summary.get("by_lever") else {} + + +def _measured_note(prior_status: Optional[str]) -> str: + """The measured note, qualified by what discovery currently says. + + Recorded results and installed packages are two independent facts, and they disagree in + an ordinary way: a developer measures a lever for a week, then uninstalls or disables it. + Reporting ``no_package`` and dropping the rows would hide a real measurement behind a + packaging detail; reporting them unqualified would imply the lever is still running. + Both facts get said. + """ + if prior_status == STATUS_NO_PACKAGE: + return ( + _STATUS_NOTE[STATUS_MEASURED] + + " — from turns already recorded; no lever package is installed now" + ) + if prior_status == STATUS_ALL_OFF: + return ( + _STATUS_NOTE[STATUS_MEASURED] + + " — from turns already recorded; every installed lever is now off" + ) + return _STATUS_NOTE[STATUS_MEASURED] + + +def refresh_measured( + payload: Mapping[str, Any], store: Any, *, since: Optional[float] = None +) -> Dict[str, Any]: + """A rail payload with its live half re-read from ``store``. + + ``insights._build_payload`` is memoised on a transcript fingerprint, which is exactly + right for the installed/modes half — that changes when a package is installed, not when a + turn is proxied. The measured half moves on every turn, so caching it with the rest would + freeze the one number on the rail that is supposed to be alive. Same treatment + ``build`` already gives ``live``. + + Returns a copy. The cached payload is shared, and writing the live keys into it is how a + per-request value ends up served to the next caller. + """ + out = dict(payload) + measured = _measured(store, since=since) + if not measured: + return out + out["measured"] = measured + out["turns_observed"] = measured.get("turns_observed", 0) + out["status"] = STATUS_MEASURED + out["note"] = _measured_note(out.get("status")) + return out + + +def rail_payload( + sessions: Sequence[Mapping[str, Any]], + *, + counter: Optional[TokenCounter] = None, + counter_note: str = "", + credential: Optional[str] = None, + scheme: str = "api_key", + store: Any = None, + since: Optional[float] = None, + config: Optional[Mapping[str, Any]] = None, +) -> Dict[str, Any]: + """What the dashboard needs to render the live half of the lever rail. + + Runs only levers resolved to a non-``off`` mode, over the sessions already scoped to the + dashboard's range and agent filter. Cheap and total when nothing is installed, which is + the common path: one entry-point lookup and an early return. + + ``store`` is the sidecar's :class:`~ace.gateway.local_store.LocalStore`. It carries the + measured half, recorded turn by turn as levers ran on the proxy path, and reading it is + what makes a measured result survive the request that produced it. + + ``credential``/``scheme`` are for a caller that holds one — the proxy path, which is the + only place a credential exists at all under ``no_key: true``. The dashboard renders + outside any request and passes neither, so it falls back to the environment and usually + reports :data:`STATUS_NO_COUNTER` with the reason attached. That is the honest state, not + a degraded one: this half of the rail is measured on proxied turns. + """ + t0 = time.monotonic() + found = _levers() + base: Dict[str, Any] = { + "installed": [ + {"id": r.id, "label": getattr(r.lever, "label", r.id), + "risk": getattr(r.lever, "risk", ""), "dist": r.dist, + "requires_content": bool(getattr(r.lever, "requires_content", False))} + for r in found + ], + "modes": {}, + "by_lever": {}, + "entries": [], + "counter": counter_note, + "measured": {}, + "turns_observed": 0, + "elapsed_ms": 0.0, + } + # Read once, up front: measured rows are recorded history and stay true regardless of what + # is installed or enabled *right now*. Deciding `no_package` before looking would hide a + # real measurement behind a packaging detail. + measured_rows = _measured(store, since=since) + + def _terminal(status: str) -> Dict[str, Any]: + base.update(status=status, note=_STATUS_NOTE[status]) + if measured_rows: + base["measured"] = measured_rows + base["turns_observed"] = measured_rows.get("turns_observed", 0) + base.update(status=STATUS_MEASURED, note=_measured_note(status)) + base["elapsed_ms"] = (time.monotonic() - t0) * 1000.0 + return base + + if not found: + return _terminal(STATUS_NO_PACKAGE) + + modes = resolve_modes(found, config=config) + base["modes"] = modes + active = [r for r in found if modes.get(r.id, MODE_OFF) != MODE_OFF] + if not active: + return _terminal(STATUS_ALL_OFF) + + # The measured half is READ, not recomputed. Levers run once, against the real request + # body, in the proxy's background task (`levers.shadow`); this reads what they recorded. + # + # It cannot be derived here instead. The dashboard has transcripts — hashes and sizes, + # no text — and an exact token delta needs the bytes. Recomputing over history would + # force a byte-ratio fallback, which is the one thing the ledger refuses to do. + if measured_rows: + base["measured"] = measured_rows + base["turns_observed"] = measured_rows.get("turns_observed", 0) + base.update(status=STATUS_MEASURED, note=_STATUS_NOTE[STATUS_MEASURED]) + base["elapsed_ms"] = (time.monotonic() - t0) * 1000.0 + return base + + if counter is None: + counter, counter_note = resolve_counter(credential, scheme) + base["counter"] = counter_note + if counter is None: + base.update( + status=STATUS_NO_COUNTER, + note=f"{_STATUS_NOTE[STATUS_NO_COUNTER]} ({counter_note})", + ) + return base + + # The measurement path holds no tool-result bytes, so a lever needing them is refused by + # `propose_safely` rather than allowed to half-run. That is why a content-requiring lever + # can be installed, enabled, and still contribute nothing here: it needs the proxy or a + # hook to supply the text. + typed = from_corpus_sessions(sessions) + now = time.time() + pairs = [] + for reg in active: + ctx = LeverContext( + count_tokens=counter, + mode=modes[reg.id], + now=now, + settings=load_settings(reg.id, config=config), + ) + for s in typed: + proposal = propose_safely(reg, s, ctx) + if proposal is not None: + pairs.append((s, proposal)) + + # Priced under a context of its own rather than whichever lever's `ctx` the loop above + # happened to exit with. The ledger reads only `count_tokens`, so the leaked binding was + # harmless today — but it silently attributed one lever's `settings` and `mode` to every + # other lever's pricing, which is exactly the kind of thing that stops being harmless the + # first time the ledger reads one more field. + pricing_ctx = LeverContext(count_tokens=counter, now=now) + report = price_all(pairs, pricing_ctx) if pairs else None + if report is not None: + base["by_lever"] = report.by_lever() + base["entries"] = [ + {"lever": e.lever, "session": e.session_id, "agent": e.agent, + "fidelity": e.fidelity, "net_usd": e.net_usd, + "gross_usd": e.gross_saving_usd, "penalty_usd": e.cache_write_penalty_usd, + "removed_tokens": e.removed_tokens, "note": e.note} + for e in report.entries + ] + measured = any(e.fidelity == FIDELITY_MEASURED and e.edits for e in report.entries) + else: + measured = False + + base["elapsed_ms"] = (time.monotonic() - t0) * 1000.0 + if measured: + base.update(status=STATUS_MEASURED, note=_STATUS_NOTE[STATUS_MEASURED]) + else: + base.update( + status=STATUS_NO_COUNTER, + note="levers ran but produced no measurable edit on these sessions", + ) + return base diff --git a/src/ace/sidecar/levers/registry.py b/src/ace/sidecar/levers/registry.py new file mode 100644 index 0000000..bdff265 --- /dev/null +++ b/src/ace/sidecar/levers/registry.py @@ -0,0 +1,232 @@ +"""ace.sidecar.levers.registry — discovery, mode resolution, and failure isolation. + +Why discovery is by entry point +------------------------------- +This package defines what a lever *is*; it deliberately contains none. Implementations ship +in a separate distribution and register themselves:: + + [project.entry-points."ace.sidecar.levers"] + trajectory_compaction = "ace_skills.compaction:TrajectoryCompaction" + +Nothing here names a lever, imports one, or fails without one. That is the property being +bought: this repository stays a measurement product that gains optimizations when a package +providing them is present, and the package providing them needs no change here to land. + +A missing lever package is the ordinary case, not an error state. :func:`discover` returns +an empty tuple and every caller keeps working — the dashboard renders its measured rail +exactly as it does today. + +The rule this module enforces +----------------------------- +**Presence supplies availability. It never supplies consent.** An installed lever resolves +to ``off`` unless the developer's own ``~/.ace/config.json`` says otherwise, and ``on`` +must be typed per lever. Nothing here defaults a lever to acting on a live session, and no +future default should: the sidecar sits in front of a real coding session, and the cost of +being wrong is a silent corruption three turns later that the developer has no way to +attribute. +""" + +from __future__ import annotations + +import json +import logging +import os +from dataclasses import dataclass +from typing import Any, Dict, Iterable, Mapping, Optional, Tuple + +from ace.sidecar.levers.protocol import ( + MODE_OFF, + MODE_SHADOW, + MODES, + Lever, + LeverContext, + Proposal, +) +from ace.sidecar.levers.types import Session + +__all__ = [ + "ENTRY_POINT_GROUP", + "CONFIG_PATH", + "RegisteredLever", + "discover", + "resolve_modes", + "load_settings", + "propose_safely", +] + +log = logging.getLogger(__name__) + +ENTRY_POINT_GROUP = "ace.sidecar.levers" + +# The same file ``ace.cli`` reads. Kept as a literal rather than imported: ``cli`` builds the +# app and would import this package, and a cycle for one path string is a poor trade. +CONFIG_PATH = os.path.expanduser("~/.ace/config.json") + + +@dataclass(frozen=True, slots=True) +class RegisteredLever: + """A discovered lever plus where it came from. + + ``dist`` is carried so the dashboard can say which package supplied a lever. A developer + seeing an optimization act on their session is entitled to know what installed it. + """ + + lever: Lever + dist: str = "" + + @property + def id(self) -> str: + return getattr(self.lever, "id", "") + + +def discover(group: str = ENTRY_POINT_GROUP) -> Tuple[RegisteredLever, ...]: + """Every lever advertised by an installed distribution. + + Each entry point is loaded independently and a broken one is skipped with a log line + rather than raised: one bad third-party package must not stop ``ace up``. The same + reasoning applies to an object that loads but does not satisfy :class:`Lever` — it is + dropped here, where the message can name the entry point, instead of failing later at a + call site that cannot. + """ + try: + from importlib.metadata import entry_points + except Exception: # pragma: no cover - importlib.metadata is stdlib on 3.12 + return () + + found: list[RegisteredLever] = [] + seen: set[str] = set() + try: + eps: Iterable[Any] = entry_points(group=group) + except Exception: + log.debug("lever entry-point lookup failed", exc_info=True) + return () + + for ep in eps: + try: + obj = ep.load() + except Exception: + log.warning("lever entry point %r failed to load; skipping", ep.name) + continue + # Both a class and a ready instance are accepted. A stateless lever is naturally a + # class; one holding a loaded model artifact is naturally an instance already built + # by the providing package. + try: + candidate = obj() if isinstance(obj, type) else obj + except Exception: + log.warning("lever %r failed to instantiate; skipping", ep.name) + continue + if not isinstance(candidate, Lever): + log.warning("lever %r does not satisfy the Lever protocol; skipping", ep.name) + continue + lever_id = getattr(candidate, "id", "") or ep.name + if lever_id in seen: + log.warning("duplicate lever id %r; keeping the first", lever_id) + continue + seen.add(lever_id) + dist = "" + try: + dist = ep.dist.name if ep.dist is not None else "" + except Exception: + pass + found.append(RegisteredLever(lever=candidate, dist=dist)) + return tuple(found) + + +def _read_config(path: Optional[str] = None) -> Mapping[str, Any]: + """``~/.ace/config.json``, or an empty mapping. A missing or unreadable file is not an error.""" + try: + with open(path or CONFIG_PATH, "r", encoding="utf-8") as fh: + data = json.load(fh) + return data if isinstance(data, Mapping) else {} + except Exception: + return {} + + +def _lever_config(config: Mapping[str, Any]) -> Mapping[str, Any]: + raw = config.get("levers") + return raw if isinstance(raw, Mapping) else {} + + +def resolve_modes( + levers: Iterable[RegisteredLever], + *, + config: Optional[Mapping[str, Any]] = None, + config_path: Optional[str] = None, +) -> Dict[str, str]: + """``{lever_id: mode}`` for every discovered lever. + + Two spellings are accepted under ``levers`` in the config, because the short one is what + people actually type:: + + {"levers": {"trajectory_compaction": "shadow"}} + {"levers": {"trajectory_compaction": {"mode": "on", "keep_recent_images": 2}}} + + An unrecognised mode string resolves to ``off``, not to the default. A typo'd ``"On"`` + that silently became ``shadow`` would be tolerable; one that silently became ``on`` would + not, and the only rule that cannot get that backwards is to refuse the value outright. + """ + cfg = config if config is not None else _read_config(config_path) + per_lever = _lever_config(cfg) + out: Dict[str, str] = {} + for reg in levers: + entry = per_lever.get(reg.id) + if isinstance(entry, Mapping): + raw = entry.get("mode", MODE_OFF) + elif isinstance(entry, str): + raw = entry + elif entry is True: + # A bare `true` is an enablement, and the safe reading of "enabled" is the mode + # that changes nothing about the request. + raw = MODE_SHADOW + else: + raw = MODE_OFF + mode = str(raw).lower() + if mode not in MODES: + log.warning("lever %r has unknown mode %r; treating as off", reg.id, raw) + mode = MODE_OFF + out[reg.id] = mode + return out + + +def load_settings( + lever_id: str, + *, + config: Optional[Mapping[str, Any]] = None, + config_path: Optional[str] = None, +) -> Mapping[str, Any]: + """This lever's own configuration block, minus ``mode``, ready for :class:`LeverContext`.""" + cfg = config if config is not None else _read_config(config_path) + entry = _lever_config(cfg).get(lever_id) + if not isinstance(entry, Mapping): + return {} + return {k: v for k, v in entry.items() if k != "mode"} + + +def propose_safely( + reg: RegisteredLever, session: Session, ctx: LeverContext +) -> Optional[Proposal]: + """Run one lever, returning ``None`` where it raised. + + The measurement path calls every lever over a developer's entire history, which is the + widest input any of this code sees and the likeliest place for a third-party lever to + meet a session shape it did not expect. One such session must cost that lever's row, not + the dashboard. + + A lever declaring ``requires_content`` is refused outright on a measure-only session + rather than allowed to raise :class:`~ace.sidecar.levers.types.ContentUnavailable` part + way through — a partial proposal is worse than none, because the ledger cannot tell it + from a complete one. + """ + if getattr(reg.lever, "requires_content", False) and not any( + call.has_content for _, _, call in session.iter_calls() + ): + return None + try: + proposal = reg.lever.propose(session, ctx) + except Exception: + log.warning("lever %r raised on session %r; skipping", reg.id, session.id, exc_info=True) + return None + if not isinstance(proposal, Proposal): + log.warning("lever %r returned %s; skipping", reg.id, type(proposal)) + return None + return proposal diff --git a/src/ace/sidecar/levers/shadow.py b/src/ace/sidecar/levers/shadow.py new file mode 100644 index 0000000..88b8758 --- /dev/null +++ b/src/ace/sidecar/levers/shadow.py @@ -0,0 +1,736 @@ +"""ace.sidecar.levers.shadow — running levers for real, on a live proxied turn. + +This is where "measured" stops being a promise. Everything else in this package either scores +transcripts (hashes and sizes, no text) or defines the contract; here the actual tool-result +bytes are in hand, the credential to count them is in flight, and the provider has just +reported what the turn really cost. + +The three things this path has that the transcript path does not +--------------------------------------------------------------- +1. **The bytes.** A ``/v1/messages`` request body carries every tool result verbatim. That is + what lets a ``requires_content`` lever run at all, and what lets the token delta be + *counted* rather than inferred from ``result_bytes / 4``. +2. **A credential.** Under ``no_key: true`` — the sidecar's own default — the OAuth token the + proxy is about to relay is the only credential in the building. See + :mod:`ace.sidecar.levers.counter`. +3. **Ground-truth usage.** The response says exactly how this turn's prompt was billed across + fresh input, cache reads and cache writes. That split is what turns a token delta into a + dollar figure without assuming anything. + +Shadow means shadow +------------------- +The relayed bytes are never touched. Levers run against a **copy**, the counterfactual is +counted, priced and recorded, and the request the developer's agent actually made goes +upstream byte-for-byte — the invariant ``ace.gateway.messages`` exists to enforce ("parse to +decide, never parse to forward"). A lever resolved to ``on`` still does not mutate the request +here; actuation is a separate seam and deliberately not this one. + +It also runs **after** the response, in a worker thread, so it costs the developer's turn +nothing. Counting is a network round trip and the counter is a synchronous client; doing +either on the hot path would trade a real latency regression for a number nobody asked to +wait for. + +What is measured, and what is honestly not +------------------------------------------ +Measured: the token delta between the real request body and the counterfactual, both counted +the same way through the provider's own counter. A constant offset in either count cancels; +the difference is exact. + +Priced: that delta against **this turn's own usage split**, newest-bucket-first — see +:func:`price_delta`. No projection over future turns. A transcript-driven lever multiplies its +saving by the turns that carried the bytes, because it can see how the session ended; a live +turn cannot, and inventing that multiplier is how a one-turn saving becomes a headline number +that never arrives. + +Not priced: ``expire`` edits, which change no bytes and therefore no prompt (see +``protocol.Edit``), and the cache-write penalty of an edit that rewrites already-cached +history, which is only observable on the *following* turn. Both are recorded and labelled +rather than guessed at. +""" + +from __future__ import annotations + +import json +import logging +import time +from dataclasses import dataclass, field +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +from ace.sidecar.levers.protocol import MODE_OFF, Edit, LeverContext, Proposal +from ace.sidecar.levers.registry import ( + RegisteredLever, + discover, + load_settings, + propose_safely, + resolve_modes, +) +from ace.sidecar.levers.types import ContentRef, Session, ToolCall, Turn, Usage + +__all__ = [ + "TOOL_RESULT", + "TOOL_USE", + "Anchor", + "LiveEdit", + "TurnMeasurement", + "body_to_session", + "apply_edits", + "price_delta", + "ShadowRunner", +] + +log = logging.getLogger(__name__) + +TOOL_USE = "tool_use" +TOOL_RESULT = "tool_result" + +_MTOK = 1_000_000.0 + +# What a dropped result is replaced by. Matches ``ledger._POINTER_TOKENS`` in intent: a +# reference, not nothing, because the applied lever does leave a marker behind and claiming +# otherwise would overstate the saving by exactly the marker. +_POINTER_TEXT = "[tool result elided by an ACE lever]" + + +@dataclass(frozen=True, slots=True) +class Anchor: + """Where one ``ToolCall`` lives in the request body, so an edit can be applied back. + + The typed model addresses calls by ``(turn_index, call_index)`` — stable and + agent-neutral — while the body addresses them by ``messages[m]["content"][b]``. Levers + only ever see the former, so something has to hold the mapping; keeping it beside the + session rather than inside it is what stops the wire format leaking into the contract + every lever compiles against. + """ + + msg_index: int + block_index: int + # True for the newest result in the body. Its bytes have not yet been written to the + # provider's cache, so editing it is prefix-safe and carries no invalidation penalty. + is_tail: bool = False + + +@dataclass(frozen=True, slots=True) +class LiveEdit: + """One edit, priced against the turn it would have changed.""" + + lever: str + kind: str + reason: str + turn_index: int + call_index: int + prefix_safe: bool = True + applied: bool = True + note: str = "" + + +@dataclass(frozen=True, slots=True) +class TurnMeasurement: + """One lever's counterfactual for one live turn. The row that gets persisted. + + ``removed_tokens`` is the whole point and is exact. ``usd`` is exact *for this turn* and + deliberately carries no forward projection. + """ + + lever: str + mode: str + model: str + request_id: str = "" + session_id: Optional[str] = None + ts: float = 0.0 + + baseline_tokens: int = 0 + counterfactual_tokens: int = 0 + removed_tokens: int = 0 + + # How the removed tokens were allocated against this turn's real usage buckets. + from_cache_write: int = 0 + from_input: int = 0 + from_cache_read: int = 0 + + usd: float = 0.0 + priced: bool = True + # The provider's own reported prompt size, kept purely as a cross-check on the baseline + # count. Never used as an operand — see ``counter.count_body``. + reported_prompt_tokens: int = 0 + + edits: Tuple[LiveEdit, ...] = () + diagnostics: Mapping[str, Any] = field(default_factory=dict) + note: str = "" + elapsed_ms: float = 0.0 + + @property + def measured(self) -> bool: + return self.priced and self.removed_tokens > 0 + + +# --------------------------------------------------------------------------------------- +# Wire format -> typed model +# --------------------------------------------------------------------------------------- + + +def _blocks(content: Any) -> List[Any]: + """A message's content as a block list. A bare string is one implicit text block.""" + if isinstance(content, list): + return content + return [] if content is None else [content] + + +def _result_text(content: Any) -> str: + """A tool result's textual payload, images excluded. + + Mirrors ``insights._digest``'s treatment: an image's base64 is never part of the text a + truncation lever reasons about, and two screenshots of the same page are never + byte-identical anyway. + """ + if isinstance(content, str): + return content + if not isinstance(content, list): + return "" if content is None else json.dumps(content, default=str) + parts: List[str] = [] + for blk in content: + if isinstance(blk, str): + parts.append(blk) + elif isinstance(blk, dict): + if blk.get("type") == "image": + continue + t = blk.get("text") + parts.append(t if isinstance(t, str) else json.dumps(blk, default=str)) + return "".join(parts) + + +def body_to_session( + body: Mapping[str, Any], + *, + session_id: str = "", + agent: str = "claude", + usage: Optional[Usage] = None, +) -> Tuple[Session, Dict[Tuple[int, int], Anchor]]: + """Adapt one ``/v1/messages`` request body into the model levers read. + + Sibling of ``types.from_corpus_session``, and deliberately not a replacement for it: that + one adapts a finished transcript (hashes and sizes, every turn's usage known), this one + adapts a request in flight (real bytes, only the current turn's usage knowable). Levers + cannot tell the difference, which is the property that lets one lever serve both paths. + + A "turn" here is an assistant message that made tool calls, paired with the results that + came back in the following user message. Historical turns carry an empty :class:`Usage` + — the body does not record what they were billed — so the live path prices with + :func:`price_delta` rather than the transcript ledger, which needs those numbers. + + ``sig``/``target``/``digest`` are computed by ``insights``' own helpers rather than + reimplemented. Two hashing conventions for the same quantity is precisely how a lever + ends up scoring one thing on transcripts and a different thing live. + """ + # Lazy: `insights` is a 2,700-line dashboard module and this is reached from the gateway. + # One-time cost, off the hot path, and it buys a single definition of these hashes. + from ace.sidecar.insights import _digest, _measure, _sig, _target + + messages = body.get("messages") + messages = messages if isinstance(messages, list) else [] + + # tool_use id -> where its result landed, so a call can be joined to its result across + # the message boundary. Anthropic pairs them by id, which is reliable here (unlike in + # some transcript formats, where position is the only anchor available). + results: Dict[str, Tuple[int, int, Any]] = {} + for mi, msg in enumerate(messages): + if not isinstance(msg, dict) or msg.get("role") != "user": + continue + for bi, blk in enumerate(_blocks(msg.get("content"))): + if isinstance(blk, dict) and blk.get("type") == TOOL_RESULT: + tid = blk.get("tool_use_id") + if isinstance(tid, str): + results[tid] = (mi, bi, blk.get("content")) + + # The newest result in the body is the one this turn just produced: it has not yet been + # written to the provider's cache, so an edit to it is free of invalidation cost. + tail_id = "" + tail_pos = (-1, -1) + for tid, (mi, bi, _) in results.items(): + if (mi, bi) > tail_pos: + tail_pos, tail_id = (mi, bi), tid + + turns: List[Turn] = [] + anchors: Dict[Tuple[int, int], Anchor] = {} + + for msg in messages: + if not isinstance(msg, dict) or msg.get("role") != "assistant": + continue + calls: List[ToolCall] = [] + ti = len(turns) + for blk in _blocks(msg.get("content")): + if not (isinstance(blk, dict) and blk.get("type") == TOOL_USE): + continue + name = str(blk.get("name") or "?") + tool_input = blk.get("input") + tool_input = tool_input if isinstance(tool_input, dict) else {} + tid = blk.get("tool_use_id") or blk.get("id") + + found = results.get(tid) if isinstance(tid, str) else None + ci = len(calls) + if found is None: + # A call whose result is not in this body — the in-flight one, typically. + # Recorded so positions stay stable, but with nothing to edit. + calls.append( + ToolCall(name=name, sig=_sig(name, tool_input), + target=_target(tool_input), call_id=tid) + ) + continue + + mi, bi, content = found + anchors[(ti, ci)] = Anchor(mi, bi, is_tail=(tid == tail_id)) + calls.append( + ToolCall( + name=name, + sig=_sig(name, tool_input), + target=_target(tool_input), + digest=_digest(content), + result_bytes=_measure(content), + call_id=tid, + # THE difference from the transcript path: real bytes, bound lazily so a + # lever that only reads sizes never materializes them. + content=ContentRef(lambda c=content: c), + ) + ) + # Historical turns carry an empty Usage deliberately: the request body does not + # record what they were billed, and fabricating a plausible number is exactly the + # kind of input that would make the transcript ledger's arithmetic silently wrong. + turns.append( + Turn(index=ti, model=str(body.get("model") or ""), calls=tuple(calls)) + ) + + # The one turn whose billing IS known is this one, and it belongs to the tail. + if turns and usage is not None: + last = turns[-1] + turns[-1] = Turn(index=last.index, model=last.model, ts=last.ts, + stop_reason=last.stop_reason, usage=usage, calls=last.calls) + + return Session(id=session_id, agent=agent, turns=tuple(turns)), anchors + + +# --------------------------------------------------------------------------------------- +# Building the counterfactual +# --------------------------------------------------------------------------------------- + + +def _truncate_content(content: Any, keep_bytes: int) -> Any: + """A tool result kept to its first ``keep_bytes`` of text, structure preserved. + + Truncation is applied to the *text*, block by block, so a result that is a list of parts + stays a list of parts and an image inside it is dropped rather than sliced into + corruption. A lever asking to keep 4 KB of a screenshot is asking for something + meaningless; returning the blocks it can honour is better than returning bytes that no + longer parse. + """ + if keep_bytes <= 0: + return _POINTER_TEXT + if isinstance(content, str): + return content if len(content) <= keep_bytes else content[:keep_bytes] + if not isinstance(content, list): + return content + + out: List[Any] = [] + budget = keep_bytes + for blk in content: + if budget <= 0: + break + if isinstance(blk, str): + out.append(blk[:budget]) + budget -= min(len(blk), budget) + elif isinstance(blk, dict) and blk.get("type") == "image": + continue # cannot be partially kept; keeping it whole would defeat the cap + elif isinstance(blk, dict): + t = blk.get("text") + if isinstance(t, str): + nb = dict(blk) + nb["text"] = t[:budget] + out.append(nb) + budget -= min(len(t), budget) + else: + out.append(blk) + else: + out.append(blk) + return out or _POINTER_TEXT + + +def apply_edits( + body: Mapping[str, Any], + edits: Sequence[Edit], + anchors: Mapping[Tuple[int, int], Anchor], +) -> Tuple[Dict[str, Any], List[LiveEdit], int]: + """The counterfactual body, plus what actually landed in it. + + Copy-on-write down the edited path only. A deep copy would duplicate a multi-megabyte + agent prompt once per lever per turn, for the sake of changing a handful of strings. + + Returns ``(new_body, applied, skipped)``. An edit that addresses a call with no result in + this body, or that is an ``expire``, changes no bytes and is reported rather than dropped + — a lever whose every edit was skipped must not look like a lever that found nothing. + """ + out = dict(body) + messages = list(body.get("messages") or []) + touched_msgs: Dict[int, Dict[str, Any]] = {} + applied: List[LiveEdit] = [] + skipped = 0 + + for edit in edits: + anchor = anchors.get((edit.turn_index, edit.call_index)) + if anchor is None: + skipped += 1 + applied.append(LiveEdit( + lever="", kind=edit.kind, reason=edit.reason, + turn_index=edit.turn_index, call_index=edit.call_index, + applied=False, note="no tool result at this position in the request body", + )) + continue + if edit.kind == "expire": + # Changes residency, not bytes. Volume levers and accounting levers do not share + # a number here for the same reason the ledger keeps them apart. + skipped += 1 + applied.append(LiveEdit( + lever="", kind=edit.kind, reason=edit.reason, + turn_index=edit.turn_index, call_index=edit.call_index, + prefix_safe=anchor.is_tail, applied=False, + note="expire changes residency, not prompt bytes — not priced on this path", + )) + continue + + msg = touched_msgs.get(anchor.msg_index) + if msg is None: + src = messages[anchor.msg_index] + msg = dict(src) + msg["content"] = list(_blocks(src.get("content"))) + touched_msgs[anchor.msg_index] = msg + messages[anchor.msg_index] = msg + + blocks = msg["content"] + if not (0 <= anchor.block_index < len(blocks)): + skipped += 1 + continue + blk = blocks[anchor.block_index] + if not isinstance(blk, dict): + skipped += 1 + continue + + nb = dict(blk) + if edit.kind == "truncate": + if edit.keep_bytes is None: + skipped += 1 + continue + nb["content"] = _truncate_content(blk.get("content"), edit.keep_bytes) + elif edit.kind == "drop": + nb["content"] = _POINTER_TEXT + elif edit.kind == "replace": + nb["content"] = edit.replacement if edit.replacement is not None else _POINTER_TEXT + else: + skipped += 1 + continue + + blocks[anchor.block_index] = nb + applied.append(LiveEdit( + lever="", kind=edit.kind, reason=edit.reason, + turn_index=edit.turn_index, call_index=edit.call_index, + prefix_safe=anchor.is_tail, applied=True, + )) + + out["messages"] = messages + return out, applied, skipped + + +# --------------------------------------------------------------------------------------- +# Pricing one live turn +# --------------------------------------------------------------------------------------- + + +def price_delta(removed: int, usage: Usage, rates) -> Tuple[float, int, int, int]: + """Value ``removed`` tokens against the turn's own billed buckets. Newest first. + + Returns ``(usd, from_cache_write, from_input, from_cache_read)``. + + The allocation is the whole argument, so it is worth stating plainly. A tool result that + a lever trims sits at the **end** of the prompt, and the end of an agent prompt is the + part that was not served from cache: it is either fresh input or the content being + written to the cache for the next turn. The cached prefix in front of it is older + material the edit never touches. So removed tokens are drawn from + ``cache_write -> input -> cache_read``, in that order, and each bucket is valued at the + rate the provider actually charged for it. + + This matters by an order of magnitude and in the direction that flatters the product, + which is why it is derived rather than assumed. Valuing everything at the cache-read rate + (~0.1x) understates a tail truncation roughly tenfold; valuing everything at the write + rate (1.25x) overstates a history rewrite by about the same. Both numbers are wrong. The + turn's own usage split is the only thing here that is ground truth, and it is free. + + Falls back to the cache-read rate — the conservative end — once the newer buckets are + exhausted, which is what happens when an edit really does reach into cached history. + """ + if removed <= 0 or rates is None: + return 0.0, 0, 0, 0 + + left = removed + from_write = min(left, max(0, usage.cache_write_tokens)) + left -= from_write + from_input = min(left, max(0, usage.input_tokens)) + left -= from_input + from_read = max(0, left) + + ttl = "5m" + if usage.cache_write_by_ttl: + ttl = max(usage.cache_write_by_ttl.items(), key=lambda kv: kv[1])[0] + + usd = ( + from_write / _MTOK * rates.cache_write_per_mtok(ttl) + + from_input / _MTOK * rates.input_per_mtok + + from_read / _MTOK * rates.cache_read_per_mtok + ) + return usd, from_write, from_input, from_read + + +# --------------------------------------------------------------------------------------- +# Orchestration +# --------------------------------------------------------------------------------------- + + +class ShadowRunner: + """Runs enabled levers against live turns and hands the results to a sink. + + Holds the discovered levers and the counter for the life of the process. Discovery walks + installed distribution metadata, which cannot change under a running process, and the + counter carries the one fact worth remembering across turns — whether its credential is + accepted at all. + + Cheap and total when nothing is installed. That is the ordinary state for the open-source + sidecar, and it must cost a proxied turn nothing measurable: :meth:`enabled` is one cached + entry-point lookup and a dict comparison. + """ + + def __init__( + self, + *, + config: Optional[Mapping[str, Any]] = None, + sink=None, + counter=None, + agent: str = "claude", + ) -> None: + self._config = config + self._sink = sink + self._counter = counter + self._agent = agent + self._levers: Optional[Tuple[RegisteredLever, ...]] = None + self._modes: Optional[Dict[str, str]] = None + + # -- wiring ------------------------------------------------------------------------ + + def _discovered(self) -> Tuple[RegisteredLever, ...]: + if self._levers is None: + try: + self._levers = discover() + except Exception: # pragma: no cover - discovery isolates its own failures + log.debug("[levers] discovery failed", exc_info=True) + self._levers = () + return self._levers + + def modes(self) -> Dict[str, str]: + if self._modes is None: + self._modes = resolve_modes(self._discovered(), config=self._config) + return self._modes + + def active(self) -> List[RegisteredLever]: + modes = self.modes() + return [r for r in self._discovered() if modes.get(r.id, MODE_OFF) != MODE_OFF] + + @property + def enabled(self) -> bool: + """Whether any lever would run. The early-out every proxied turn hits first.""" + return bool(self._discovered()) and bool(self.active()) + + def set_counter(self, counter) -> None: + """Adopt a counter built from the in-flight credential, once. + + Kept for the life of the process rather than rebuilt per turn: a rebuilt counter + would forget that the credential had already been refused and re-ask the counting + endpoint on every single turn. + """ + if self._counter is None and counter is not None: + self._counter = counter + + @property + def counter(self): + return self._counter + + # -- the measurement --------------------------------------------------------------- + + def observe( + self, + body: Mapping[str, Any], + usage: Usage, + *, + model: str = "", + request_id: str = "", + session_id: Optional[str] = None, + rates=None, + ) -> List[TurnMeasurement]: + """Measure every enabled lever against one completed turn. Blocking; call off-thread. + + Never raises. This runs after a response has already been served, and there is no + failure here worth converting into a developer-visible error. + """ + out: List[TurnMeasurement] = [] + levers = self.active() + if not levers or self._counter is None: + return out + + t0 = time.monotonic() + try: + session, anchors = body_to_session( + body, session_id=session_id or "", agent=self._agent, usage=usage + ) + except Exception: + log.debug("[levers] could not adapt request body", exc_info=True) + return out + + if rates is None: + try: + from ace.gateway.pricing import rates_for + + rates = rates_for(model or str(body.get("model") or "")) + except Exception: + rates = None + + # One baseline count, shared by every lever. Counting it per lever would multiply the + # network cost by the number installed for an answer that cannot differ. + try: + baseline = self._counter.count_body(body) + except Exception as exc: + log.debug("[levers] baseline count failed: %s", exc) + return out + + modes = self.modes() + for reg in levers: + m = self._measure_one( + reg, session, anchors, body, usage, baseline, modes.get(reg.id, MODE_OFF), + model=model or str(body.get("model") or ""), + request_id=request_id, session_id=session_id, rates=rates, + ) + if m is not None: + out.append(m) + + if out: + log.debug( + "[levers] measured %d lever(s) in %.0fms", + len(out), (time.monotonic() - t0) * 1000.0, + ) + return out + + def _measure_one( + self, + reg: RegisteredLever, + session: Session, + anchors: Mapping[Tuple[int, int], Anchor], + body: Mapping[str, Any], + usage: Usage, + baseline: int, + mode: str, + *, + model: str, + request_id: str, + session_id: Optional[str], + rates, + ) -> Optional[TurnMeasurement]: + t0 = time.monotonic() + ctx = LeverContext( + count_tokens=self._counter, + mode=mode, + now=time.time(), + settings=load_settings(reg.id, config=self._config), + ) + proposal = propose_safely(reg, session, ctx) + if proposal is None: + return None + + base_row = dict( + lever=reg.id, mode=mode, model=model, request_id=request_id, + session_id=session_id, ts=time.time(), + baseline_tokens=baseline, reported_prompt_tokens=usage.prompt_tokens, + diagnostics=dict(proposal.diagnostics), + ) + + if not proposal.edits: + # A real and useful result, not a failure — a loop guardrail's entire output is + # its diagnostics. Recorded so the rail can show the lever ran and found nothing. + return TurnMeasurement( + **base_row, counterfactual_tokens=baseline, + note="lever proposed no edits", + elapsed_ms=(time.monotonic() - t0) * 1000.0, + ) + + try: + new_body, applied, skipped = apply_edits(body, proposal.edits, anchors) + except Exception: + log.debug("[levers] %r: counterfactual body failed", reg.id, exc_info=True) + return None + + edits = tuple( + LiveEdit(lever=reg.id, kind=e.kind, reason=e.reason, turn_index=e.turn_index, + call_index=e.call_index, prefix_safe=e.prefix_safe, + applied=e.applied, note=e.note) + for e in applied + ) + if not any(e.applied for e in edits): + return TurnMeasurement( + **base_row, counterfactual_tokens=baseline, edits=edits, + note="lever proposed edits, none of which changed prompt bytes", + elapsed_ms=(time.monotonic() - t0) * 1000.0, + ) + + try: + counterfactual = self._counter.count_body(new_body) + except Exception as exc: + return TurnMeasurement( + **base_row, counterfactual_tokens=0, edits=edits, priced=False, + note=f"counterfactual could not be counted: {exc}", + elapsed_ms=(time.monotonic() - t0) * 1000.0, + ) + + removed = max(0, baseline - counterfactual) + usd, w, i, r = price_delta(removed, usage, rates) + note = "" if rates is not None else "model has no catalog entry — unpriced, not free" + if any(e.applied and not e.prefix_safe for e in edits): + # The invalidation cost lands on the NEXT turn's cache write, which has not + # happened yet. Saying so is the difference between a net figure and a gross one + # wearing a net figure's label. + note = (note + "; " if note else "") + ( + "touches already-cached history — the cache-write penalty falls on the next " + "turn and is not netted here" + ) + + return TurnMeasurement( + **base_row, + counterfactual_tokens=counterfactual, + removed_tokens=removed, + from_cache_write=w, from_input=i, from_cache_read=r, + usd=usd, priced=rates is not None, + edits=edits, note=note, + elapsed_ms=(time.monotonic() - t0) * 1000.0, + ) + + # -- the async entry point the proxy uses ------------------------------------------ + + async def observe_async(self, *args, **kwargs) -> List[TurnMeasurement]: + """:meth:`observe` on a worker thread, results handed to the sink. + + The counter is a synchronous HTTP client; awaiting it directly would block the event + loop that is serving every other turn. Runs detached, after the response. + """ + import asyncio + + try: + rows = await asyncio.to_thread(self.observe, *args, **kwargs) + except Exception: # pragma: no cover - a shadow run never surfaces + log.debug("[levers] shadow run failed", exc_info=True) + return [] + if rows and self._sink is not None: + try: + self._sink(rows) + except Exception: + log.debug("[levers] shadow sink failed", exc_info=True) + return rows diff --git a/src/ace/sidecar/levers/types.py b/src/ace/sidecar/levers/types.py new file mode 100644 index 0000000..176a40a --- /dev/null +++ b/src/ace/sidecar/levers/types.py @@ -0,0 +1,278 @@ +"""ace.sidecar.levers.types — the normalized session model every lever reads. + +Why this layer exists +--------------------- +``insights._scan``, ``_scan_antigravity`` and ``_scan_codex`` already emit one shape — +"corpus-shaped sessions" — from three completely different on-disk formats. That shape is +the only agent-agnostic thing in the codebase, and it is what makes one lever work for +Claude Code, Antigravity and Codex without knowing which produced the session. + +So a lever is defined against **this** model and never against a provider's wire format. A +lever that took an Anthropic ``messages[]`` body would work for exactly one of the three +agents and would have to be rewritten for the fourth. Adding an agent is then a scanner, +not a lever change. + +Hashes and sizes, not content +----------------------------- +The corpus is deliberately "numbers and hashes only": ``target`` and ``digest`` are +truncated SHA-256, ``result_bytes`` is a size. That is a privacy property worth keeping — +the dashboard reads a developer's whole transcript history and nothing about it needs the +text. + +It is also sufficient for the entire measurement half. Every lever in ``strategies.py`` +(read de-dup, supersede, age-out, truncate) decides purely on ``sig``/``digest``/ +``result_bytes``, which is why they can be scored on transcripts alone. Only *actuation* +needs bytes — you cannot truncate text you do not have — so content arrives through the +optional :class:`ContentRef` seam, resolved lazily and only in the proxy/hook path where +the bytes are in hand anyway. ``ContentRef`` is ``None`` in measure-only mode, and a lever +that declares ``requires_content`` is simply not offered there. + +Provider neutrality in the usage record +--------------------------------------- +:class:`Usage` carries a total ``cache_write_tokens`` plus a ``by_ttl`` breakdown rather +than Anthropic's ``ephemeral_5m``/``ephemeral_1h`` field names. The cache-write premium is +a *provider* property — Anthropic charges 1.25x at the 5-minute TTL and 2x at one hour; +other providers price prefix reuse differently and some charge no write premium at all. +Pricing that difference is the ledger's job (see ``ace.gateway.pricing``); the lever must +never see it, or a lever tuned against one provider's cache economics will quietly give +wrong answers on another. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Tuple + +__all__ = [ + "ContentUnavailable", + "ContentRef", + "ToolCall", + "Usage", + "Turn", + "Session", + "from_corpus_session", + "from_corpus_sessions", +] + + +class ContentUnavailable(RuntimeError): + """Raised when a lever asks for bytes that this run does not have. + + Reaching this is a wiring bug, not a runtime condition: a lever declaring + ``requires_content = True`` must never be handed a measure-only session. The registry + filters on that flag, so this exception exists to make a filtering mistake loud rather + than to be caught. + """ + + +@dataclass(frozen=True, slots=True) +class ContentRef: + """A lazy handle to a tool result's actual bytes. + + Deliberately not the bytes themselves. A session can hold thousands of tool results and + the measurement path wants none of them; materializing every result to run a lever that + only reads sizes would turn a transcript scan into a memory problem for no gain. + """ + + _resolve: Optional[Callable[[], Any]] = None + + @property + def available(self) -> bool: + return self._resolve is not None + + def resolve(self) -> Any: + """The result body, in whatever shape the agent recorded it (str or list-of-parts).""" + if self._resolve is None: + raise ContentUnavailable( + "tool result content is not available in measure-only mode" + ) + return self._resolve() + + +@dataclass(frozen=True, slots=True) +class ToolCall: + """One tool invocation and the result it put into context. + + ``sig`` vs ``target`` is the distinction that decides whether a de-dup lever is worth + anything. ``target`` hashes the primary path argument alone, so three disjoint slices + of one file share it; ``sig`` hashes the *whole* input, so ``offset``/``limit`` + participate. Keying de-dup on ``target`` measured $36.88 of headroom on the reference + corpus where keying on ``sig`` + ``digest`` measures $0.33 — the same lever, two orders + of magnitude apart. Prefer ``sig``, and require ``digest`` equality before claiming + bytes are redundant. + """ + + name: str + sig: str + target: Optional[str] = None + digest: Optional[str] = None + result_bytes: int = 0 + call_id: Optional[str] = None + content: Optional[ContentRef] = None + + @property + def has_content(self) -> bool: + return self.content is not None and self.content.available + + +@dataclass(frozen=True, slots=True) +class Usage: + """One turn's billed token counts, as the provider reported them. + + These are ground truth — read off the transcript, not derived — which is what lets a + counterfactual be stated against a real bill instead of against an estimate. Anything a + lever *proposes* has to be counted separately and exactly (see + ``protocol.TokenCounter``); never infer the counterfactual by scaling these. + """ + + input_tokens: int = 0 + output_tokens: int = 0 + cache_read_tokens: int = 0 + cache_write_tokens: int = 0 + # TTL label -> tokens written at that TTL, e.g. {"5m": 12000, "1h": 0}. Empty when the + # provider reports no breakdown; the total above still stands on its own. + cache_write_by_ttl: Mapping[str, int] = field(default_factory=dict) + + @property + def prompt_tokens(self) -> int: + """Everything that was in the prompt this turn, cached or not. + + ``input_tokens`` already EXCLUDES the cached buckets on Anthropic, so this is a sum + and not a max. Subtracting ``cache_read`` from ``input`` to "correct" it under-reports + prompt volume — an easy bug with no visible symptom. + """ + return self.input_tokens + self.cache_read_tokens + self.cache_write_tokens + + +@dataclass(frozen=True, slots=True) +class Turn: + """One API request. Not one transcript record. + + Claude Code writes one record per content block and repeats the whole ``usage`` object + on each; the scanners join on message id before emitting. Counting per record instead + inflates prompt volume 1.95x and output 2.34x. A lever receives turns already joined and + must not try to re-derive them. + """ + + index: int + model: str = "" + ts: Optional[float] = None # epoch seconds + stop_reason: Optional[str] = None + usage: Usage = field(default_factory=Usage) + calls: Tuple[ToolCall, ...] = () + + +@dataclass(frozen=True, slots=True) +class Session: + """One agent session, normalized. The unit a lever reasons over. + + ``agent`` is one of ``insights.AGENTS`` ("claude", "antigravity", "codex"). A lever may + read it — some tool names are agent-specific — but must not require a particular value: + the whole point of this model is that a lever written today keeps working when a fourth + scanner lands. + """ + + id: str + agent: str + kind: str = "main" # "main" | "subagent" + parent: Optional[str] = None + turns: Tuple[Turn, ...] = () + + @property + def n_turns(self) -> int: + return len(self.turns) + + def iter_calls(self): + """``(turn_index, call_index, ToolCall)`` over the whole session, in order.""" + for t in self.turns: + for ci, call in enumerate(t.calls): + yield t.index, ci, call + + +def _usage_from_corpus(t: Mapping[str, Any]) -> Usage: + by_ttl: Dict[str, int] = {} + for label, key in (("5m", "ephemeral_5m_input_tokens"), ("1h", "ephemeral_1h_input_tokens")): + v = int(t.get(key) or 0) + if v: + by_ttl[label] = v + return Usage( + input_tokens=int(t.get("input_tokens") or 0), + output_tokens=int(t.get("output_tokens") or 0), + cache_read_tokens=int(t.get("cache_read_input_tokens") or 0), + cache_write_tokens=int(t.get("cache_creation_input_tokens") or 0), + cache_write_by_ttl=by_ttl, + ) + + +def from_corpus_session( + raw: Mapping[str, Any], + *, + content_for: Optional[Callable[[int, int], Optional[Callable[[], Any]]]] = None, +) -> Session: + """Adapt one ``insights`` session dict into the typed model. + + This function is the entire cost of supporting a new agent: write a scanner that emits + the corpus shape and every lever works on it unchanged. + + ``content_for(turn_index, call_index)`` is the actuation hook. It returns a zero-arg + callable producing that result's body, or ``None`` where the bytes are not held. Omit it + entirely for the measurement path — which is every transcript-driven caller — and every + ``ToolCall.content`` is ``None``. + """ + turns = [] + for i, t in enumerate(raw.get("turns") or []): + calls = [] + for ci, c in enumerate(t.get("calls") or []): + resolver = content_for(i, ci) if content_for is not None else None + calls.append( + ToolCall( + name=str(c.get("name") or "?"), + sig=str(c.get("sig") or ""), + target=c.get("target"), + digest=c.get("digest"), + result_bytes=int(c.get("result_bytes") or 0), + call_id=c.get("id"), + content=ContentRef(resolver) if resolver is not None else None, + ) + ) + turns.append( + Turn( + index=i, + model=str(t.get("model") or ""), + ts=_epoch(t.get("ts")), + stop_reason=t.get("stop_reason"), + usage=_usage_from_corpus(t), + calls=tuple(calls), + ) + ) + return Session( + id=str(raw.get("session") or ""), + agent=str(raw.get("agent_type") or ""), + kind=str(raw.get("kind") or "main"), + parent=raw.get("parent"), + turns=tuple(turns), + ) + + +def from_corpus_sessions(rows: Sequence[Mapping[str, Any]]) -> Tuple[Session, ...]: + """Measure-only adaptation of a whole scan. The common case.""" + return tuple(from_corpus_session(r) for r in rows) + + +def _epoch(v: Any) -> Optional[float]: + """Corpus timestamps are ISO strings; the model wants seconds. + + Kept local rather than imported from ``insights``: the dependency runs from insights + into this package, and reversing it for a six-line helper would make the two mutually + importable. + """ + if isinstance(v, (int, float)): + return float(v) + if not isinstance(v, str) or not v: + return None + import datetime + + try: + return datetime.datetime.fromisoformat(v.replace("Z", "+00:00")).timestamp() + except ValueError: + return None diff --git a/src/ace/sidecar/strategies.py b/src/ace/sidecar/strategies.py index 41aeab5..0d9f259 100644 --- a/src/ace/sidecar/strategies.py +++ b/src/ace/sidecar/strategies.py @@ -30,7 +30,36 @@ from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple -BYTES_PER_TOKEN = 4.0 +# Byte-equivalents per token, for converting the byte-turns `simulate` counts into the +# tokens and dollars `score` reports. +# +# Measured, not assumed. Against 4,512 text-only, single-tool-call results drawn from the +# local Claude Code corpus, the observed characters-per-token distribution is: +# +# p10 1.51 p25 1.84 median 2.16 p75 2.41 p90 2.64 p95 2.82 p99 4.34 +# +# The previous value of 4.0 sat at the **99th percentile** of that distribution — not a +# central estimate but its extreme tail. 4.0 is the familiar figure for English prose; agent +# tool output is code, JSON, logs, diffs and file paths, which tokenize far denser. The same +# mistake, in the same direction, is recorded in ``ace.gateway.tokenizer``: a long-context +# gate computed at ~1.33 tokens/word on content that is really up to 3.3. +# +# The measurement is derived per turn as ``(prompt[i+1] - prompt[i]) - output[i]``, which is +# ground truth on both sides but is contaminated upward on the token leg by anything else +# that entered the prompt between the two turns (injected reminders, re-read context). That +# contamination can only *reduce* the observed ratio, so the honest estimate lives in the +# upper tail rather than at the median. 2.8 is that tail (~p95). +# +# Direction of the correction matters: `score` divides by this constant, so a LOWER value +# reports MORE tokens and more dollars. Moving 4.0 -> 2.8 raises every byte-turn headroom +# figure on the rail by ~1.43x. That is the direction that warrants caution, which is why +# the value chosen is the conservative end of the measured band and not its median. +# +# COUPLED: ``insights._CHARS_PER_TOKEN`` must hold the same value. `_measure` converts an +# image's known token count *into* byte-equivalents by multiplying by it, and the division +# here converts back — so images round-trip exactly when the two agree and are mispriced by +# their ratio when they do not. See :func:`_check_image_bridge`. +BYTES_PER_TOKEN = 2.8 POINTER_BYTES = 120 WRITE_TOOLS = ("Edit", "Write", "NotebookEdit") TTL_SECONDS = 3600.0 @@ -356,6 +385,42 @@ def accounting(sessions: List[Dict[str, Any]], rates_for) -> Dict[str, float]: return dict(out) +_BRIDGE_CHECKED = False + + +def _check_image_bridge() -> None: + """Warn once if ``insights._CHARS_PER_TOKEN`` has drifted from :data:`BYTES_PER_TOKEN`. + + The two constants are a matched pair, not two opinions about the same quantity. + ``insights._measure`` prices an image at its real token count and multiplies by + ``_CHARS_PER_TOKEN`` purely to keep one unit flowing through the pipeline; the division in + :func:`score` undoes it. Any value works so long as both sides use the SAME one — and when + they diverge, every image-bearing result is mispriced by exactly their ratio, silently, + with no error and no visible symptom beyond a lever's number moving. + + Checked lazily rather than at import: ``insights`` imports this module, so a module-level + import here would close the cycle. + """ + global _BRIDGE_CHECKED + if _BRIDGE_CHECKED: + return + _BRIDGE_CHECKED = True + try: + from ace.sidecar.insights import _CHARS_PER_TOKEN + except Exception: + return + if abs(float(_CHARS_PER_TOKEN) - BYTES_PER_TOKEN) > 1e-9: + import logging + + logging.getLogger(__name__).warning( + "[strategies] insights._CHARS_PER_TOKEN=%s != BYTES_PER_TOKEN=%s — image-bearing " + "tool results are mispriced by %.2fx. Set them to the same value.", + _CHARS_PER_TOKEN, + BYTES_PER_TOKEN, + float(_CHARS_PER_TOKEN) / BYTES_PER_TOKEN, + ) + + def score( sessions: List[Dict[str, Any]], s: Strategy, @@ -374,6 +439,7 @@ def score( rate = r.cache_read_per_mtok break for lever, byte_turns in simulate(sess, s).items(): + _check_image_bridge() tok = byte_turns / BYTES_PER_TOKEN tokens[lever] += tok usd[lever] += tok / 1e6 * rate diff --git a/tests/test_lever_ledger.py b/tests/test_lever_ledger.py new file mode 100644 index 0000000..9bc04b4 --- /dev/null +++ b/tests/test_lever_ledger.py @@ -0,0 +1,160 @@ +"""The ledger's arithmetic — the place a lever's proposal becomes money. + +Every rate here is a stand-in, so these tests assert the *arithmetic* and not the shipped +price list. The counter is ``len(text) // 4`` for the same reason: exact expected token counts +are hand-checkable, which is what makes a wrong answer legible rather than merely red. + +The properties under test are the three ways this arithmetic has already gone wrong in this +codebase: ignoring the cache-write penalty, summing levers that overlap, and rendering an +unpriced model as free. +""" + +from __future__ import annotations + +import pytest + +from ace.gateway.pricing import Rates +from ace.sidecar import levers as L + +RATES = Rates(model="m", input_per_mtok=3.0, output_per_mtok=15.0, + cache_read_per_mtok=0.30, source="test", as_of="2026-08-27") +LOOKUP = lambda m: RATES if m == "m" else None # noqa: E731 + +CTX = L.LeverContext(count_tokens=lambda t, *, model: len(t) // 4, mode=L.MODE_SHADOW) + +BIG = 40_000 +DUMP = "x" * BIG # 10,000 tokens at 4 bytes/token + +TRUNCATE = L.Proposal("truncate_dumps", ( + L.Edit(turn_index=0, call_index=0, kind="truncate", reason="dump", keep_bytes=4_000), +)) +EXPIRE_AT_5 = L.Proposal("compaction", ( + L.Edit(turn_index=0, call_index=0, kind="expire", reason="stale", live_until=5), +)) + + +def session(n_turns=10, model="m", prompt=50_000, cache_read=45_000, with_content=True): + """`n` turns with one big Bash dump created at turn 0, prompt growing each turn.""" + turns = [] + for i in range(n_turns): + calls = ( + [{"name": "Bash", "sig": "s0", "digest": "d0", "result_bytes": BIG}] + if i == 0 else [] + ) + turns.append({ + "model": model, "ts": None, "input_tokens": 100, "output_tokens": 50, + "cache_read_input_tokens": 0 if i == 0 else cache_read + i * 2_000, + "cache_creation_input_tokens": prompt if i == 0 else 2_000, + "calls": calls, + }) + raw = {"session": "s1", "agent_type": "claude", "kind": "main", "turns": turns} + content = (lambda ti, ci: (lambda: DUMP)) if with_content else None + return L.from_corpus_session(raw, content_for=content) + + +# -- the prefix-safe case: a fresh dump, trimmed before it is ever cached ------------------ + + +def test_prefix_safe_truncation_is_measured_and_free_of_penalty(): + e = L.price_proposal(session(10), TRUNCATE, CTX, rates_lookup=LOOKUP) + assert e.fidelity == L.FIDELITY_MEASURED + c = e.edits[0] + assert c.removed_tokens == 10_000 - 1_000 # exact, counted, not inferred + assert c.apply_at == 1 and c.turns_carried == 9 + assert c.prefix_safe and c.invalidated_tokens == 0 + assert c.cache_write_penalty_usd == pytest.approx(0.0) + # Priced at the CACHE-READ rate: these tokens were resident in a cached prefix and + # re-read each turn at ~0.1x. Valuing them at the input rate would inflate the lever 10x. + assert c.gross_saving_usd == pytest.approx((9_000 / 1e6) * 0.30 * 9) + assert c.net_usd == pytest.approx(c.gross_saving_usd) + assert c.break_even_turn is None # in profit immediately + assert e.rate_sources == (("m", "test", "2026-08-27"),) + + +# -- the history-rewriting case: the penalty that makes a "saving" cost money -------------- + + +def test_editing_cached_history_carries_a_penalty_and_can_be_a_net_loss(): + c = L.price_proposal(session(10), EXPIRE_AT_5, CTX, rates_lookup=LOOKUP).edits[0] + assert c.apply_at == 6 and c.turns_carried == 4 + assert not c.prefix_safe, "editing cached history is never prefix-safe" + assert c.invalidated_tokens > 0 + assert c.cache_write_penalty_usd > 0.0 + assert c.gross_saving_usd == pytest.approx((10_000 / 1e6) * 0.30 * 4) + if c.net_usd < 0: + assert c.break_even_turn is not None and c.break_even_turn > 10, ( + "a losing edit must break even beyond the session it ran in" + ) + + +def test_the_ttl_is_read_from_the_turn_not_assumed(): + """`strategies.TTL_SECONDS` hard-codes one hour while Claude Code defaults to 5m, and the + two carry different write premiums (2x vs 1.25x).""" + s = L.from_corpus_session({ + "session": "s2", "agent_type": "claude", "turns": [ + dict(model="m", input_tokens=100, cache_read_input_tokens=0, + cache_creation_input_tokens=50_000, ephemeral_1h_input_tokens=50_000, + output_tokens=50, + calls=[{"name": "Bash", "sig": "s", "result_bytes": BIG}]), + *[dict(model="m", input_tokens=100, cache_read_input_tokens=45_000, + cache_creation_input_tokens=0, output_tokens=50, calls=[]) + for _ in range(9)], + ]}, content_for=lambda ti, ci: (lambda: DUMP)) + c = L.price_proposal(s, EXPIRE_AT_5, CTX, rates_lookup=LOOKUP).edits[0] + assert c.cache_write_ttl in ("5m", "1h") + + +# -- the three refusals ------------------------------------------------------------------- + + +def test_no_content_means_no_dollars_and_it_says_why(): + """The refusal that keeps a measured claim measured: no `result_bytes / 4` fallback.""" + e = L.price_proposal( + session(10, with_content=False), TRUNCATE, CTX, rates_lookup=LOOKUP + ) + assert e.fidelity == L.FIDELITY_UNMEASURABLE + assert e.edits == () + assert "cannot be counted exactly" in e.note + assert e.net_usd == pytest.approx(0.0) + + +def test_an_unpriced_model_reports_real_tokens_and_absent_dollars(): + """A silent $0.00 looks like a cost win. Tokens are real; dollars are absent.""" + e = L.price_proposal( + session(10, model="unknown-model"), TRUNCATE, CTX, rates_lookup=LOOKUP + ) + assert e.fidelity == L.FIDELITY_UNPRICED + assert not e.priced + assert e.removed_tokens == 9_000 + assert e.net_usd == 0.0 + assert "not free" in e.note + + +def test_a_lever_firing_on_the_last_turn_saves_nothing_and_says_so(): + e = L.price_proposal(session(1), TRUNCATE, CTX, rates_lookup=LOOKUP) + assert e.edits[0].turns_carried == 0 + assert e.edits[0].gross_saving_usd == pytest.approx(0.0) + + +def test_an_edit_free_proposal_is_measured_not_an_error(): + e = L.price_proposal( + session(10), L.Proposal("loop_guard", (), {"loops_detected": 3}), CTX, + rates_lookup=LOOKUP, + ) + assert e.fidelity == L.FIDELITY_MEASURED + assert e.diagnostics == {"loops_detected": 3} + assert not e.priced and e.note == "lever proposed no edits" + + +# -- the report ranks, and refuses to total ----------------------------------------------- + + +def test_the_report_ranks_and_has_no_total(): + """Two levers can target the same bytes; scored alone their figures overlap, so adding + them produces a number larger than anything they could jointly deliver.""" + s = session(10) + rep = L.price_all([(s, TRUNCATE), (s, EXPIRE_AT_5)], CTX, rates_lookup=LOOKUP) + assert not hasattr(rep, "total_usd") + ranked = rep.ranked() + assert ranked[0][1] >= ranked[1][1] + assert rep.unmeasured() == () diff --git a/tests/test_lever_shadow.py b/tests/test_lever_shadow.py new file mode 100644 index 0000000..81ff522 --- /dev/null +++ b/tests/test_lever_shadow.py @@ -0,0 +1,470 @@ +"""The live path: counting with the in-flight credential, and measuring a proxied turn. + +This is the half that makes "measured" true rather than simulated, so the tests are mostly +about the two things that could quietly make it false: + +* the relayed request must go upstream byte-for-byte, whatever a lever proposes; +* a token delta must be *counted*, and a credential the counting endpoint refuses must + produce an explanation rather than a blank. + +No live provider call is made anywhere here. The upstream relay and the counting endpoint are +both driven through ``httpx.MockTransport``, which is the same discipline the rest of this +route's suite uses. +""" + +from __future__ import annotations + +import asyncio +import json +import sqlite3 + +import httpx +import pytest +from fastapi import FastAPI + +from ace.gateway.local_store import LocalStore +from ace.gateway.messages import MessagesConfig, install_messages_route +from ace.gateway.messages_auth import MODE_LOOPBACK, AuthConfig +from ace.sidecar import levers as L +from ace.sidecar.levers import rail +from ace.sidecar.levers.counter import AnthropicCounter, resolve_counter +from ace.sidecar.levers.shadow import ( + ShadowRunner, + apply_edits, + body_to_session, + price_delta, +) +from ace.sidecar.levers.types import Usage + +DUMP = "ERROR line\n" * 3000 + + +def body(dump=DUMP): + """A Claude Code shaped turn: one tool call, one big result, a system prompt and tools.""" + return { + "model": "claude-sonnet-5", "max_tokens": 1024, "system": "You are a coding agent.", + "tools": [{"name": "Bash", "description": "run", "input_schema": {"type": "object"}}], + "messages": [ + {"role": "user", "content": "find the bug"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "tu_1", "name": "Bash", + "input": {"command": "cat big.log"}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "tu_1", "content": dump}]}, + ], + } + + +class Truncate: + id, label = "truncate_dumps", "Truncate large tool dumps" + risk, requires_content = L.RISK_LOW, False + + def propose(self, session, ctx): + cap = int(ctx.settings.get("keep_bytes", 2048)) + return L.Proposal(self.id, tuple( + L.Edit(turn_index=t, call_index=c, kind="truncate", reason="over cap", + keep_bytes=cap) + for t, c, x in session.iter_calls() if x.result_bytes > cap + ), {"scanned": session.n_turns}) + + +class LoopGuard: + id, label = "loop_guard", "Loop guardrail" + risk, requires_content = L.RISK_NONE, False + + def propose(self, session, ctx): + return L.Proposal(self.id, (), {"loops_detected": 2, "detail": "dropped: not a number"}) + + +def counting_transport(calls=None): + """A stand-in counting endpoint: tokens == serialized bytes / 4.""" + def handler(req): + if calls is not None: + calls.append(json.loads(req.content)) + return httpx.Response(200, json={"input_tokens": len(req.content) // 4}) + return httpx.MockTransport(handler) + + +def upstream_transport(seen=None): + def handler(req): + if seen is not None: + seen["body"] = req.content + seen["headers"] = dict(req.headers) + return httpx.Response(200, json={ + "id": "msg_1", "type": "message", "role": "assistant", + "model": "claude-sonnet-5", "content": [{"type": "text", "text": "ok"}], + "usage": {"input_tokens": 1200, "output_tokens": 90, + "cache_read_input_tokens": 30000, + "cache_creation_input_tokens": 12000}}) + return httpx.MockTransport(handler) + + +# -- the counter -------------------------------------------------------------------------- + + +def test_no_credential_is_an_ordinary_outcome_with_a_reason(): + """`no_key: true` is this sidecar's own default, so this is the common state.""" + counter, note = resolve_counter(None) + if counter is None: # no key exported in this environment + assert "no credential available" in note + else: # a developer who did export one + assert "count_tokens" in note + + +def test_an_oauth_token_is_presented_as_bearer_with_the_required_beta(): + """An OAuth token sent as `x-api-key` is rejected, and /v1/messages needs the beta.""" + seen = [] + def handler(req): + seen.append(dict(req.headers)) + return httpx.Response(200, json={"input_tokens": 7}) + c = AnthropicCounter("sk-ant-oat-tok", "bearer", + client=httpx.Client(transport=httpx.MockTransport(handler))) + assert c("hello", model="claude-sonnet-5") == 7 + assert "authorization" in seen[0] + assert "x-api-key" not in seen[0] + assert seen[0]["anthropic-beta"] == "oauth-2025-04-20" + + +def test_count_body_strips_fields_the_endpoint_rejects(): + calls = [] + c = AnthropicCounter("k", "api_key", client=httpx.Client(transport=counting_transport(calls))) + c.count_body({"model": "m", "messages": [], "system": "S", "tools": [], + "stream": True, "max_tokens": 5, "temperature": 0.7, "metadata": {}}) + assert sorted(calls[0]) == ["messages", "model", "system", "tools"] + + +def test_a_refused_credential_latches_and_explains(): + """The OAuth-vs-API-key question is settled here, as a side effect of the first count — + there is no preflight probe. A refusal must be remembered, not re-asked every turn.""" + n = {"i": 0} + def deny(req): + n["i"] += 1 + return httpx.Response(401, json={"error": "unauthorized"}) + c = AnthropicCounter("sk-ant-oat-tok", "bearer", + client=httpx.Client(transport=httpx.MockTransport(deny))) + for _ in range(3): + with pytest.raises(RuntimeError): + c("x", model="m") + assert n["i"] == 1, "a definitive refusal must stop the calling" + assert not c.usable + assert "OAuth token" in c.note and "401" in c.note + + +def test_a_throttle_does_not_disable_counting(): + """A 429 says nothing about the credential. Latching on it would look exactly like the + feature not working.""" + n = {"i": 0} + def throttle(req): + n["i"] += 1 + return httpx.Response(429, json={}) + c = AnthropicCounter("k", "api_key", + client=httpx.Client(transport=httpx.MockTransport(throttle))) + for _ in range(3): + with pytest.raises(Exception): + c("x", model="m") + assert n["i"] == 3 + assert c.usable + + +# -- wire format -> typed model ----------------------------------------------------------- + + +def test_body_adapts_to_the_same_model_a_transcript_does(): + s, anchors = body_to_session(body(), session_id="s1") + assert s.agent == "claude" and s.n_turns == 1 + (ti, ci, call), = list(s.iter_calls()) + assert (ti, ci) == (0, 0) and call.name == "Bash" + assert call.result_bytes == len(DUMP) + # THE difference from the transcript path: real bytes are in hand. + assert call.has_content and call.content.resolve() == DUMP + assert anchors[(0, 0)].is_tail + + +def test_the_newest_result_is_the_prefix_safe_one(): + b = body() + b["messages"] += [ + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "tu_2", "name": "Read", "input": {"file_path": "/a"}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "tu_2", "content": "short"}]}, + ] + _, anchors = body_to_session(b) + assert not anchors[(0, 0)].is_tail, "already cached history" + assert anchors[(1, 0)].is_tail, "produced this turn, not yet written to cache" + + +def test_historical_turns_carry_no_fabricated_usage(): + """The body does not record what earlier turns were billed, and inventing a plausible + number is what would make the ledger's arithmetic silently wrong.""" + u = Usage(input_tokens=1200, cache_read_tokens=30000, cache_write_tokens=12000) + b = body() + b["messages"] += [ + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "tu_2", "name": "Read", "input": {"file_path": "/a"}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "tu_2", "content": "short"}]}, + ] + s, _ = body_to_session(b, usage=u) + assert s.turns[0].usage.prompt_tokens == 0 + assert s.turns[-1].usage.prompt_tokens == u.prompt_tokens + + +# -- the counterfactual -------------------------------------------------------------------- + + +def test_apply_edits_never_mutates_the_request(): + """Shadow means shadow. The relayed body and the counterfactual share no mutated object.""" + b = body() + _, anchors = body_to_session(b) + new, applied, skipped = apply_edits( + b, [L.Edit(turn_index=0, call_index=0, kind="truncate", reason="r", keep_bytes=2048)], + anchors, + ) + assert len(b["messages"][2]["content"][0]["content"]) == len(DUMP) + assert len(new["messages"][2]["content"][0]["content"]) == 2048 + assert [e.applied for e in applied] == [True] and skipped == 0 + + +def test_a_list_shaped_result_stays_a_list(): + b = body(dump=None) + b["messages"][2]["content"][0]["content"] = [{"type": "text", "text": "y" * 9000}] + _, anchors = body_to_session(b) + new, _, _ = apply_edits( + b, [L.Edit(turn_index=0, call_index=0, kind="truncate", reason="r", keep_bytes=1000)], + anchors, + ) + out = new["messages"][2]["content"][0]["content"] + assert isinstance(out, list) and len(out[0]["text"]) == 1000 + + +def test_expire_changes_no_bytes_and_says_so(): + """Volume levers and accounting levers must not share a number.""" + b = body() + _, anchors = body_to_session(b) + new, applied, skipped = apply_edits( + b, [L.Edit(turn_index=0, call_index=0, kind="expire", reason="stale", live_until=3)], + anchors, + ) + assert skipped == 1 + assert not applied[0].applied + assert "not priced on this path" in applied[0].note + assert new["messages"][2]["content"][0]["content"] == DUMP + + +# -- pricing one live turn ------------------------------------------------------------------ + + +def test_removed_tokens_are_drawn_newest_bucket_first(): + """The allocation IS the pricing argument: the end of an agent prompt is the part that + was not served from cache, and the same delta is worth ~12x more coming out of a write.""" + from ace.gateway.pricing import Rates + r = Rates(model="m", input_per_mtok=3.0, output_per_mtok=15.0, + cache_read_per_mtok=0.30, source="t", as_of="x") + u = Usage(input_tokens=1200, cache_read_tokens=30000, cache_write_tokens=12000, + cache_write_by_ttl={"5m": 12000}) + + usd, w, i, rd = price_delta(5_000, u, r) + assert (w, i, rd) == (5_000, 0, 0) + assert usd == pytest.approx(5_000 / 1e6 * r.cache_write_per_mtok("5m")) + + _, w, i, rd = price_delta(13_000, u, r) + assert (w, i, rd) == (12_000, 1_000, 0) + + _, w, i, rd = price_delta(20_000, u, r) + assert (w, i, rd) == (12_000, 1_200, 6_800), "the overflow falls back to the cache rate" + + +def test_an_unpriced_model_yields_no_dollars(): + assert price_delta(5_000, Usage(cache_write_tokens=9_000), None) == (0.0, 0, 0, 0) + + +# -- the whole path, through the real route -------------------------------------------------- + + +def make_app(store, *, seen=None, config=None, levers=(Truncate, LoopGuard)): + runner = ShadowRunner( + config=config or {"levers": {"truncate_dumps": "shadow", "loop_guard": "shadow"}}, + sink=store.record_lever_turns, + ) + runner._levers = tuple(L.RegisteredLever(lever=k(), dist="ace-skills") for k in levers) + runner.set_counter( + AnthropicCounter("k", "api_key", client=httpx.Client(transport=counting_transport())) + ) + app = FastAPI() + install_messages_route( + app, + config=MessagesConfig(base_url="https://upstream.test", timeout_s=5), + client=httpx.AsyncClient(transport=upstream_transport(seen)), + auth_config=AuthConfig(mode=MODE_LOOPBACK, local_api_key="k"), + accountant=store, shadow=runner, + ) + return app, runner + + +async def drive(app, n=1, payload=None): + sent = json.dumps(payload or body()).encode() + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://t" + ) as c: + for _ in range(n): + r = await c.post("/v1/messages", content=sent, + headers={"x-api-key": "k", "content-type": "application/json"}) + assert r.status_code == 200 + return sent + + +@pytest.fixture +def store(tmp_path): + return LocalStore(str(tmp_path / "telemetry.db")) + + +async def settle(store, rows): + for _ in range(100): + await asyncio.sleep(0.01) + if store.lever_summary()["rows"] >= rows: + return + raise AssertionError(f"shadow rows never reached {rows}") + + +def test_the_relayed_bytes_are_untouched_whatever_a_lever_proposes(store): + """THE fidelity invariant. A lever that would strip 90% of the prompt must still leave + the request that actually goes upstream byte-identical.""" + seen = {} + app, _ = make_app(store, seen=seen) + + async def main(): + sent = await drive(app) + await settle(store, 2) + return sent + + sent = asyncio.run(main()) + assert seen["body"] == sent + assert store.lever_summary()["by_lever"], "the lever did run" + + +def test_a_proxied_turn_is_measured_persisted_and_ranked(store): + app, _ = make_app(store) + + async def main(): + await drive(app, n=3) + await settle(store, 6) + + asyncio.run(main()) + s = store.lever_summary() + assert s["rows"] == 6 and s["turns_observed"] == 3 + by = {r["lever"]: r for r in s["by_lever"]} + + trunc = by["truncate_dumps"] + assert trunc["turns"] == 3 and trunc["removed_tokens"] > 0 and trunc["usd"] > 0 + assert trunc["edits_applied"] == 3 + # The dump is new this turn, so it comes out of the cache-write bucket. + assert trunc["from_cache_write"] == trunc["removed_tokens"] + + # An edit-free lever is recorded, not dropped: "it ran and found nothing" is a result. + assert by["loop_guard"]["turns"] == 3 + assert by["loop_guard"]["removed_tokens"] == 0 + + assert "total_usd" not in s, "levers overlap; a total would exceed what they can deliver" + + +def test_the_store_keeps_numbers_only(store): + """The one invariant of this store. Diagnostics are third-party authored and are the + only field that could carry a developer's own text.""" + app, _ = make_app(store) + + async def main(): + await drive(app) + await settle(store, 2) + + asyncio.run(main()) + got = { + d for (d,) in sqlite3.connect(store.path).execute( + "SELECT DISTINCT diagnostics FROM lever_turns" + ) + } + assert json.dumps({"loops_detected": 2}) in got + assert not any("dropped: not a number" in (d or "") for d in got) + + +def test_real_spend_is_recorded_alongside_the_counterfactual(store): + """`turns` and `lever_turns` are siblings: one is what was billed, the other is a prompt + that was never sent. Both have to be there for the saving to mean anything.""" + app, _ = make_app(store) + + async def main(): + await drive(app, n=2) + await settle(store, 4) + + asyncio.run(main()) + assert store.summary()["turns"] == 2 + assert store.summary()["cost_usd"] > 0 + + +def test_the_rail_reports_measured_only_once_something_was_measured(store): + app, _ = make_app(store) + before = rail.rail_payload([], store=store) + assert before["status"] in (rail.STATUS_NO_PACKAGE, rail.STATUS_ALL_OFF) + assert before["measured"] == {} + + async def main(): + await drive(app) + await settle(store, 2) + + asyncio.run(main()) + after = rail.rail_payload([], store=store) + assert after["status"] == rail.STATUS_MEASURED + assert after["turns_observed"] == 1 + assert {r["lever"] for r in after["measured"]["by_lever"]} == { + "truncate_dumps", "loop_guard" + } + + +@pytest.mark.parametrize( + "discovered,expected_qualifier", + [ + ((), "no lever package is installed now"), + ("installed-but-off", "every installed lever is now off"), + ], +) +def test_recorded_results_survive_the_lever_being_removed( + store, monkeypatch, discovered, expected_qualifier +): + """Recorded results and installed packages are independent facts, and they disagree in an + ordinary way: a developer measures a lever for a week, then uninstalls or disables it. + + Dropping the rows would hide a real measurement behind a packaging detail; reporting them + unqualified would imply the lever is still running. Both facts have to be said. + + Discovery is pinned rather than left to the environment — whether a lever package happens + to be installed in the venv running the suite is not what this test is about. + """ + app, _ = make_app(store) + + async def main(): + await drive(app) + await settle(store, 2) + + asyncio.run(main()) + + if discovered == "installed-but-off": + discovered = (L.RegisteredLever(lever=Truncate(), dist="ace-skills"),) + monkeypatch.setattr(rail, "_DISCOVERED", discovered) + + payload = rail.rail_payload([], store=store, config={}) + assert payload["status"] == rail.STATUS_MEASURED + assert expected_qualifier in payload["note"] + assert payload["measured"]["by_lever"], "the rows are still reported" + + +def test_nothing_enabled_costs_the_turn_nothing(store): + """The ordinary state for the open-source sidecar: one cached entry-point lookup.""" + app, runner = make_app(store, config={"levers": {}}) + assert not runner.enabled + + async def main(): + await drive(app) + await asyncio.sleep(0.05) + + asyncio.run(main()) + assert store.lever_summary()["rows"] == 0 + assert store.summary()["turns"] == 1, "accounting still happened" diff --git a/tests/test_levers.py b/tests/test_levers.py new file mode 100644 index 0000000..cafc69d --- /dev/null +++ b/tests/test_levers.py @@ -0,0 +1,205 @@ +"""The lever contract: the normalized model, mode resolution, and failure isolation. + +Nothing here installs a real lever. The point of ``ace.sidecar.levers`` is that it defines +what a lever *is* and contains none, so these tests stand in local implementations and assert +the properties the package promises third-party code — above all that presence never implies +consent, and that one bad lever costs its own row rather than the dashboard. +""" + +from __future__ import annotations + +import logging + +import pytest + +from ace.sidecar import levers as L + +# One session, three agents' worth of shape. `agent_type` is "codex" deliberately: the model +# is agent-neutral and a test that only ever exercises Claude Code would not prove it. +RAW = { + "session": "s1", "agent_type": "codex", "kind": "main", + "turns": [ + {"model": "gpt-5", "ts": "2026-08-27T10:00:00Z", "input_tokens": 100, + "output_tokens": 20, "cache_read_input_tokens": 900, + "cache_creation_input_tokens": 400, "ephemeral_5m_input_tokens": 400, + "calls": [{"id": "t1", "name": "Bash", "sig": "abc", "target": "deadbeef", + "digest": "d1", "result_bytes": 240000}]}, + {"model": "gpt-5", "ts": "2026-08-27T10:01:00Z", "input_tokens": 50, + "output_tokens": 10, "cache_read_input_tokens": 1400, + "cache_creation_input_tokens": 0, + "calls": [{"name": "Read", "sig": "xyz", "digest": "d1", "result_bytes": 1200}]}, + ], +} + + +class Truncator: + """Reads sizes only, so it scores from transcripts alone — the common case.""" + + id, label = "truncate_dumps", "Truncate large tool dumps" + risk, requires_content = L.RISK_LOW, False + + def propose(self, session, ctx): + cap = int(ctx.settings.get("keep_bytes", 4096)) + return L.Proposal( + lever=self.id, + edits=tuple( + L.Edit(turn_index=ti, call_index=ci, kind="truncate", + reason="dump over cap", sig=c.sig, keep_bytes=cap) + for ti, ci, c in session.iter_calls() if c.result_bytes > cap + ), + diagnostics={"scanned": session.n_turns}, + ) + + +class NeedsBytes(Truncator): + id, requires_content = "needs_bytes", True + + def propose(self, session, ctx): + body = session.turns[0].calls[0].content.resolve() + return L.Proposal(lever=self.id, diagnostics={"bytes": len(body)}) + + +class Boom(Truncator): + id = "boom" + + def propose(self, session, ctx): + raise ValueError("bad session") + + +@pytest.fixture +def ctx(): + return L.LeverContext( + count_tokens=lambda t, *, model: len(t) // 4, + mode=L.MODE_SHADOW, + settings={"keep_bytes": 4096}, + ) + + +@pytest.fixture +def session(): + return L.from_corpus_session(RAW) + + +@pytest.fixture +def registered(): + return L.RegisteredLever(lever=Truncator(), dist="ace-skills") + + +# -- the normalized model ---------------------------------------------------------------- + + +def test_one_adapter_serves_any_agent(session): + assert (session.agent, session.n_turns, session.kind) == ("codex", 2, "main") + assert session.turns[0].ts is not None + assert list(session.iter_calls())[1][:2] == (1, 0) + + +def test_prompt_tokens_sums_the_cached_buckets(session): + """`input_tokens` EXCLUDES the cached buckets, so this is a sum and not a max. + + Subtracting cache_read from input to "correct" it under-reports prompt volume — a bug + with no visible symptom. + """ + assert session.turns[0].usage.prompt_tokens == 100 + 900 + 400 + + +def test_ttl_breakdown_is_provider_neutral(session): + assert session.turns[0].usage.cache_write_by_ttl == {"5m": 400} + + +def test_measure_only_sessions_carry_no_content(session): + assert session.turns[0].calls[0].content is None + assert not session.turns[0].calls[0].has_content + + +# -- levers ------------------------------------------------------------------------------ + + +def test_a_lever_scores_on_hashes_and_sizes_alone(registered, session, ctx): + assert isinstance(registered.lever, L.Lever) + p = L.propose_safely(registered, session, ctx) + assert p is not None and len(p.edits) == 1 + assert p.edits[0].keep_bytes == 4096 + + +def test_edit_free_proposal_is_a_real_result(session, ctx): + """A loop guardrail's whole output is its diagnostics; `if proposal:` must not eat it.""" + p = L.Proposal("loop_guard", (), {"loops_detected": 3}) + assert not p.edits + assert p.diagnostics == {"loops_detected": 3} + + +# -- presence is not consent ------------------------------------------------------------- + + +@pytest.mark.parametrize( + "config,expected", + [ + ({}, L.MODE_OFF), + ({"levers": {"truncate_dumps": "shadow"}}, L.MODE_SHADOW), + ({"levers": {"truncate_dumps": {"mode": "on"}}}, L.MODE_ON), + # A bare `true` is an enablement, and the safe reading of "enabled" is the mode that + # changes nothing about the request. + ({"levers": {"truncate_dumps": True}}, L.MODE_SHADOW), + ], +) +def test_mode_resolution(registered, config, expected): + assert L.resolve_modes([registered], config=config) == {"truncate_dumps": expected} + + +def test_an_installed_lever_defaults_to_off(registered): + """The rule the registry exists to enforce. An unconfigured lever must never act.""" + assert L.resolve_modes([registered], config={}) == {"truncate_dumps": L.MODE_OFF} + + +def test_an_unknown_mode_resolves_to_off_not_to_the_default(registered, caplog): + """A typo becoming `shadow` is tolerable; a typo becoming `on` is not.""" + with caplog.at_level(logging.WARNING): + modes = L.resolve_modes([registered], config={"levers": {"truncate_dumps": "Bogus"}}) + assert modes == {"truncate_dumps": L.MODE_OFF} + + +def test_settings_exclude_mode(): + got = L.load_settings( + "truncate_dumps", + config={"levers": {"truncate_dumps": {"mode": "on", "keep_bytes": 99}}}, + ) + assert got == {"keep_bytes": 99} + + +# -- requires_content --------------------------------------------------------------------- + + +def test_content_requiring_lever_is_refused_not_raised(session, ctx): + """Refused outright rather than allowed to half-run: a partial proposal is worse than + none, because the ledger cannot tell it from a complete one.""" + nb = L.RegisteredLever(lever=NeedsBytes()) + assert L.propose_safely(nb, session, ctx) is None + + +def test_content_requiring_lever_runs_where_an_actuator_supplied_bytes(ctx): + s = L.from_corpus_session( + RAW, content_for=lambda ti, ci: (lambda: "x" * 240000) if ti == 0 else None + ) + assert s.turns[0].calls[0].has_content + assert not s.turns[1].calls[0].has_content + p = L.propose_safely(L.RegisteredLever(lever=NeedsBytes()), s, ctx) + assert p is not None and p.diagnostics == {"bytes": 240000} + + +def test_content_ref_refuses_in_measure_only_mode(): + with pytest.raises(L.ContentUnavailable): + L.ContentRef().resolve() + + +# -- failure isolation -------------------------------------------------------------------- + + +def test_a_throwing_lever_costs_its_own_row_and_nothing_else(session, ctx, caplog): + with caplog.at_level(logging.WARNING): + assert L.propose_safely(L.RegisteredLever(lever=Boom()), session, ctx) is None + + +def test_no_lever_package_is_the_ordinary_case(): + """An empty entry-point group is not an error state — it is the open-source default.""" + assert L.discover("ace.sidecar.levers.nonexistent") == () diff --git a/tests/test_quality_metrics.py b/tests/test_quality_metrics.py new file mode 100644 index 0000000..d07e9d9 --- /dev/null +++ b/tests/test_quality_metrics.py @@ -0,0 +1,276 @@ +"""Tests for ace.sidecar code quality, verification hygiene, and reliability metrics.""" + +from __future__ import annotations + +from typing import Any, Dict, List + +import pytest + +from ace.sidecar.dashboard_render import render +from ace.sidecar.insights import ( + _classify_call, + _build_payload, + format_prometheus_metrics, + quality_metrics, +) + + +def test_classify_call_test_commands() -> None: + # Pytest + cl = _classify_call("Bash", {"command": "pytest -v tests/"}) + assert cl["is_test_run"] is True + assert cl["is_edit"] is False + + # npm test + cl = _classify_call("run_command", {"CommandLine": "npm test"}) + assert cl["is_test_run"] is True + + # ruff check + cl = _classify_call("Bash", {"command": "ruff check --fix ."}) + assert cl["is_test_run"] is True + + # cargo test + cl = _classify_call("exec_command", {"command": "cargo test --all"}) + assert cl["is_test_run"] is True + + # non-test bash command + cl = _classify_call("Bash", {"command": "git status"}) + assert cl["is_test_run"] is False + + +def test_classify_call_edits_and_views() -> None: + # Edit tool + cl = _classify_call("Edit", {"file_path": "src/main.py"}) + assert cl["is_edit"] is True + assert cl["is_view"] is False + assert cl["raw_target"] == "src/main.py" + assert cl["is_src_file"] is True + assert cl["is_test_file"] is False + + # write_to_file on test file + cl = _classify_call("write_to_file", {"TargetFile": "/app/tests/test_api.py"}) + assert cl["is_edit"] is True + assert cl["is_test_file"] is True + assert cl["is_src_file"] is False + + # replace_file_content + cl = _classify_call("replace_file_content", {"TargetFile": "web/app.tsx"}) + assert cl["is_edit"] is True + assert cl["is_src_file"] is True + + # View / read_file + cl = _classify_call("view_file", {"AbsolutePath": "/app/README.md"}) + assert cl["is_view"] is True + assert cl["is_edit"] is False + assert cl["raw_target"] == "/app/README.md" + + +def test_quality_metrics_empty() -> None: + qm = quality_metrics([]) + assert qm["available"] is False + assert qm["quality_score"] == 100 + assert qm["grade"] == "A" + assert qm["verification_rate_pct"] == 100.0 + assert qm["first_pass_success_rate_pct"] == 100.0 + assert qm["thrashed_files_count"] == 0 + + +def test_quality_metrics_clean_verified_session() -> None: + sess: List[Dict[str, Any]] = [ + { + "session": "s1", + "agent_type": "claude", + "turns": [ + { + "model": "claude-sonnet-4-6", + "calls": [ + { + "name": "view_file", + "raw_target": "src/app.py", + "sig": "sig_v1", + "is_view": True, + "is_edit": False, + "is_test_run": False, + }, + { + "name": "replace_file_content", + "raw_target": "src/app.py", + "sig": "sig_e1", + "is_view": False, + "is_edit": True, + "is_test_run": False, + "is_src_file": True, + "is_test_file": False, + }, + { + "name": "write_to_file", + "raw_target": "tests/test_app.py", + "sig": "sig_e2", + "is_view": False, + "is_edit": True, + "is_test_run": False, + "is_src_file": False, + "is_test_file": True, + }, + { + "name": "Bash", + "raw_target": "pytest", + "sig": "sig_t1", + "is_view": False, + "is_edit": False, + "is_test_run": True, + "is_error": False, + }, + ], + } + ], + } + ] + qm = quality_metrics(sess) + assert qm["available"] is True + assert qm["verification_rate"] == 1.0 + assert qm["verification_rate_pct"] == 100.0 + assert qm["first_pass_success_rate"] == 1.0 + assert qm["tool_error_rate"] == 0.0 + assert qm["thrashed_files_count"] == 0 + assert qm["redundant_reads_count"] == 0 + assert qm["sessions_with_edits"] == 1 + assert qm["sessions_with_tests"] == 1 + assert qm["quality_score"] >= 90 + assert qm["grade"] == "A" + + +def test_quality_metrics_unverified_and_thrashed_session() -> None: + # 1 session editing a file 4 times (thrashing), zero tests, 1 tool error + sess: List[Dict[str, Any]] = [ + { + "session": "s2", + "agent_type": "antigravity", + "turns": [ + { + "model": "gemini-3.6-flash", + "calls": [ + { + "name": "view_file", + "raw_target": "src/flaky.py", + "sig": "v1", + "is_view": True, + }, + { + "name": "view_file", + "raw_target": "src/flaky.py", + "sig": "v1", + "is_view": True, # Redundant read + }, + { + "name": "edit", + "raw_target": "src/flaky.py", + "is_edit": True, + "is_src_file": True, + "is_error": True, # Error 1 + }, + ], + }, + { + "model": "gemini-3.6-flash", + "calls": [ + { + "name": "edit", + "raw_target": "src/flaky.py", + "is_edit": True, + "is_src_file": True, + }, + { + "name": "edit", + "raw_target": "src/flaky.py", + "is_edit": True, + "is_src_file": True, + }, + { + "name": "edit", + "raw_target": "src/flaky.py", + "is_edit": True, + "is_src_file": True, + }, + ], + }, + ], + } + ] + qm = quality_metrics(sess) + assert qm["available"] is True + assert qm["verification_rate"] == 0.0 # Zero tests run + assert qm["sessions_with_edits"] == 1 + assert qm["sessions_with_tests"] == 0 + assert qm["thrashed_files_count"] == 1 + assert "src/flaky.py" in qm["thrashed_files_list"] + assert qm["redundant_reads_count"] == 1 + assert qm["tool_error_rate"] > 0.0 + assert qm["quality_score"] < 70 + assert qm["grade"] in ("C", "D", "F") + + +def test_quality_in_payload_and_prometheus() -> None: + sess: List[Dict[str, Any]] = [ + { + "session": "s1", + "agent_type": "claude", + "cwds": ["/test/repo"], + "turns": [ + { + "model": "claude-sonnet-4-6", + "input_tokens": 1000, + "output_tokens": 200, + "cache_read_input_tokens": 800, + "cache_creation_input_tokens": 100, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "blocks": {"text": 1, "tool_use": 2}, + "calls": [ + { + "name": "Edit", + "raw_target": "app.py", + "is_edit": True, + "is_src_file": True, + }, + { + "name": "Bash", + "command": "pytest", + "is_test_run": True, + }, + ], + "ts": "2026-08-27T12:00:00Z", + } + ], + "events": [ + (1787832000.0, "prompt", (), 1787832000.0), + (1787832005.0, "assistant", ("Edit", "Bash"), 1787832005.0), + ], + "path": "/dummy/s1.jsonl", + "bytes": 500, + "mtime": 1787832010.0, + "snippet": "fix bug and test", + } + ] + + payload = _build_payload(sess, capture=None, range_key="all", agent="all", store_path=None) + assert "quality" in payload + qm = payload["quality"] + assert qm["available"] is True + assert qm["verification_rate_pct"] == 100.0 + assert qm["quality_score"] >= 80 + + # Prometheus export check + prom_text = format_prometheus_metrics(payload) + assert "ace_quality_score" in prom_text + assert "ace_quality_verification_rate 1.0" in prom_text + assert "ace_quality_first_pass_success_rate 1.0" in prom_text + assert "ace_quality_thrashed_files_total 0" in prom_text + assert "ace_quality_redundant_reads_total 0" in prom_text + + # Render dashboard check + html = render(payload) + assert "CODE QUALITY & RELIABILITY" in html or "CODE QUALITY & RELIABILITY" in html + assert "quality_score" in html + assert "verification_rate" in html + assert "first_pass_success" in html From 1b23e7f29b6d1656f077f7fb9ccfad42f5ada982 Mon Sep 17 00:00:00 2001 From: ACE Engineering Date: Thu, 27 Aug 2026 19:24:14 -0700 Subject: [PATCH 2/8] feat(quality): add per-agent and per-model code quality breakdown matrix --- src/ace/sidecar/dashboard_render.py | 84 ++++++++++++++++++++++++++- src/ace/sidecar/insights.py | 89 ++++++++++++++++++++++++----- tests/test_quality_metrics.py | 78 ++++++++++++++++++++++++- 3 files changed, 233 insertions(+), 18 deletions(-) diff --git a/src/ace/sidecar/dashboard_render.py b/src/ace/sidecar/dashboard_render.py index 2595a12..019120c 100644 --- a/src/ace/sidecar/dashboard_render.py +++ b/src/ace/sidecar/dashboard_render.py @@ -807,6 +807,87 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"" ) + breakdown_rows = [] + # Agent breakdown rows + by_agent = qm.get("by_agent") or {} + for ak, a_info in by_agent.items(): + a_score = a_info.get("quality_score", 100) + a_grade = a_info.get("grade", "A") + a_v_rate = a_info.get("verification_rate_pct", 100.0) + a_fsr = a_info.get("first_pass_success_rate_pct", 100.0) + a_thrash = a_info.get("thrashed_files_count", 0) + a_rec = a_info.get("avg_error_recovery_turns", 1.0) + a_sess = a_info.get("sessions", 0) + badge_style = ( + "color:var(--mint);border-color:#1d3b2e;background:#0F231A" + if ak == "antigravity" + else "color:var(--blue);border-color:#1e355b;background:#0d1c33" + ) + score_badge = ( + "color:var(--mint);border-color:#1d3b2e;background:#0F231A" + if a_score >= 80 + else ( + "color:var(--gold);border-color:#3d3014;background:#241D0E" + if a_score >= 60 + else "color:var(--crit);border-color:#4a1e17;background:#2a110e" + ) + ) + breakdown_rows.append( + f"" + f"{escape(a_info.get('label', ak))}" + f"{a_score} ({a_grade})" + f"{a_v_rate}%" + f"{a_fsr}%" + f"{'' + str(a_thrash) + '' if a_thrash > 0 else '0'}" + f"{a_rec} turns" + f"{a_sess}" + f"" + ) + + # Model breakdown rows + by_model = qm.get("by_model") or [] + for m_info in by_model: + m_name = m_info.get("model", "unknown") + m_score = m_info.get("quality_score", 100) + m_grade = m_info.get("grade", "A") + m_v_rate = m_info.get("verification_rate_pct", 100.0) + m_fsr = m_info.get("first_pass_success_rate_pct", 100.0) + m_thrash = m_info.get("thrashed_files_count", 0) + m_rec = m_info.get("avg_error_recovery_turns", 1.0) + m_sess = m_info.get("sessions", 0) + score_badge = ( + "color:var(--mint);border-color:#1d3b2e;background:#0F231A" + if m_score >= 80 + else ( + "color:var(--gold);border-color:#3d3014;background:#241D0E" + if m_score >= 60 + else "color:var(--crit);border-color:#4a1e17;background:#2a110e" + ) + ) + breakdown_rows.append( + f"" + f"{escape(m_name)}" + f"{m_score} ({m_grade})" + f"{m_v_rate}%" + f"{m_fsr}%" + f"{'' + str(m_thrash) + '' if m_thrash > 0 else '0'}" + f"{m_rec} turns" + f"{m_sess}" + f"" + ) + + matrix_table = "" + if breakdown_rows: + matrix_table = ( + f"
    " + f"" + f"" + f"" + f"{''.join(breakdown_rows)}" + f"
    engine / modelscoreverificationfirst-pass successthrash fileshealing turnssessions
    " + f"
    " + ) + return ( f"
    {''.join(tiles)}
    " f"
    " @@ -819,7 +900,8 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"
    {redundant_reads} redundant duplicate file reads
    " f"
    " f"{thrash_html}" - f"
    Measures how safely and stably coding agents operate in your repository. High verification rates and low thrash indicate clean first-pass execution without prompt churn.
    " + f"{matrix_table}" + f"
    Measures how safely and stably coding agents operate in your repository. Correlates cost against first-pass tool correctness and test diligence.
    " f"" ) diff --git a/src/ace/sidecar/insights.py b/src/ace/sidecar/insights.py index 9439ee9..daac0e1 100644 --- a/src/ace/sidecar/insights.py +++ b/src/ace/sidecar/insights.py @@ -1982,18 +1982,7 @@ def totals(sess: List[Dict[str, Any]]) -> Dict[str, Any]: # ------------------------------------------------- code quality & reliability metrics -def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: - """Calculates unified code quality, verification hygiene, and reliability metrics. - - Evaluates across all agent sessions: - - Verification Hygiene: share of editing sessions running test suites / linters. - - Edit Thrash / Rework: files modified 3+ times within a single session. - - First-Pass Success Rate (FSR): share of tool executions with zero errors on initial run. - - Error Healing Latency: average conversation turns to resolve tool execution errors. - - Redundant File Reads: consecutive duplicate view/reads of unchanged files. - - Test-to-Code Ratio: ratio of test file edits vs source file edits. - - Composite Score: 0-100 overall quality and reliability index. - """ +def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: total_sessions = len(sess) if not total_sessions: return { @@ -2176,6 +2165,67 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: } +def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: + """Calculates unified code quality, verification hygiene, and reliability metrics. + + Includes top-line metrics along with breakdowns: + - by_agent: Quality scores partitioned per agent engine (Claude Code, Antigravity, Codex). + - by_model: Quality scores partitioned per LLM model. + """ + overall = _calc_quality_block(sess) + if not sess: + overall["by_agent"] = {} + overall["by_model"] = [] + return overall + + # Group by agent + by_agent: Dict[str, Any] = {} + agent_groups: Dict[str, List[Dict[str, Any]]] = {} + for s in sess: + ak = s.get("agent_type") or AGENT_CLAUDE + agent_groups.setdefault(ak, []).append(s) + + for ak, a_sess in agent_groups.items(): + block = _calc_quality_block(a_sess) + by_agent[ak] = { + "agent": ak, + "label": AGENTS.get(ak, ak.capitalize()), + "sessions": len(a_sess), + **block, + } + + # Group by model + model_sessions: Dict[str, List[Dict[str, Any]]] = {} + for s in sess: + models_in_s = set(t.get("model") for t in s.get("turns", []) if t.get("model")) + for m in models_in_s: + projected_turns = [t for t in s.get("turns", []) if t.get("model") == m] + if projected_turns: + model_sessions.setdefault(m, []).append( + { + "session": s.get("session"), + "agent_type": s.get("agent_type"), + "turns": projected_turns, + "events": s.get("events", []), + } + ) + + by_model: List[Dict[str, Any]] = [] + for m_name, m_sess in sorted(model_sessions.items(), key=lambda kv: -len(kv[1])): + block = _calc_quality_block(m_sess) + by_model.append( + { + "model": m_name, + "sessions": len(m_sess), + **block, + } + ) + + overall["by_agent"] = by_agent + overall["by_model"] = by_model + return overall + + # ------------------------------------------------- time budget (docs/analysis_docs §1 and §2) # Declared order is presentation order for ties; the payload sorts by size. @@ -3127,15 +3177,24 @@ def format_prometheus_metrics(d: Dict[str, Any]) -> str: qm = d.get("quality") or {} lines.append("# HELP ace_quality_score Composite code quality and verification score (0-100).") lines.append("# TYPE ace_quality_score gauge") - lines.append(f'ace_quality_score {qm.get("quality_score", 100)}') + lines.append(f'ace_quality_score{{agent="all"}} {qm.get("quality_score", 100)}') + for agent_id, q_info in (qm.get("by_agent") or {}).items(): + lines.append(f'ace_quality_score{{agent="{agent_id}"}} {q_info.get("quality_score", 100)}') + for m_info in (qm.get("by_model") or []): + m_name = m_info.get("model", "unknown") + lines.append(f'ace_quality_score{{model="{m_name}"}} {m_info.get("quality_score", 100)}') lines.append("# HELP ace_quality_verification_rate Share of edited sessions that ran automated tests or linters.") lines.append("# TYPE ace_quality_verification_rate gauge") - lines.append(f'ace_quality_verification_rate {qm.get("verification_rate", 1.0)}') + lines.append(f'ace_quality_verification_rate{{agent="all"}} {qm.get("verification_rate", 1.0)}') + for agent_id, q_info in (qm.get("by_agent") or {}).items(): + lines.append(f'ace_quality_verification_rate{{agent="{agent_id}"}} {q_info.get("verification_rate", 1.0)}') lines.append("# HELP ace_quality_first_pass_success_rate Share of tool calls that succeeded on first pass.") lines.append("# TYPE ace_quality_first_pass_success_rate gauge") - lines.append(f'ace_quality_first_pass_success_rate {qm.get("first_pass_success_rate", 1.0)}') + lines.append(f'ace_quality_first_pass_success_rate{{agent="all"}} {qm.get("first_pass_success_rate", 1.0)}') + for agent_id, q_info in (qm.get("by_agent") or {}).items(): + lines.append(f'ace_quality_first_pass_success_rate{{agent="{agent_id}"}} {q_info.get("first_pass_success_rate", 1.0)}') lines.append("# HELP ace_quality_tool_error_rate Share of tool executions that returned errors.") lines.append("# TYPE ace_quality_tool_error_rate gauge") diff --git a/tests/test_quality_metrics.py b/tests/test_quality_metrics.py index d07e9d9..7290d3a 100644 --- a/tests/test_quality_metrics.py +++ b/tests/test_quality_metrics.py @@ -263,8 +263,8 @@ def test_quality_in_payload_and_prometheus() -> None: # Prometheus export check prom_text = format_prometheus_metrics(payload) assert "ace_quality_score" in prom_text - assert "ace_quality_verification_rate 1.0" in prom_text - assert "ace_quality_first_pass_success_rate 1.0" in prom_text + assert 'ace_quality_verification_rate{agent="all"} 1.0' in prom_text + assert 'ace_quality_first_pass_success_rate{agent="all"} 1.0' in prom_text assert "ace_quality_thrashed_files_total 0" in prom_text assert "ace_quality_redundant_reads_total 0" in prom_text @@ -274,3 +274,77 @@ def test_quality_in_payload_and_prometheus() -> None: assert "quality_score" in html assert "verification_rate" in html assert "first_pass_success" in html + + +def test_quality_metrics_by_agent_and_model() -> None: + sess: List[Dict[str, Any]] = [ + # Session 1: Claude using Sonnet - verified, high quality + { + "session": "s1", + "agent_type": "claude", + "cwds": ["/test"], + "turns": [ + { + "model": "claude-sonnet-4-6", + "input_tokens": 1000, + "output_tokens": 200, + "cache_read_input_tokens": 800, + "cache_creation_input_tokens": 0, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "calls": [ + {"name": "Edit", "raw_target": "src/a.py", "is_edit": True, "is_src_file": True}, + {"name": "Bash", "command": "pytest", "is_test_run": True, "is_error": False}, + ], + } + ], + "events": [], + }, + # Session 2: Antigravity using Gemini Flash - unverified, error + { + "session": "s2", + "agent_type": "antigravity", + "cwds": ["/test"], + "turns": [ + { + "model": "gemini-3.6-flash", + "input_tokens": 500, + "output_tokens": 100, + "cache_read_input_tokens": 300, + "cache_creation_input_tokens": 0, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "calls": [ + {"name": "write_to_file", "raw_target": "src/b.py", "is_edit": True, "is_src_file": True, "is_error": True}, + ], + } + ], + "events": [], + }, + ] + + qm = quality_metrics(sess) + assert "by_agent" in qm + assert "claude" in qm["by_agent"] + assert "antigravity" in qm["by_agent"] + + claude_q = qm["by_agent"]["claude"] + assert claude_q["verification_rate_pct"] == 100.0 + assert claude_q["quality_score"] >= 85 + + agy_q = qm["by_agent"]["antigravity"] + assert agy_q["verification_rate_pct"] == 0.0 + assert agy_q["first_pass_success_rate_pct"] == 0.0 + + assert "by_model" in qm + models = [m["model"] for m in qm["by_model"]] + assert "claude-sonnet-4-6" in models + assert "gemini-3.6-flash" in models + + # Dashboard render with comparative table + payload = _build_payload(sess, capture=None, range_key="all", agent="all", store_path=None) + html = render(payload) + assert "Claude Code" in html or "claude" in html + assert "claude-sonnet-4-6" in html + assert "gemini-3.6-flash" in html + assert "engine / model" in html From 6616b707804c0c0ca6dd007cc86be9c038be3976 Mon Sep 17 00:00:00 2001 From: ACE Engineering Date: Thu, 27 Aug 2026 19:30:41 -0700 Subject: [PATCH 3/8] fix(quality): exclude internal brain artifacts from thrash count and style comparative matrix --- src/ace/sidecar/dashboard_render.py | 7 +++++-- src/ace/sidecar/insights.py | 9 ++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/ace/sidecar/dashboard_render.py b/src/ace/sidecar/dashboard_render.py index 019120c..68fe72f 100644 --- a/src/ace/sidecar/dashboard_render.py +++ b/src/ace/sidecar/dashboard_render.py @@ -879,8 +879,11 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: matrix_table = "" if breakdown_rows: matrix_table = ( - f"
    " - f"" + f"
    " + f"
    " + f"ENGINE & MODEL RELIABILITY COMPARISON" + f"
    " + f"
    " f"" f"" f"{''.join(breakdown_rows)}" diff --git a/src/ace/sidecar/insights.py b/src/ace/sidecar/insights.py index daac0e1..c06f233 100644 --- a/src/ace/sidecar/insights.py +++ b/src/ace/sidecar/insights.py @@ -2047,7 +2047,14 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: session_has_edit = True total_edits += 1 raw_t = c.get("raw_target") or c.get("target") or "unknown" - session_file_edits[raw_t] = session_file_edits.get(raw_t, 0) + 1 + is_artifact = ( + "/.gemini/antigravity/brain/" in raw_t + or "/.system_generated/" in raw_t + or raw_t.endswith("walkthrough.md") + or raw_t.endswith("implementation_plan.md") + ) + if not is_artifact: + session_file_edits[raw_t] = session_file_edits.get(raw_t, 0) + 1 if c.get("is_test_file"): test_edits_count += 1 elif c.get("is_src_file"): From 247d6215e4ff387448ba9f2fa27c4b46ef10a916 Mon Sep 17 00:00:00 2001 From: ACE Engineering Date: Thu, 27 Aug 2026 19:32:12 -0700 Subject: [PATCH 4/8] style(quality): increase font size, cell padding, and header contrast for reliability comparison matrix --- src/ace/sidecar/dashboard_render.py | 59 +++++++++++++++++------------ 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/src/ace/sidecar/dashboard_render.py b/src/ace/sidecar/dashboard_render.py index 68fe72f..b993cc1 100644 --- a/src/ace/sidecar/dashboard_render.py +++ b/src/ace/sidecar/dashboard_render.py @@ -821,7 +821,11 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: badge_style = ( "color:var(--mint);border-color:#1d3b2e;background:#0F231A" if ak == "antigravity" - else "color:var(--blue);border-color:#1e355b;background:#0d1c33" + else ( + "color:var(--blue);border-color:#1e355b;background:#0d1c33" + if ak == "claude" + else "color:var(--purple, #c084fc);border-color:#3b1e5b;background:#1a0d33" + ) ) score_badge = ( "color:var(--mint);border-color:#1d3b2e;background:#0F231A" @@ -833,14 +837,14 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: ) ) breakdown_rows.append( - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" f"" ) @@ -865,28 +869,35 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: ) ) breakdown_rows.append( - f"" - f"" - f"" - f"" - f"" - f"" - f"" - f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" f"" ) matrix_table = "" if breakdown_rows: matrix_table = ( - f"
    " - f"
    " - f"ENGINE & MODEL RELIABILITY COMPARISON" + f"
    " + f"
    " + f"ENGINE & MODEL RELIABILITY COMPARISON" f"
    " - f"
    engine / modelscoreverificationfirst-pass successthrash fileshealing turnssessions
    {escape(a_info.get('label', ak))}{a_score} ({a_grade}){a_v_rate}%{a_fsr}%{'' + str(a_thrash) + '' if a_thrash > 0 else '0'}{a_rec} turns{a_sess}
    {escape(a_info.get('label', ak))}{a_score} ({a_grade}){a_v_rate}%{a_fsr}%{'' + str(a_thrash) + '' if a_thrash > 0 else '0'}{a_rec} turns{a_sess}
    {escape(m_name)}{m_score} ({m_grade}){m_v_rate}%{m_fsr}%{'' + str(m_thrash) + '' if m_thrash > 0 else '0'}{m_rec} turns{m_sess}
    {escape(m_name)}{m_score} ({m_grade}){m_v_rate}%{m_fsr}%{'' + str(m_thrash) + '' if m_thrash > 0 else '0'}{m_rec} turns{m_sess}
    " - f"" - f"" - f"{''.join(breakdown_rows)}" + f"
    engine / modelscoreverificationfirst-pass successthrash fileshealing turnssessions
    " + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"{''.join(breakdown_rows)}" f"
    ENGINE / MODELSCOREVERIFICATIONFIRST-PASS SUCCESSTHRASH FILESHEALING TURNSSESSIONS
    " f"
    " ) From 26b71284e5c7a41f9dd0a84dd5a248972fb555b8 Mon Sep 17 00:00:00 2001 From: ACE Engineering Date: Thu, 27 Aug 2026 23:36:28 -0700 Subject: [PATCH 5/8] fix(quality): calibrate scoring, parse tool errors in antigravity transcripts, add task completion success metrics --- src/ace/sidecar/dashboard_render.py | 29 +++++--- src/ace/sidecar/insights.py | 110 +++++++++++++++++----------- tests/test_quality_metrics.py | 2 +- 3 files changed, 88 insertions(+), 53 deletions(-) diff --git a/src/ace/sidecar/dashboard_render.py b/src/ace/sidecar/dashboard_render.py index b993cc1..47df73e 100644 --- a/src/ace/sidecar/dashboard_render.py +++ b/src/ace/sidecar/dashboard_render.py @@ -726,6 +726,7 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: score = qm.get("quality_score", 100) grade = qm.get("grade", "A") + c_rate = qm.get("task_completion_rate_pct", 100.0) v_rate = qm.get("verification_rate_pct", 100.0) fsr = qm.get("first_pass_success_rate_pct", 100.0) err_rate = qm.get("tool_error_rate_pct", 0.0) @@ -735,15 +736,17 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: test_code_ratio = qm.get("test_to_code_ratio", 1.0) sessions_edits = qm.get("sessions_with_edits", 0) sessions_tests = qm.get("sessions_with_tests", 0) + clean_completed = qm.get("clean_completed_sessions", 0) score_color = ( "var(--mint)" if score >= 80 else ("var(--gold)" if score >= 60 else "var(--crit)") ) + c_cls = "" if c_rate >= 80 else ("warn" if c_rate >= 60 else "crit") v_cls = "" if v_rate >= 75 else ("warn" if v_rate >= 50 else "crit") fsr_cls = "" if fsr >= 85 else ("warn" if fsr >= 70 else "crit") - thrash_cls = "" if thrash_cnt == 0 else ("warn" if thrash_cnt <= 2 else "crit") + thrash_cls = "" if thrash_cnt == 0 else ("warn" if thrash_cnt <= 5 else "crit") tiles = [ _st( @@ -751,7 +754,15 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"{score}/ 100", f"Grade {grade}", delta="COMPOSITE", - title="Weighted reliability index across verification hygiene (35%), first-pass tool success (35%), edit stability (15%), and test balance (15%).", + title="Weighted reliability index across task completion (35%), verification hygiene (30%), first-pass tool success (20%), and edit stability (15%).", + ), + _st( + "task_completion", + f"{c_rate}%", + f"{clean_completed} verified sessions", + delta="TASK RESOLUTION", + dcls=c_cls, + title="Percentage of sessions that resolved cleanly without trailing tool errors or unverified changes.", ), _st( "verification_rate", @@ -780,17 +791,10 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: _st( "healing_latency", f"{recovery_turns} turns", - "avg turns to recover", + f"{redundant_reads} redundant reads", delta="ERROR HEALING", title="Average number of conversation turns required for the agent to resolve a failed tool execution and resume forward progress.", ), - _st( - "context_waste", - f"{redundant_reads} reads", - f"test/code ratio: {test_code_ratio}x", - delta="REDUNDANCY", - title="Consecutive duplicate reads of identical files without intervening edits.", - ), ] thrashed_files_list = qm.get("thrashed_files_list") or [] @@ -813,6 +817,7 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: for ak, a_info in by_agent.items(): a_score = a_info.get("quality_score", 100) a_grade = a_info.get("grade", "A") + a_comp = a_info.get("task_completion_rate_pct", 100.0) a_v_rate = a_info.get("verification_rate_pct", 100.0) a_fsr = a_info.get("first_pass_success_rate_pct", 100.0) a_thrash = a_info.get("thrashed_files_count", 0) @@ -840,6 +845,7 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"" f"{escape(a_info.get('label', ak))}" f"{a_score} ({a_grade})" + f"{a_comp}%" f"{a_v_rate}%" f"{a_fsr}%" f"{'' + str(a_thrash) + '' if a_thrash > 0 else '0'}" @@ -854,6 +860,7 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: m_name = m_info.get("model", "unknown") m_score = m_info.get("quality_score", 100) m_grade = m_info.get("grade", "A") + m_comp = m_info.get("task_completion_rate_pct", 100.0) m_v_rate = m_info.get("verification_rate_pct", 100.0) m_fsr = m_info.get("first_pass_success_rate_pct", 100.0) m_thrash = m_info.get("thrashed_files_count", 0) @@ -872,6 +879,7 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"" f"{escape(m_name)}" f"{m_score} ({m_grade})" + f"{m_comp}%" f"{m_v_rate}%" f"{m_fsr}%" f"{'' + str(m_thrash) + '' if m_thrash > 0 else '0'}" @@ -891,6 +899,7 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"" f"ENGINE / MODEL" f"SCORE" + f"COMPLETION" f"VERIFICATION" f"FIRST-PASS SUCCESS" f"THRASH FILES" diff --git a/src/ace/sidecar/insights.py b/src/ace/sidecar/insights.py index c06f233..f7168fa 100644 --- a/src/ace/sidecar/insights.py +++ b/src/ace/sidecar/insights.py @@ -153,6 +153,11 @@ def _sig(name: str, tool_input: Dict[str, Any]) -> str: re.IGNORECASE, ) +_ERR_MSG_RE = re.compile( + r"(encountered error in tool execution|the command exited with code [1-9]|exit code [1-9]|operation not permitted|command failed|fatal:|traceback \(most recent call last\)|syntaxerror|typeerror|keyerror|assertionerror|modulenotfounderror|permission denied|no such file or directory)", + re.IGNORECASE, +) + _TEST_FILE_RE = re.compile( r"(^|[/\\])(tests?|spec|__tests__)[/\\]|(\.|_)(test|spec)\.[a-zA-Z0-9]+$", re.IGNORECASE, @@ -612,29 +617,7 @@ def _scan_antigravity(root: str) -> List[Dict[str, Any]]: if text and not text.startswith("<"): first_snippet = text[:110] events.append((at, "prompt", (), at)) - continue - - if stype in ("TOOL_RESULT", "SYSTEM_RESULT"): - body = ( - rec.get("content") or rec.get("output") or rec.get("result") - ) - tid = ( - rec.get("tool_use_id") - or rec.get("call_id") - or f"call_{len(events)}" - ) - result_bytes[tid] = _measure(body) - dg = _digest(body) - if dg: - result_digests[tid] = dg - is_err = ( - str(rec.get("status", "")).upper() in ("ERROR", "FAILED") - or bool(rec.get("error")) - or bool(rec.get("is_error")) - ) - if is_err: - result_errors[tid] = True - events.append((at, "tool_result", (), at)) + current_turn_calls = None continue if ( @@ -654,11 +637,9 @@ def _scan_antigravity(root: str) -> List[Dict[str, Any]]: if isinstance(tc, dict): nm = tc.get("name") or "tool" args = tc.get("args") or tc.get("input") or {} - call_id = tc.get("id") or f"call_{len(turns)}_{idx_c}" cl = _classify_call(nm, args) calls.append( { - "id": call_id, "name": nm, "target": _target(args), "sig": _sig(nm, args), @@ -668,6 +649,7 @@ def _scan_antigravity(root: str) -> List[Dict[str, Any]]: "is_view": cl["is_view"], "is_test_file": cl["is_test_file"], "is_src_file": cl["is_src_file"], + "is_error": False, } ) @@ -717,26 +699,40 @@ def _scan_antigravity(root: str) -> List[Dict[str, Any]]: ), } turns.append(turn) + current_turn_calls = calls if calls else None events.append( (at, "assistant", tuple(c["name"] for c in calls), at) ) + continue + + if current_turn_calls and ( + stype in ("GENERIC", "TOOL_RESULT", "SYSTEM_MESSAGE", "SYSTEM_RESULT") + or ssource in ("SYSTEM", "MODEL") + ): + body = rec.get("content") or rec.get("output") or rec.get("result") or "" + st = str(rec.get("status", "")).upper() + is_err = ( + st in ("ERROR", "FAILED") + or bool(rec.get("error")) + or bool(rec.get("is_error")) + or bool(_ERR_MSG_RE.search(str(body))) + ) + for c in current_turn_calls: + if is_err: + c["is_error"] = True + c["result_bytes"] = _measure(body) + dg = _digest(body) + if dg: + c["digest"] = dg + events.append((at, "tool_result", (), at)) + continue except Exception: continue if turns: - for t in turns: - for c in t["calls"]: - cid_tag = c.pop("id", None) - if cid_tag and cid_tag in result_bytes: - c["result_bytes"] = result_bytes[cid_tag] - if cid_tag and cid_tag in result_digests: - c["digest"] = result_digests[cid_tag] - if cid_tag and cid_tag in result_errors: - c["is_error"] = True - events.sort(key=lambda e: e[0]) name = f"agy_{cid[:8]}" if cid else f"agy_{len(sessions)+1}" - session: Dict[str, Any] = { + session = { "session": name, "kind": "main", "agent_type": AGENT_ANTIGRAVITY, @@ -1989,6 +1985,8 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: "available": False, "quality_score": 100, "grade": "A", + "task_completion_rate": 1.0, + "task_completion_rate_pct": 100.0, "verification_rate": 1.0, "verification_rate_pct": 100.0, "first_pass_success_rate": 1.0, @@ -2002,15 +2000,19 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: "thrashed_files_list": [], "rework_thrash_rate": 0.0, "rework_thrash_rate_pct": 0.0, + "edit_stability": 1.0, + "edit_stability_pct": 100.0, "redundant_reads_count": 0, "avg_error_recovery_turns": 1.0, "test_to_code_ratio": 1.0, "sessions_with_edits": 0, "sessions_with_tests": 0, + "clean_completed_sessions": 0, } sessions_with_edits = 0 sessions_with_tests = 0 + clean_completed_sessions = 0 total_edits = 0 total_tests = 0 total_tool_calls = 0 @@ -2029,6 +2031,7 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: session_file_edits: Dict[str, int] = {} last_view_sig: Optional[str] = None pending_error_turn: Optional[int] = None + last_turn_had_error = False for turn_idx, t in enumerate(turns): turn_has_error = False @@ -2080,8 +2083,17 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: recovery_turns_list.append(max(1, turn_idx - pending_error_turn)) pending_error_turn = None + if turns and any(c.get("is_error") for c in turns[-1].get("calls") or []): + last_turn_had_error = True + if session_has_edit: sessions_with_edits += 1 + if session_has_test and not last_turn_had_error: + clean_completed_sessions += 1 + else: + if not last_turn_had_error: + clean_completed_sessions += 1 + if session_has_test: sessions_with_tests += 1 @@ -2108,12 +2120,21 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: else 0.0 ) + task_completion_rate = ( + (clean_completed_sessions / total_sessions) + if total_sessions > 0 + else 1.0 + ) + rework_thrash_rate = ( (thrashed_files_count / max(1, len(all_thrashed_files) + total_edits)) if total_edits > 0 else 0.0 ) + thrash_ratio = (thrashed_files_count / max(1, sessions_with_edits)) if sessions_with_edits > 0 else 0.0 + edit_stability = max(0.0, 1.0 - (thrash_ratio * 1.0)) + test_to_code_ratio = ( (test_edits_count / src_edits_count) if src_edits_count > 0 @@ -2126,13 +2147,13 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: else 1.0 ) - # Score calculation (0-100) - # Verification: 35%, FSR: 35%, Thrash Freedom: 15%, Test/Code balance: 15% + # Balanced 0-100 score: + # 35% Verified Task Completion, 30% Verification Diligence, 20% First-Pass Tool Success, 15% Edit Stability (Thrash-Free) raw_score = ( - 0.35 * verification_rate - + 0.35 * first_pass_success_rate - + 0.15 * max(0.0, 1.0 - (rework_thrash_rate * 2)) - + 0.15 * min(1.0, test_to_code_ratio) + 0.35 * task_completion_rate + + 0.30 * verification_rate + + 0.20 * first_pass_success_rate + + 0.15 * edit_stability ) * 100.0 quality_score = max(0, min(100, int(round(raw_score)))) @@ -2151,6 +2172,8 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: "available": True, "quality_score": quality_score, "grade": grade, + "task_completion_rate": round(task_completion_rate, 4), + "task_completion_rate_pct": round(task_completion_rate * 100.0, 1), "verification_rate": round(verification_rate, 4), "verification_rate_pct": round(verification_rate * 100.0, 1), "first_pass_success_rate": round(first_pass_success_rate, 4), @@ -2164,11 +2187,14 @@ def _calc_quality_block(sess: List[Dict[str, Any]]) -> Dict[str, Any]: "thrashed_files_list": sorted(list(all_thrashed_files))[:10], "rework_thrash_rate": round(rework_thrash_rate, 4), "rework_thrash_rate_pct": round(rework_thrash_rate * 100.0, 1), + "edit_stability": round(edit_stability, 4), + "edit_stability_pct": round(edit_stability * 100.0, 1), "redundant_reads_count": redundant_reads_count, "avg_error_recovery_turns": round(avg_error_recovery_turns, 1), "test_to_code_ratio": round(test_to_code_ratio, 2), "sessions_with_edits": sessions_with_edits, "sessions_with_tests": sessions_with_tests, + "clean_completed_sessions": clean_completed_sessions, } diff --git a/tests/test_quality_metrics.py b/tests/test_quality_metrics.py index 7290d3a..02e6512 100644 --- a/tests/test_quality_metrics.py +++ b/tests/test_quality_metrics.py @@ -347,4 +347,4 @@ def test_quality_metrics_by_agent_and_model() -> None: assert "Claude Code" in html or "claude" in html assert "claude-sonnet-4-6" in html assert "gemini-3.6-flash" in html - assert "engine / model" in html + assert "ENGINE / MODEL" in html From d2618af86aa81edd4e7d68ae2b25010451f94e30 Mon Sep 17 00:00:00 2001 From: ACE Engineering Date: Thu, 27 Aug 2026 23:38:11 -0700 Subject: [PATCH 6/8] chore: remove unrelated lever and gateway changes from quality branch --- src/ace/gateway/local_store.py | 173 +------ src/ace/gateway/messages.py | 120 +---- src/ace/sidecar/app.py | 21 - src/ace/sidecar/insights.py | 39 -- src/ace/sidecar/levers/__init__.py | 115 ----- src/ace/sidecar/levers/counter.py | 240 ---------- src/ace/sidecar/levers/ledger.py | 450 ------------------ src/ace/sidecar/levers/protocol.py | 205 -------- src/ace/sidecar/levers/rail.py | 276 ----------- src/ace/sidecar/levers/registry.py | 232 --------- src/ace/sidecar/levers/shadow.py | 736 ----------------------------- src/ace/sidecar/levers/types.py | 278 ----------- src/ace/sidecar/strategies.py | 68 +-- tests/test_lever_ledger.py | 160 ------- tests/test_lever_shadow.py | 470 ------------------ tests/test_levers.py | 205 -------- 16 files changed, 6 insertions(+), 3782 deletions(-) delete mode 100644 src/ace/sidecar/levers/__init__.py delete mode 100644 src/ace/sidecar/levers/counter.py delete mode 100644 src/ace/sidecar/levers/ledger.py delete mode 100644 src/ace/sidecar/levers/protocol.py delete mode 100644 src/ace/sidecar/levers/rail.py delete mode 100644 src/ace/sidecar/levers/registry.py delete mode 100644 src/ace/sidecar/levers/shadow.py delete mode 100644 src/ace/sidecar/levers/types.py delete mode 100644 tests/test_lever_ledger.py delete mode 100644 tests/test_lever_shadow.py delete mode 100644 tests/test_levers.py diff --git a/src/ace/gateway/local_store.py b/src/ace/gateway/local_store.py index 808d344..d088812 100644 --- a/src/ace/gateway/local_store.py +++ b/src/ace/gateway/local_store.py @@ -23,13 +23,12 @@ from __future__ import annotations -import json import logging import os import sqlite3 import threading import time -from typing import Any, Dict, Iterable, List, Mapping, Optional +from typing import Any, Dict, List, Optional log = logging.getLogger("ace.gateway.local_store") @@ -54,51 +53,6 @@ ); CREATE INDEX IF NOT EXISTS idx_turns_ts ON turns(ts); CREATE INDEX IF NOT EXISTS idx_turns_session ON turns(session_id); - --- One row per lever per proxied turn: what an installed lever, run for real against the --- actual request body, measured. Sibling of `turns` rather than columns on it, because a --- turn has N of these (one per enabled lever) and because every row here is a --- COUNTERFACTUAL -- a prompt that was never sent -- while every row in `turns` is what the --- provider actually billed. Merging the two would put a real charge and a hypothetical --- saving in one record with nothing to tell them apart. --- --- Why this table has to exist at all: a measured result is produced once, in a background --- task, moments after a response is served. Without a row here it is logged and lost, and --- the dashboard is back to simulating headroom over transcripts. -CREATE TABLE IF NOT EXISTS lever_turns ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ts REAL NOT NULL, - request_id TEXT, - session_id TEXT, - lever TEXT NOT NULL, - mode TEXT, - model TEXT, - -- Both sides of the counterfactual, kept so the delta can be re-derived rather than - -- trusted. Counted the same way through the provider's own counter; only their - -- difference is exact. - baseline_tokens INTEGER DEFAULT 0, - counterfactual_tokens INTEGER DEFAULT 0, - removed_tokens INTEGER DEFAULT 0, - -- How the removed tokens were allocated against this turn's real usage buckets. Kept - -- because it is the entire pricing argument: the same token delta is worth ~12x more - -- coming out of a cache write than out of a cache read. - from_cache_write INTEGER DEFAULT 0, - from_input INTEGER DEFAULT 0, - from_cache_read INTEGER DEFAULT 0, - usd REAL DEFAULT 0.0, - -- 0 means the model had no catalog entry: tokens are real, dollars are absent. Must - -- never render as $0.00 of saving -- a silent zero looks like a measured result. - priced INTEGER DEFAULT 1, - -- 0 where an edit touched already-cached history, whose cache-write penalty lands on - -- the NEXT turn and is therefore not netted into `usd`. - prefix_safe INTEGER DEFAULT 1, - edits_applied INTEGER DEFAULT 0, - -- Lever-authored counters, numeric values only -- see LocalStore._numeric_diagnostics. - diagnostics TEXT, - note TEXT -); -CREATE INDEX IF NOT EXISTS idx_lever_turns_ts ON lever_turns(ts); -CREATE INDEX IF NOT EXISTS idx_lever_turns_lever ON lever_turns(lever); """ @@ -152,133 +106,8 @@ def record_log(self, row: Any) -> None: except Exception: # pragma: no cover - defensive log.debug("[local_store] failed to record a turn", exc_info=True) - @staticmethod - def _numeric_diagnostics(diagnostics: Any) -> Optional[str]: - """A lever's diagnostics, numbers only, as JSON — or ``None``. - - This store's one invariant is that it holds numbers and never text from a developer's - session. Diagnostics are authored by a third-party lever package, so they are the one - field here that could carry arbitrary strings — a lever logging the command it - matched would quietly put a shell line into the database. Numeric values survive, - everything else is dropped, and the invariant stays a property of the code rather - than a promise about third-party behaviour. - """ - if not isinstance(diagnostics, Mapping): - return None - clean = { - str(k): v - for k, v in diagnostics.items() - if isinstance(v, (int, float)) and not isinstance(v, bool) - } - return json.dumps(clean, sort_keys=True) if clean else None - - def record_lever_turns(self, rows: Iterable[Any]) -> int: - """Persist measured lever results for one turn. Never raises. - - Takes the whole batch for a turn in one transaction: the rows describe a single - request, and half of them landing would leave the rail ranking levers against - different denominators. - """ - prepared = [] - for m in rows or (): - try: - edits = getattr(m, "edits", ()) or () - prepared.append(( - getattr(m, "ts", None) or time.time(), - getattr(m, "request_id", ""), - getattr(m, "session_id", None), - getattr(m, "lever", ""), - getattr(m, "mode", ""), - getattr(m, "model", ""), - int(getattr(m, "baseline_tokens", 0)), - int(getattr(m, "counterfactual_tokens", 0)), - int(getattr(m, "removed_tokens", 0)), - int(getattr(m, "from_cache_write", 0)), - int(getattr(m, "from_input", 0)), - int(getattr(m, "from_cache_read", 0)), - float(getattr(m, "usd", 0.0)), - 1 if getattr(m, "priced", True) else 0, - 0 if any(e.applied and not e.prefix_safe for e in edits) else 1, - sum(1 for e in edits if e.applied), - self._numeric_diagnostics(getattr(m, "diagnostics", None)), - getattr(m, "note", ""), - )) - except Exception: - log.debug("[local_store] skipped a malformed lever row", exc_info=True) - if not prepared: - return 0 - try: - with self._lock: - self._db.executemany( - "INSERT INTO lever_turns (ts, request_id, session_id, lever, mode, model," - " baseline_tokens, counterfactual_tokens, removed_tokens," - " from_cache_write, from_input, from_cache_read, usd, priced," - " prefix_safe, edits_applied, diagnostics, note)" - " VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", - prepared, - ) - self._db.commit() - return len(prepared) - except Exception: # pragma: no cover - defensive - log.debug("[local_store] failed to record lever turns", exc_info=True) - return 0 - # -- read -------------------------------------------------------------------------- - def lever_summary(self, since: Optional[float] = None) -> Dict[str, Any]: - """Measured lever results, aggregated per lever. What the rail's live half renders. - - Aggregated **per lever and never totalled**, the same discipline - ``levers.ledger.LedgerReport`` documents: two levers can claim the same bytes, so - adding their savings produces a number larger than anything they could jointly - deliver. Ranking answers the question actually being asked. - - Only ``priced`` rows contribute dollars. Unpriced rows still contribute their token - counts and are surfaced separately — a model with no catalog entry saved real tokens, - and rendering that as $0.00 would read as "this lever does nothing". - """ - where, args = ("WHERE ts >= ?", (since,)) if since else ("", ()) - with self._lock: - cur = self._db.execute( - f"""SELECT lever, - COUNT(*) AS turns, - COALESCE(SUM(removed_tokens), 0) AS removed_tokens, - COALESCE(SUM(CASE WHEN priced=1 THEN usd END), 0.0) AS usd, - COALESCE(SUM(from_cache_write), 0) AS from_cache_write, - COALESCE(SUM(from_input), 0) AS from_input, - COALESCE(SUM(from_cache_read), 0) AS from_cache_read, - SUM(CASE WHEN priced=0 THEN 1 ELSE 0 END) AS unpriced_turns, - SUM(CASE WHEN prefix_safe=0 THEN 1 ELSE 0 END) AS unsafe_turns, - COALESCE(SUM(edits_applied), 0) AS edits_applied, - MAX(ts) AS last_ts - FROM lever_turns {where} - GROUP BY lever - ORDER BY usd DESC""", - args, - ) - cols = [c[0] for c in cur.description] - by_lever = [dict(zip(cols, r)) for r in cur.fetchall()] - cur = self._db.execute( - f"SELECT COUNT(*), COUNT(DISTINCT request_id) FROM lever_turns {where}", args - ) - n_rows, n_turns = cur.fetchone() - return { - "by_lever": by_lever, - "rows": n_rows, - "turns_observed": n_turns, - # Deliberately absent: a `total_usd`. See the docstring. - } - - def recent_lever_turns(self, limit: int = 50) -> List[Dict[str, Any]]: - with self._lock: - cur = self._db.execute( - "SELECT ts, lever, mode, model, removed_tokens, usd, priced, prefix_safe," - " edits_applied, note FROM lever_turns ORDER BY ts DESC LIMIT ?", - (limit,), - ) - cols = [c[0] for c in cur.description] - return [dict(zip(cols, r)) for r in cur.fetchall()] - def summary(self, since: Optional[float] = None) -> Dict[str, Any]: """Aggregates for the dashboard.""" where, args = ("WHERE ts >= ?", (since,)) if since else ("", ()) diff --git a/src/ace/gateway/messages.py b/src/ace/gateway/messages.py index 6d7a0ff..5daac83 100644 --- a/src/ace/gateway/messages.py +++ b/src/ace/gateway/messages.py @@ -538,46 +538,6 @@ async def _body() -> AsyncIterator[bytes]: ) -class _LocalRequestLog: - """The telemetry row shape, for a sidecar that has no telemetry package. - - ``ace.observability.telemetry.RequestLog`` is the cloud gateway's row and does not exist - in this distribution — the sidecar was extracted from that tree without it. The import - was unconditional, it raised ``ModuleNotFoundError`` on **every** turn, and the caller - catches ``Exception`` around accounting so that nothing must ever cost a developer their - response. The result was silent: ``~/.ace/telemetry.db`` stayed empty, ``store.summary()`` - returned zeros, and the dashboard's live panel reported no spend on a sidecar that was - relaying traffic correctly the whole time. - - A permissive attribute bag rather than a fixed dataclass, because the consumer - (``LocalStore.record_log``) reads by ``getattr`` with defaults, and pinning a field list - here would create a second definition to keep in step with the cloud one. - """ - - __slots__ = ("__dict__",) - - def __init__(self, **fields: Any) -> None: - self.__dict__.update(fields) - - def __repr__(self) -> str: # pragma: no cover - debugging aid - return f"_LocalRequestLog({self.__dict__!r})" - - -def _request_log_class(): - """The cloud gateway's ``RequestLog`` when this tree has one, else the local stand-in. - - Resolved per call and not cached: the import is cheap once Python has it in - ``sys.modules``, and caching a negative result would defeat a deployment that adds the - telemetry package later. - """ - try: - from ace.observability.telemetry import RequestLog - - return RequestLog - except Exception: - return _LocalRequestLog - - def usage_to_request_log( usage: StreamUsage, *, @@ -597,7 +557,7 @@ def usage_to_request_log( cache, not ACE's semantic cache (``cache_hit`` / ``cache_served``, left False here — no ACE lever ran on this path in Phase 0). See the RequestLog field comments. """ - RequestLog = _request_log_class() + from ace.observability.telemetry import RequestLog cost = usage.cost() write_5m, write_1h = usage.split_cache_writes() @@ -637,28 +597,6 @@ def usage_to_request_log( ) -def _levers_usage(usage: StreamUsage): - """Project this route's usage record onto the provider-neutral one levers read. - - A deliberate narrowing, not a copy. ``levers.types.Usage`` carries a TTL *breakdown* - rather than Anthropic's ``ephemeral_5m``/``ephemeral_1h`` field names, because the - cache-write premium is a provider property and a lever tuned against one provider's cache - economics gives wrong answers on another. Doing the translation here keeps the wire - vocabulary on this side of the seam. - """ - from ace.sidecar.levers.types import Usage - - write_5m, write_1h = usage.split_cache_writes() - by_ttl = {k: v for k, v in (("5m", write_5m), ("1h", write_1h)) if v} - return Usage( - input_tokens=usage.input_tokens or 0, - output_tokens=usage.output_tokens or 0, - cache_read_tokens=usage.cache_read_input_tokens or 0, - cache_write_tokens=write_5m + write_1h, - cache_write_by_ttl=by_ttl, - ) - - def install_messages_route( app, *, @@ -669,32 +607,16 @@ def install_messages_route( capture=None, auth_config: Optional["AuthConfig"] = None, byok=None, - shadow=None, ) -> None: """Mount ``POST /v1/messages`` on ``app``. Self-contained on purpose: it takes an app and a config rather than threading through - ``proxy.create_app``'s ~40-parameter factory — keeping the seam this thin is what lets - the P0-5 local sidecar mount this route alone, without the cloud gateway's - cache/router/telemetry stack. + ``proxy.create_app``'s ~40-parameter factory. Phase 0 runs no levers, so it needs none + of that wiring — and keeping the seam this thin is what lets the P0-5 local sidecar + mount this route alone, without the cloud gateway's cache/router/telemetry stack. ``client`` injects an ``httpx.AsyncClient`` so the P0-4 conformance suite can drive this exact production branch through ``MockTransport`` with no live call. - - ``shadow`` is an optional :class:`ace.sidecar.levers.shadow.ShadowRunner`. It is the one - place this module does anything a Phase 0 relay did not, and it was designed to be - unable to violate the fidelity invariant: - - * it never sees ``raw`` — only ``parsed``, the throwaway copy this route already makes to - decide streaming and model, so there is no object shared with what goes upstream; - * it runs **after** the response has been served, on a worker thread, so a counting round - trip cannot land in the developer's turn latency; - * it is skipped entirely — one cached entry-point lookup — when no lever package is - installed, which is the ordinary state. - - It also supplies the credential problem's only solution: under ``no_key: true`` the - relayed token is the sole credential that can reach the counting endpoint, and it exists - only for the life of this request. """ cfg = config or MessagesConfig.from_env() auth_cfg = auth_config or AuthConfig.from_env() @@ -794,40 +716,6 @@ def _sink(usage: StreamUsage) -> None: log.debug("[messages] accountant.record_log failed", exc_info=True) if on_usage is not None: on_usage(usage) - _shadow(usage) - - def _shadow(usage: StreamUsage) -> None: - """Hand this turn to the levers, detached. Never touches the served response. - - Ordered last in `_sink` on purpose: accounting is the thing that must not be lost, - and a lever package is third-party code. Anything that goes wrong past this point - costs a measurement, never a turn. - """ - if shadow is None or not shadow.enabled: - return - try: - from ace.sidecar.levers.counter import resolve_counter - - # The in-flight credential, adopted once. `set_counter` keeps the first one - # for the life of the process so a refusal is remembered instead of re-asked - # on every turn. - if shadow.counter is None and api_key: - counter, _ = resolve_counter(api_key, auth.scheme) - shadow.set_counter(counter) - - import asyncio - - asyncio.get_running_loop().create_task( - shadow.observe_async( - parsed, - _levers_usage(usage), - model=usage.model or parsed.get("model") or "", - request_id=req_id, - session_id=session_id, - ) - ) - except Exception: # pragma: no cover - a shadow run never surfaces - log.debug("[messages] shadow lever run could not start", exc_info=True) url = cfg.base_url.rstrip("/") + MESSAGES_PATH diff --git a/src/ace/sidecar/app.py b/src/ace/sidecar/app.py index b3b8b79..96cf808 100644 --- a/src/ace/sidecar/app.py +++ b/src/ace/sidecar/app.py @@ -75,26 +75,6 @@ def build_sidecar_app( mode=MODE_LOOPBACK, local_api_key=api_key or auth_env.local_api_key ) - # The measured half of the lever rail. Constructed unconditionally and cheap when nothing - # is installed — `ShadowRunner.enabled` is one cached entry-point lookup — so the - # open-source sidecar on its own pays nothing for a feature it does not have. - # - # Its sink is the telemetry store, which is the whole point of item 4: a lever result is - # produced once, in a background task moments after a response is served, and without a - # row in `lever_turns` it is logged and lost. - shadow = None - try: - from ace.sidecar.levers.shadow import ShadowRunner - - sink = ( - accountant.record_lever_turns - if hasattr(accountant, "record_lever_turns") - else None - ) - shadow = ShadowRunner(sink=sink) - except Exception: # pragma: no cover - levers never block the sidecar starting - log.debug("[sidecar] lever shadow runner unavailable", exc_info=True) - install_messages_route( app, config=cfg, @@ -102,7 +82,6 @@ def build_sidecar_app( accountant=accountant, capture=capture, client=client, - shadow=shadow, ) @app.get("/dashboard", response_class=HTMLResponse) diff --git a/src/ace/sidecar/insights.py b/src/ace/sidecar/insights.py index f7168fa..2973ab3 100644 --- a/src/ace/sidecar/insights.py +++ b/src/ace/sidecar/insights.py @@ -2924,35 +2924,6 @@ def span( _BUILD_CACHE_MAX = 32 -def _lever_rail_payload(scoped: List[Dict[str, Any]]) -> Dict[str, Any]: - """The live half of the lever rail, or a payload saying why there isn't one. - - Imported lazily and wrapped: ``ace.sidecar.levers`` discovers third-party packages, and - nothing a stranger's distribution does at import time may take this dashboard down. A - failure here costs the live column and leaves every measured figure on the page intact. - """ - try: - from ace.sidecar.levers.rail import rail_payload - - return rail_payload(scoped) - except Exception: - log.warning("[levers] rail payload failed; rendering headroom only", exc_info=True) - return {"status": "no_package", "note": "lever rail unavailable", "installed": []} - - -def _refresh_lever_rail(payload: Dict[str, Any], store: Any) -> Dict[str, Any]: - """Live half of the lever rail, re-read from the telemetry store. Never raises.""" - if store is None or not payload: - return payload - try: - from ace.sidecar.levers.rail import refresh_measured - - return refresh_measured(payload, store) - except Exception: - log.warning("[levers] measured rail refresh failed", exc_info=True) - return payload - - def _build_payload( all_sessions: List[Dict[str, Any]], capture: Optional[Dict[str, Any]], @@ -2988,11 +2959,6 @@ def _build_payload( "scorecards": ( scorecards(scoped, agg.get("cost_usd") or 0.0) if agg["available"] else None ), - # The *measured* half of the lever rail, beside `scorecards`' simulated headroom. - # The two are different claims and the renderer must not merge them: headroom is a - # byte-turn estimate of what a lever would be worth, `levers` is what an installed - # one actually measured. Its `status` says which of the two exists. - "levers": _lever_rail_payload(scoped), "files": session_files(agent=agent, all_sessions=all_sessions), "capture": capture or {}, "recommendations": recommendations(agg, capture, sess=scoped), @@ -3056,11 +3022,6 @@ def build( # live keys below are per-request and must not be written into the shared cached dict. out = dict(payload) out["live"] = store.summary() if store is not None else {"turns": 0} - # Re-read for the same reason `live` is: the measured lever rail moves on every proxied - # turn, while the payload around it is memoised on a transcript fingerprint that a - # proxied turn does not change. Cached with the rest, the one live number on the rail - # would be frozen at whatever it read when the transcripts last changed. - out["levers"] = _refresh_lever_rail(out.get("levers") or {}, store) out["recent"] = store.recent(30) if store is not None else [] return out diff --git a/src/ace/sidecar/levers/__init__.py b/src/ace/sidecar/levers/__init__.py deleted file mode 100644 index bf3ed50..0000000 --- a/src/ace/sidecar/levers/__init__.py +++ /dev/null @@ -1,115 +0,0 @@ -"""ace.sidecar.levers — the public contract optimization modules are written against. - -This package contains no optimizations. It defines the normalized session model every lever -reads (:mod:`~ace.sidecar.levers.types`), what a lever is allowed to do -(:mod:`~ace.sidecar.levers.protocol`), and how installed ones are found -(:mod:`~ace.sidecar.levers.registry`). Implementations ship separately and register through -the ``ace.sidecar.levers`` entry-point group. - -Three properties are worth stating once, because everything here follows from them. - -**One lever, every agent.** Levers read the corpus shape that ``insights._scan``, -``_scan_antigravity`` and ``_scan_codex`` already agree on, never a provider's wire format. -Supporting a fourth coding agent is a scanner, not a lever rewrite. - -**Measurement is universal; actuation is not.** Scoring runs off transcripts, so it works -for every agent the sidecar can read, with no proxy and no hooks. Rewriting bytes needs a -write path, and only some agents have one. ``Lever.requires_content`` is where a lever -declares which half it needs, and the registry refuses the mismatch rather than degrading. - -**Levers propose; the ledger prices.** No lever returns a dollar figure. Pricing happens -once, in :mod:`~ace.sidecar.levers.ledger`, where the tokenizer and the rate catalog live — -which is what keeps a saving auditable and keeps provider-specific cache economics out of a -lever that is meant to be provider-neutral. The ledger prices nothing it cannot count -exactly, and it ranks levers rather than totalling them. -""" - -from ace.sidecar.levers.ledger import ( - FIDELITY_MEASURED, - FIDELITY_UNMEASURABLE, - FIDELITY_UNPRICED, - EditCost, - LedgerEntry, - LedgerReport, - price_all, - price_proposal, -) -from ace.sidecar.levers.protocol import ( - MODE_OFF, - MODE_ON, - MODE_SHADOW, - MODES, - RISK_HIGH, - RISK_LOW, - RISK_MEDIUM, - RISK_NONE, - Edit, - EditKind, - Lever, - LeverContext, - Proposal, - TokenCounter, -) -from ace.sidecar.levers.registry import ( - CONFIG_PATH, - ENTRY_POINT_GROUP, - RegisteredLever, - discover, - load_settings, - propose_safely, - resolve_modes, -) -from ace.sidecar.levers.types import ( - ContentRef, - ContentUnavailable, - Session, - ToolCall, - Turn, - Usage, - from_corpus_session, - from_corpus_sessions, -) - -__all__ = [ - # types - "Session", - "Turn", - "ToolCall", - "Usage", - "ContentRef", - "ContentUnavailable", - "from_corpus_session", - "from_corpus_sessions", - # protocol - "Lever", - "LeverContext", - "Proposal", - "Edit", - "EditKind", - "TokenCounter", - "MODE_OFF", - "MODE_SHADOW", - "MODE_ON", - "MODES", - "RISK_NONE", - "RISK_LOW", - "RISK_MEDIUM", - "RISK_HIGH", - # registry - "discover", - "resolve_modes", - "load_settings", - "propose_safely", - "RegisteredLever", - "ENTRY_POINT_GROUP", - "CONFIG_PATH", - # ledger - "price_proposal", - "price_all", - "LedgerEntry", - "LedgerReport", - "EditCost", - "FIDELITY_MEASURED", - "FIDELITY_UNMEASURABLE", - "FIDELITY_UNPRICED", -] diff --git a/src/ace/sidecar/levers/counter.py b/src/ace/sidecar/levers/counter.py deleted file mode 100644 index cd2289a..0000000 --- a/src/ace/sidecar/levers/counter.py +++ /dev/null @@ -1,240 +0,0 @@ -"""ace.sidecar.levers.counter — an exact token counter, built from the credential in hand. - -Why this module exists at all ------------------------------ -The ledger prices a prompt that was never sent. The baseline side of every counterfactual is -ground truth — the provider's own per-turn counts, read off the transcript — but the proposed -side is text nobody submitted, so its tokens have to be *produced*. Approximating there is -what turns a measured saving back into an estimate, which is the one thing -:mod:`ace.sidecar.levers.ledger` refuses to do. Hence an exact counter, and hence a network -call: for Claude the only exact counter is Anthropic's ``POST /v1/messages/count_tokens``. - -Why it takes the credential instead of reading the environment --------------------------------------------------------------- -The previous version sniffed ``ANTHROPIC_API_KEY`` out of ``os.environ``. On the deployment -that matters that variable is empty: the sidecar's own default is ``{"no_key": true}``, and a -Claude Code session on a **subscription** never has an API key at all — it authenticates with -an OAuth token that exists only for the life of a request, in the ``Authorization`` header the -proxy is already relaying. - -So the credential is passed in, from whoever has one: - -* the proxy turn path hands over the in-flight credential it is about to relay upstream - (:mod:`ace.gateway.messages`), which is the only path that has one under ``no_key``; -* the dashboard falls back to the environment, for a developer who does export a key. - -There is no preflight probe ---------------------------- -An earlier plan for this module called for one live call to establish whether a subscription -OAuth token is accepted by the counting endpoint. It isn't needed: the first real count -answers the same question as a side effect, and a dedicated ping only adds a round trip and a -second code path that can disagree with the first. - -What *is* needed is that a refusal be remembered and explained. :class:`AnthropicCounter` -latches the first authentication failure, stops calling, and keeps the reason as -:attr:`~AnthropicCounter.note`, so the rail can render ``no_counter — the counting endpoint -rejected this credential (401)`` rather than an unexplained blank. A transport blip is -treated differently and is *not* latched: it costs one edit, not the whole feature. - -Presentation is delegated, never re-derived --------------------------------------------- -Building the auth headers here would be a second implementation of a rule this repository -already got wrong once: an OAuth token sent as ``x-api-key`` is rejected, and ``/v1/messages`` -additionally requires the ``oauth-2025-04-20`` beta. ``messages_auth.upstream_auth_headers`` -owns that rule for the relay, so it owns it here too. The counting endpoint lives under the -same ``/v1/messages`` prefix and takes the same credentials as the route it belongs to. -""" - -from __future__ import annotations - -import logging -import os -import threading -from typing import Any, Mapping, Optional, Tuple - -import httpx - -from ace.gateway.messages_auth import ( - SCHEME_API_KEY, - SCHEME_BEARER, - upstream_auth_headers, -) - -__all__ = ["COUNT_TOKENS_PATH", "COUNTABLE_FIELDS", "AnthropicCounter", "resolve_counter"] - -log = logging.getLogger(__name__) - -# The only request fields ``/v1/messages/count_tokens`` accepts. Everything else on a real -# turn — ``stream``, ``max_tokens``, ``temperature``, ``metadata`` — is rejected as an unknown -# parameter, so a body cannot be forwarded to the counter as-is. -# -# All four listed here contribute tokens and must be kept: dropping ``system`` or ``tools`` -# would under-count the prompt by the largest stable part of an agent request. That does not -# matter for a *delta* between two counts taken the same way, but it matters enormously for -# the cross-check against the provider's own reported prompt size. -COUNTABLE_FIELDS = ("model", "messages", "system", "tools", "tool_choice", "thinking") - -ANTHROPIC_DEFAULT_BASE_URL = "https://api.anthropic.com" -COUNT_TOKENS_PATH = "/v1/messages/count_tokens" - -# Long enough for a real call, short enough that a hung endpoint cannot stall a dashboard -# render or add itself to a developer's turn latency. -_TIMEOUT_S = 10.0 - -# A model must be named for the endpoint to answer, and the count is model-family specific. -# Only used when the caller supplies nothing — every real call carries the turn's own model. -_FALLBACK_MODEL = "claude-sonnet-5" - -# Statuses that mean "this credential will never work here". Latched. Anything else — a 429, -# a 500, a timeout — is transient and must not disable counting for the whole process. -_FATAL_AUTH_STATUSES = (401, 403) - - -class AnthropicCounter: - """Exact token counts from Anthropic, over one credential. Satisfies ``TokenCounter``. - - Callable, and deliberately stateful: the state is the single fact worth remembering - across calls, which is whether this credential is accepted at all. - - Raises rather than returning a sentinel on failure. The ledger already treats a raising - counter as "this edit is unmeasurable" and prices nothing for it, which is the correct - outcome — a counter that returned ``0`` for an uncountable string would silently report - the entire original as saved. - """ - - __slots__ = ("_credential", "_scheme", "_url", "_client", "_lock", "_dead", "note", "calls") - - def __init__( - self, - credential: str, - scheme: str = SCHEME_API_KEY, - *, - base_url: Optional[str] = None, - client: Optional[httpx.Client] = None, - ) -> None: - self._credential = credential - self._scheme = scheme - self._url = (base_url or ANTHROPIC_DEFAULT_BASE_URL).rstrip("/") + COUNT_TOKENS_PATH - # Injectable so the suite can drive this exact branch through MockTransport with no - # live call — the same discipline `install_messages_route` uses for its relay client. - self._client = client - self._lock = threading.Lock() - self._dead: Optional[str] = None - self.note = "Anthropic /v1/messages/count_tokens" - self.calls = 0 - - @property - def usable(self) -> bool: - """False once the credential has been definitively refused.""" - return self._dead is None - - def _http(self) -> httpx.Client: - if self._client is None: - self._client = httpx.Client(timeout=_TIMEOUT_S) - return self._client - - def __call__(self, text: str, *, model: str) -> int: - """Tokens in one standalone string. The ``TokenCounter`` protocol's shape.""" - return self.count_body( - {"model": model or _FALLBACK_MODEL, - "messages": [{"role": "user", "content": text}]} - ) - - def count_body(self, body: Mapping[str, Any]) -> int: - """Tokens in a whole ``/v1/messages`` request — system prompt and tools included. - - This is what the live shadow path needs. A lever's edit lands inside one tool result - buried in a long ``messages`` array, and the quantity that matters is what the whole - prompt would have cost, not what the edited fragment costs on its own: an edit can - change block boundaries and therefore tokenize differently in place than in isolation. - - Only a *delta* between two bodies counted this way is exact. Comparing one of these - against the provider's reported ``prompt_tokens`` is a cross-check, not a measurement - — the two include slightly different scaffolding. - """ - if self._dead is not None: - raise RuntimeError(self._dead) - - payload = {k: body[k] for k in COUNTABLE_FIELDS if k in body} - payload.setdefault("model", _FALLBACK_MODEL) - - headers = { - "anthropic-version": "2023-06-01", - "content-type": "application/json", - } - # The OAuth beta is merged in here exactly as the relay does it; presenting a - # subscription token without it is the failure this indirection exists to avoid. - headers.update(upstream_auth_headers(self._credential, self._scheme)) - - resp = self._http().post( - self._url, - json=payload, - headers=headers, - timeout=_TIMEOUT_S, - ) - - if resp.status_code in _FATAL_AUTH_STATUSES: - # Latch, and say which credential shape was refused. This is the line that turns - # "the dashboard shows nothing" into an answerable question, and it is the one - # place the OAuth-vs-API-key outcome is actually established. - kind = "OAuth token" if self._scheme == SCHEME_BEARER else "API key" - with self._lock: - self._dead = ( - f"the counting endpoint rejected this {kind} ({resp.status_code}) — " - f"exact counts need a credential it accepts" - ) - self.note = self._dead - log.warning("[levers] %s", self._dead) - raise RuntimeError(self._dead) - - # Not latched: a rate limit or a 5xx says nothing about the credential, and disabling - # measurement for the process because one call was throttled would be a bug that - # looks exactly like the feature not working. - resp.raise_for_status() - - n = int((resp.json() or {}).get("input_tokens", -1)) - if n < 0: - raise RuntimeError("counting endpoint returned no input_tokens") - with self._lock: - self.calls += 1 - return n - - -def resolve_counter( - credential: Optional[str] = None, - scheme: str = SCHEME_API_KEY, - *, - base_url: Optional[str] = None, - client: Optional[httpx.Client] = None, -) -> Tuple[Optional[AnthropicCounter], str]: - """An exact counter and where its credential came from, or ``(None, reason)``. - - Exactness is the whole requirement, so only the model vendor's own counter is offered. - ``tiktoken`` is deliberately not a fallback: it is OpenAI's BPE and merely a *proxy* for - anything else, and a proxy here turns a measured saving into an estimate wearing a dollar - sign. Neither is ``bytes / 4`` — see ``strategies.BYTES_PER_TOKEN``, where the real - measured ratio on agent tool output is closer to 2.8 and the 4.0 everyone reaches for sits - at the 99th percentile of the distribution. - - Returning ``None`` is an ordinary outcome and not a failure. It costs the live column and - leaves the simulated headroom rail exactly as it is. - """ - if not credential: - # No in-flight credential: the dashboard path, rendered outside any request. An - # exported key is the only thing that can serve it. - env_key = (os.getenv("ANTHROPIC_API_KEY") or "").strip() - env_tok = (os.getenv("ANTHROPIC_AUTH_TOKEN") or "").strip() - if env_key: - credential, scheme = env_key, SCHEME_API_KEY - elif env_tok: - credential, scheme = env_tok, SCHEME_BEARER - else: - return None, ( - "no credential available — this sidecar runs on `no_key: true`, so exact " - "counts come from the token a proxied turn relays, or from an exported " - "ANTHROPIC_API_KEY" - ) - - counter = AnthropicCounter(credential, scheme, base_url=base_url, client=client) - kind = "relayed OAuth token" if scheme == SCHEME_BEARER else "API key" - return counter, f"Anthropic /v1/messages/count_tokens via {kind}" diff --git a/src/ace/sidecar/levers/ledger.py b/src/ace/sidecar/levers/ledger.py deleted file mode 100644 index 677a2e9..0000000 --- a/src/ace/sidecar/levers/ledger.py +++ /dev/null @@ -1,450 +0,0 @@ -"""ace.sidecar.levers.ledger — the one place a lever's proposal becomes money. - -The rule --------- -**No estimates.** Both sides of every figure here are either measured or absent. - -The baseline side is ground truth: the provider's own per-turn token counts, read off the -transcript. The proposed side is a prompt that was never sent, so its tokens have to be -produced — and the only honest way to produce them is to count the actual text with the -provider's own counter (``ctx.count_tokens``: Anthropic's ``/v1/messages/count_tokens`` for -Claude, tiktoken for OpenAI models where it is that vendor's own BPE, Gemini's counting -endpoint for Gemini). - -Where the text is not in hand, this module returns :data:`FIDELITY_UNMEASURABLE` and prices -nothing. It does **not** fall back to ``result_bytes / 4``. A ratio-derived saving is an -estimate wearing a dollar sign, and one number here that a developer can contradict against -their own invoice discredits the measured half of the dashboard along with it. Ranking -levers without content is a real and useful job — it is what ``strategies.py`` does, in -byte-turns, explicitly labelled a simulation — but it is not this module's job. - -The three arithmetic traps this module exists to avoid ------------------------------------------------------ -**1. Ignoring the cache-write penalty.** Every lever removes tokens from a prompt that the -provider was mostly serving from cache at ~0.1x. Removing them saves that cheap rate, not -the fresh-input rate. And an edit that changes content the cache has *already* seen -invalidates the prefix from that point, so the next turn re-writes it at a premium (1.25x at -Anthropic's 5-minute TTL, 2x at one hour). A lever that reports gross saving and omits the -penalty can report a win on an edit that cost money. :class:`EditCost` carries both legs and -``net_usd`` is the only figure meant to be quoted. - -**2. Summing levers.** Two levers can target the same bytes; scored alone their figures -overlap. :class:`LedgerReport` therefore ranks and never totals — the same discipline -``strategies.STANDALONE`` already documents. - -**3. Calling an unpriced model free.** A model with no catalog entry yields -``priced=False`` and zeroes, which must render as "unpriced", never as $0.00 of spend. A -silent zero looks like a cost win. - -What is out of scope --------------------- -Only *volume* is priced here: edits that put fewer tokens in a later prompt. Accounting -levers — buying a longer cache TTL, normalising a mutating field so a prefix stops -breaking — convert price without sending less, produce no :class:`Edit`, and are worth -exactly nothing against a token cap. Conflating the two is the easiest way to overstate this -product, so they do not share a number with it. -""" - -from __future__ import annotations - -import math -from dataclasses import dataclass, field -from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple - -from ace.gateway.pricing import Rates, rates_for -from ace.sidecar.levers.protocol import Edit, LeverContext, Proposal -from ace.sidecar.levers.types import Session, ToolCall, Turn - -__all__ = [ - "FIDELITY_MEASURED", - "FIDELITY_UNMEASURABLE", - "FIDELITY_UNPRICED", - "EditCost", - "LedgerEntry", - "LedgerReport", - "price_proposal", - "price_all", -] - -# Counted exactly, with a catalog rate behind every dollar. The only tier that may be quoted. -FIDELITY_MEASURED = "measured" -# The text was not in hand, so the token delta could not be counted. No dollars, by design. -FIDELITY_UNMEASURABLE = "unmeasurable" -# Counted exactly, but the model has no catalog entry. Tokens are real; dollars are absent. -FIDELITY_UNPRICED = "unpriced" - -_MTOK = 1_000_000.0 - -# What a dropped result is replaced by in context — a short pointer, not nothing. Matches -# ``strategies.POINTER_BYTES``; at ~4 bytes/token it is a rounding error, but counting it as -# zero would claim a saving the applied lever does not actually deliver. -_POINTER_TOKENS = 30 - - -@dataclass(frozen=True, slots=True) -class EditCost: - """One edit, priced against the turns that actually carried its bytes. - - ``prefix_safe`` is the distinction that decides whether this edit is nearly free or has - to earn back a penalty first. A tool result created at turn *i* first enters the prompt - at turn *i+1* and is written to the cache there for the first time. Editing it before - that write costs nothing — the same single write happens, just smaller. Editing content - the cache has already stored invalidates the prefix from that point on, and the next turn - re-writes the remainder at the write premium. - - That is why tail-acting levers (truncate a fresh dump, strip a screenshot on the way in) - can ship far earlier than history-rewriting ones: they are prefix-safe by construction - and carry ``cache_write_penalty_usd == 0``. - """ - - lever: str - turn_index: int - call_index: int - kind: str - reason: str - model: str - - removed_tokens: int = 0 - # First turn whose prompt is smaller because of this edit. - apply_at: int = 0 - # How many turns carried the removed tokens and now do not. - turns_carried: int = 0 - - prefix_safe: bool = True - invalidated_tokens: int = 0 - cache_write_ttl: str = "5m" - - gross_saving_usd: float = 0.0 - cache_write_penalty_usd: float = 0.0 - priced: bool = True - - @property - def net_usd(self) -> float: - """The only figure meant to be quoted. May be negative — that is the point.""" - return self.gross_saving_usd - self.cache_write_penalty_usd - - @property - def per_turn_saving_usd(self) -> float: - return self.gross_saving_usd / self.turns_carried if self.turns_carried else 0.0 - - @property - def break_even_turn(self) -> Optional[int]: - """The turn at which this edit stops costing money and starts saving it. - - ``None`` when there is no penalty to earn back (a prefix-safe edit is in profit - immediately) or when the edit saves nothing per turn. This is the number worth - putting in front of a developer: "compacting here costs $0.04 now and saves - $0.011/turn — you break even at turn 7, and this session ran 60." - """ - per_turn = self.per_turn_saving_usd - if self.cache_write_penalty_usd <= 0.0 or per_turn <= 0.0: - return None - return self.apply_at + math.ceil(self.cache_write_penalty_usd / per_turn) - - -@dataclass(frozen=True, slots=True) -class LedgerEntry: - """One lever, on one session, priced. - - ``fidelity`` qualifies every number above it and must be carried into the UI. A - :data:`FIDELITY_UNMEASURABLE` entry has real ``diagnostics`` and no dollars; rendering it - beside a measured entry without the label is how an estimate ends up quoted as a - measurement. - """ - - lever: str - session_id: str - agent: str - fidelity: str = FIDELITY_MEASURED - edits: Tuple[EditCost, ...] = () - diagnostics: Mapping[str, Any] = field(default_factory=dict) - # Why nothing was priced, when fidelity is not MEASURED. - note: str = "" - # Provenance for every rate used, so a figure can cite the price list that produced it. - rate_sources: Tuple[Tuple[str, str, str], ...] = () # (model, source, as_of) - - @property - def removed_tokens(self) -> int: - return sum(e.removed_tokens for e in self.edits) - - @property - def gross_saving_usd(self) -> float: - return sum(e.gross_saving_usd for e in self.edits) - - @property - def cache_write_penalty_usd(self) -> float: - return sum(e.cache_write_penalty_usd for e in self.edits) - - @property - def net_usd(self) -> float: - return self.gross_saving_usd - self.cache_write_penalty_usd - - @property - def priced(self) -> bool: - return self.fidelity == FIDELITY_MEASURED and bool(self.edits) - - -@dataclass(frozen=True, slots=True) -class LedgerReport: - """Every lever's entries, ranked. Deliberately without a total. - - There is no ``total_usd`` here and there must not be one. Levers are scored alone, so two - of them can claim the same bytes and their figures overlap; adding them up produces a - number that is larger than anything the levers could jointly deliver. Ranking answers the - question that is actually being asked — which lever is worth building or enabling first. - """ - - entries: Tuple[LedgerEntry, ...] = () - - def by_lever(self) -> Dict[str, float]: - """``{lever_id: net usd}`` summed across sessions — safe, because it is one lever.""" - out: Dict[str, float] = {} - for e in self.entries: - if e.priced: - out[e.lever] = out.get(e.lever, 0.0) + e.net_usd - return out - - def ranked(self) -> List[Tuple[str, float]]: - """Levers, best first. The ordering the rail exists to show.""" - return sorted(self.by_lever().items(), key=lambda kv: -kv[1]) - - def unmeasured(self) -> Tuple[LedgerEntry, ...]: - """Entries that produced no dollars, with the reason. Surface these, do not drop them.""" - return tuple(e for e in self.entries if e.fidelity != FIDELITY_MEASURED) - - -def _call_at(session: Session, edit: Edit) -> Optional[ToolCall]: - if not (0 <= edit.turn_index < len(session.turns)): - return None - calls = session.turns[edit.turn_index].calls - if not (0 <= edit.call_index < len(calls)): - return None - return calls[edit.call_index] - - -def _count(text: Any, model: str, ctx: LeverContext) -> Optional[int]: - """Exact token count, or ``None`` when the text is not countable text. - - A non-string body (a list of content parts) is serialized the way the provider would - render it only if the lever already reduced it to text. Anything else returns ``None`` - and the edit goes unmeasured — guessing at a multimodal part's token cost is precisely - the estimate this module refuses to make. - """ - if text is None: - return None - if not isinstance(text, str): - return None - try: - n = ctx.count_tokens(text, model=model) - except Exception: - return None - return int(n) if n is not None and n >= 0 else None - - -def _removed_tokens( - edit: Edit, call: ToolCall, model: str, ctx: LeverContext -) -> Optional[int]: - """Tokens this edit takes out of every later prompt. Exact, or ``None``. - - Requires the original bytes for every kind except ``expire``, which removes the whole - result from later prompts and so still needs the result's own token count. - """ - if call.content is None or not call.content.available: - return None - try: - body = call.content.resolve() - except Exception: - return None - - original = _count(body, model, ctx) - if original is None: - return None - - if edit.kind == "drop": - return max(0, original - _POINTER_TOKENS) - if edit.kind == "expire": - # Nothing is rewritten; the result simply stops being resident after ``live_until``. - return original - if edit.kind == "replace": - kept = _count(edit.replacement or "", model, ctx) - return None if kept is None else max(0, original - kept) - if edit.kind == "truncate": - if edit.keep_bytes is None: - return None - if not isinstance(body, str): - return None - kept = _count(body[: edit.keep_bytes], model, ctx) - return None if kept is None else max(0, original - kept) - return None - - -def _apply_at(edit: Edit) -> int: - """First turn whose prompt this edit changes. - - A tool result created at turn *i* is not in turn *i*'s own prompt — it lands in *i+1*'s. - ``expire`` instead takes effect the turn after the result stops being worth keeping. - """ - if edit.kind == "expire": - return (edit.live_until if edit.live_until is not None else edit.turn_index) + 1 - return edit.turn_index + 1 - - -def _invalidated_tokens(session: Session, edit_turn: int, apply_at: int) -> int: - """Cached tokens the prefix loses when this edit lands after the content was cached. - - Derived from two ground-truth numbers and nothing else: the prompt size at the turn the - content was created and at the turn the edit takes effect. What sits between them is what - the cache holds beyond the edit point and must be re-written. - - Capped by the tokens actually served from cache at ``apply_at`` — a prefix cannot lose - more than it held, and reporting a penalty larger than the cache read would overstate the - cost of every history-rewriting lever. - - When the prompt SHRANK between the two turns the subtraction is meaningless: something - else already rewrote the history (a compaction, a context edit), so the edited content's - position can no longer be derived from sizes. The answer there is the whole cached - prefix, not zero. Both readings are wrong, and they are wrong in opposite directions — - assuming zero understates the penalty, which overstates the saving, which is the one - error this module exists to prevent. - """ - turns = session.turns - if not (0 <= edit_turn < len(turns)) or not (0 <= apply_at < len(turns)): - return 0 - cached = turns[apply_at].usage.cache_read_tokens - grew = turns[apply_at].usage.prompt_tokens - turns[edit_turn].usage.prompt_tokens - if grew < 0: - return max(0, int(cached)) - return max(0, min(int(grew), int(cached))) - - -def _ttl_for(turn: Turn) -> str: - """The TTL this turn's cache writes were actually billed at. - - Read from the turn rather than assumed. ``strategies.TTL_SECONDS`` hard-codes one hour - while the default on a Claude Code session is the 5-minute tier, and the two carry - different write premiums (2x vs 1.25x) — an assumed TTL prices the penalty wrong in - whichever direction the assumption is off. - """ - by_ttl = turn.usage.cache_write_by_ttl - if by_ttl: - return max(by_ttl.items(), key=lambda kv: kv[1])[0] - return "5m" - - -def price_proposal( - session: Session, - proposal: Proposal, - ctx: LeverContext, - *, - rates_lookup: Callable[[str], Optional[Rates]] = rates_for, -) -> LedgerEntry: - """Price one lever's proposal against one session. The core of this module. - - Returns an entry rather than raising: an unmeasurable proposal is an ordinary outcome - (the measurement path has no content for any of them) and the caller needs the - diagnostics either way. - """ - n = session.n_turns - costs: List[EditCost] = [] - sources: Dict[str, Tuple[str, str]] = {} - unmeasured = 0 - - for edit in proposal.edits: - call = _call_at(session, edit) - if call is None: - unmeasured += 1 - continue - - apply_at = _apply_at(edit) - turns_carried = n - apply_at - if turns_carried <= 0: - # The result never reached another prompt, so removing it saves nothing. Recorded - # as a zero rather than dropped: "this lever fired on the last turn and therefore - # saved nothing" is a real and useful thing for a rail row to say. - turns_carried = 0 - - model = session.turns[min(apply_at, n - 1)].model if n else "" - removed = _removed_tokens(edit, call, model, ctx) - if removed is None: - unmeasured += 1 - continue - - rates = rates_lookup(model) - prefix_safe = apply_at <= edit.turn_index + 1 - invalidated = ( - 0 if prefix_safe else _invalidated_tokens(session, edit.turn_index, apply_at) - ) - ttl = _ttl_for(session.turns[min(apply_at, n - 1)]) if n else "5m" - - if rates is None: - costs.append( - EditCost( - lever=proposal.lever, turn_index=edit.turn_index, - call_index=edit.call_index, kind=edit.kind, reason=edit.reason, - model=model, removed_tokens=removed, apply_at=apply_at, - turns_carried=turns_carried, prefix_safe=prefix_safe, - invalidated_tokens=invalidated, cache_write_ttl=ttl, priced=False, - ) - ) - continue - - sources[model] = (rates.source, rates.as_of) - # Priced at the CACHE-READ rate, not the fresh-input rate. These tokens were resident - # in a cached prefix and re-read each turn at ~0.1x; valuing them at the input rate - # would inflate every lever tenfold. It is also the conservative direction. - gross = (removed / _MTOK) * rates.cache_read_per_mtok * turns_carried - # The penalty is the DELTA between writing those tokens and reading them, not the - # full write price: they were going to be paid for either way. - penalty = (invalidated / _MTOK) * ( - rates.cache_write_per_mtok(ttl) - rates.cache_read_per_mtok - ) - costs.append( - EditCost( - lever=proposal.lever, turn_index=edit.turn_index, - call_index=edit.call_index, kind=edit.kind, reason=edit.reason, - model=model, removed_tokens=removed, apply_at=apply_at, - turns_carried=turns_carried, prefix_safe=prefix_safe, - invalidated_tokens=invalidated, cache_write_ttl=ttl, - gross_saving_usd=gross, cache_write_penalty_usd=max(0.0, penalty), - priced=True, - ) - ) - - if not costs: - note = ( - "no tool-result content available; token delta cannot be counted exactly" - if proposal.edits - else "lever proposed no edits" - ) - return LedgerEntry( - lever=proposal.lever, session_id=session.id, agent=session.agent, - fidelity=FIDELITY_UNMEASURABLE if proposal.edits else FIDELITY_MEASURED, - diagnostics=dict(proposal.diagnostics), note=note, - ) - - fidelity = ( - FIDELITY_MEASURED if all(c.priced for c in costs) else FIDELITY_UNPRICED - ) - diagnostics = dict(proposal.diagnostics) - if unmeasured: - diagnostics["edits_unmeasured"] = unmeasured - return LedgerEntry( - lever=proposal.lever, session_id=session.id, agent=session.agent, - fidelity=fidelity, edits=tuple(costs), diagnostics=diagnostics, - note="" if fidelity == FIDELITY_MEASURED else "model has no catalog entry — unpriced, not free", - rate_sources=tuple((m, s, a) for m, (s, a) in sorted(sources.items())), - ) - - -def price_all( - pairs: Sequence[Tuple[Session, Proposal]], - ctx: LeverContext, - *, - rates_lookup: Callable[[str], Optional[Rates]] = rates_for, -) -> LedgerReport: - """Price many ``(session, proposal)`` pairs into one ranked report.""" - return LedgerReport( - entries=tuple( - price_proposal(s, p, ctx, rates_lookup=rates_lookup) for s, p in pairs - ) - ) diff --git a/src/ace/sidecar/levers/protocol.py b/src/ace/sidecar/levers/protocol.py deleted file mode 100644 index 01d52c8..0000000 --- a/src/ace/sidecar/levers/protocol.py +++ /dev/null @@ -1,205 +0,0 @@ -"""ace.sidecar.levers.protocol — what a lever is, and what it is forbidden to do. - -A lever proposes; it never prices ---------------------------------- -:meth:`Lever.propose` returns :class:`Edit` objects and nothing else. It does not return -dollars, tokens saved, or a percentage. Pricing happens once, in the ledger, which owns the -tokenizer and the rate catalog and is the only thing that knows the provider's cache-write -premium. - -That split is not tidiness, it is the credibility of the number. A lever that reports its -own saving is a lever that can overstate it, and the three ways this arithmetic has already -gone wrong in this codebase were all self-reporting: - -* compaction savings denominated in whitespace words and priced per BPE token, which - undersold the leg by the word->BPE ratio (see ``ace.gateway.tokenizer``); -* a de-dup lever keyed on file path rather than full tool input, measuring $36.88 where the - provable version measures $0.33; -* the same avoided call counted twice — once as avoided, once as a counterfactual. - -With pricing centralized, a lever cannot commit any of them. It also means a lever needs no -knowledge of which provider it is running against, which is what lets one implementation -serve Claude Code, Antigravity and Codex. - -Counting must be exact, so the counter is injected --------------------------------------------------- -The baseline side of every counterfactual is ground truth: the provider's own token counts, -read off the transcript. The proposed side is a prompt that was never sent, so its tokens -have to be produced — and an approximation there turns a measured claim into an estimate. - -:class:`TokenCounter` is therefore a seam with an exact implementation per model family: -Anthropic's ``POST /v1/messages/count_tokens`` for Claude (tiktoken is a *proxy* for -non-OpenAI models, not a truth), tiktoken for OpenAI models where it is that provider's own -BPE, and the provider's counting endpoint for Gemini. A lever calls ``ctx.count_tokens`` -and stays out of that decision. - -Modes ------ -``off`` / ``shadow`` / ``on`` mirror the cloud gateway's vocabulary on purpose, so the two -products' telemetry reads as one thing. ``shadow`` is the default and should stay the -default: this process sits in front of a developer's real coding session, and a lever that -silently rewrites a prompt owns every unexplained agent failure that follows. Shadow costs -nothing and proves the same number. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import ( - Any, - ClassVar, - Literal, - Mapping, - Optional, - Protocol, - Tuple, - runtime_checkable, -) - -from ace.sidecar.levers.types import Session - -__all__ = [ - "MODE_OFF", - "MODE_SHADOW", - "MODE_ON", - "MODES", - "RISK_NONE", - "RISK_LOW", - "RISK_MEDIUM", - "RISK_HIGH", - "EditKind", - "Edit", - "Proposal", - "TokenCounter", - "LeverContext", - "Lever", -] - -MODE_OFF = "off" -MODE_SHADOW = "shadow" -MODE_ON = "on" -MODES = (MODE_OFF, MODE_SHADOW, MODE_ON) - -# Same vocabulary as ``strategies.LEVER_RISK``, so a lever's declared risk and the rail's -# scored risk are comparable without a mapping table. -RISK_NONE, RISK_LOW, RISK_MEDIUM, RISK_HIGH = "NONE", "LOW", "MEDIUM", "HIGH" - -EditKind = Literal["truncate", "drop", "replace", "expire"] - - -@dataclass(frozen=True, slots=True) -class Edit: - """One proposed change to one tool result. - - Addressed positionally by ``(turn_index, call_index)`` rather than by tool-call id: - ids exist in Claude Code transcripts and not reliably elsewhere, and position is - unambiguous in every scanner's output. ``sig`` rides along for debugging and for the - ledger's audit line, never as the key. - - The four kinds split along a line that decides what an edit costs: - - ``truncate`` / ``drop`` / ``replace`` change bytes. Applied at the tail — to a result - that has not yet entered the cached prefix — they are free. Applied to history they - invalidate the cached prefix from that point on and the next turn pays a full cache - write, which is why the ledger nets that penalty before reporting anything. - - ``expire`` changes nothing about the bytes. It asserts the result stops being worth - keeping resident after ``live_until``, which is an accounting claim about residency, not - a smaller prompt. Volume levers help against a token cap; ``expire`` helps only against a - bill. Conflating the two is the easiest way to overstate this product, so the ledger - reports them separately and never sums them. - """ - - turn_index: int - call_index: int - kind: EditKind - reason: str - sig: str = "" - # truncate: bytes retained from the head (and, if the lever keeps a tail, from the end). - keep_bytes: Optional[int] = None - # replace: the substituted text. Only ever populated in actuation mode, where the lever - # was handed the original bytes to begin with. - replacement: Optional[str] = None - # expire: the last turn index at which this result is still worth holding in context. - live_until: Optional[int] = None - - -@dataclass(frozen=True, slots=True) -class Proposal: - """What one lever would do to one session. No savings figure — see the module docstring.""" - - lever: str - edits: Tuple[Edit, ...] = () - # Free-form counters a lever wants surfaced for debugging or for its dashboard row - # ("loops_detected": 3). Never priced, never summed into a saving. - diagnostics: Mapping[str, Any] = field(default_factory=dict) - - # No ``__bool__``. An edit-free proposal is a legitimate and important result: a loop - # guardrail's whole output is "I detected three runaway tool cycles" with nothing to - # rewrite, and the most valuable thing a truncation lever can report on a clean session - # is that it found nothing to do. Defining truthiness as "has edits" makes ``if - # proposal:`` quietly discard both. Callers test ``proposal.edits`` when they mean edits - # and ``proposal is None`` when they mean the lever declined or failed. - - -class TokenCounter(Protocol): - """Exact token count for ``text`` under ``model``. Must not approximate. - - Implementations may be slow and may do I/O — Anthropic's counting endpoint is a network - call. Levers should call it on whole segments rather than per word, and the runtime is - free to batch or sample across turns; that policy lives in the runtime, not here. - """ - - def __call__(self, text: str, *, model: str) -> int: ... - - -@dataclass(frozen=True, slots=True) -class LeverContext: - """Everything a lever is allowed to depend on. - - Deliberately small. It carries no rate catalog (levers do not price), no database, no - HTTP client and no agent identity beyond what ``Session.agent`` already says. A lever - needing something absent here is a lever reaching past its contract — extend this - dataclass rather than importing around it, so the dependency stays visible at the seam. - """ - - count_tokens: TokenCounter - mode: str = MODE_SHADOW - now: float = 0.0 - # Per-lever configuration from ``~/.ace/config.json``, already narrowed to this lever's - # own key. A lever must tolerate an empty mapping: the common case is a user who enabled - # it and tuned nothing. - settings: Mapping[str, Any] = field(default_factory=dict) - - -@runtime_checkable -class Lever(Protocol): - """One optimization, scored or applied against the normalized session model. - - Implementations live outside this repository. This protocol and - :mod:`ace.sidecar.levers.types` are the entire public surface they compile against, and - both are versioned as a contract: adding an optional field is fine, changing the meaning - of one is not. - - ``requires_content`` is the honest declaration of what a lever needs. A lever reading only - ``sig``/``digest``/``result_bytes`` scores from transcripts alone and therefore works for - every agent the sidecar can read, with no proxy and no hooks. A lever that must rewrite - text needs the bytes in hand, so the registry offers it only where an actuator supplied - them — today that is Claude Code's proxy and hook paths. Declaring ``False`` and then - calling ``ContentRef.resolve`` raises rather than silently degrading. - """ - - id: ClassVar[str] - label: ClassVar[str] - risk: ClassVar[str] - requires_content: ClassVar[bool] - - def propose(self, session: Session, ctx: LeverContext) -> Proposal: - """Edits this lever would make to ``session``. Must not mutate ``session``. - - Called on the measurement path for every session in a developer's history, so it is - expected to be cheap in the ``requires_content = False`` case and to raise nothing: - a lever that throws on one malformed session must not take the dashboard down with - it. The registry isolates failures, but a lever should not rely on that. - """ - ... diff --git a/src/ace/sidecar/levers/rail.py b/src/ace/sidecar/levers/rail.py deleted file mode 100644 index b58a012..0000000 --- a/src/ace/sidecar/levers/rail.py +++ /dev/null @@ -1,276 +0,0 @@ -"""ace.sidecar.levers.rail — the dashboard's view of installed levers. - -Sits between the scanner and the renderer so neither has to know about levers. -``insights._build_payload`` calls :func:`rail_payload` and hands the result through; the -renderer reads it. Nothing here scans transcripts and nothing here writes HTML. - -What this is honest about -------------------------- -The rail already shows what each lever would be *worth* — ``strategies.standalone_levers``, -a byte-turn simulation over the developer's own sessions. That is a headroom estimate and it -is labelled one. This module adds the other half: what an installed lever, run for real, -actually measured. - -Those two numbers must never be confused, so :func:`rail_payload` reports a ``status`` that -says which of them exists, and the renderer is expected to show it. Four states, and three of -them mean "no live number": - -``no_package`` nothing registers against the ``ace.sidecar.levers`` entry-point group. - The ordinary state for the open-source sidecar on its own. -``all_off`` levers are installed but every one resolves to ``off`` in - ``~/.ace/config.json``. Presence is not consent; this is the default even - after installing a lever package. -``no_counter`` levers ran, but no exact token counter is configured, so the ledger priced - nothing. A byte-ratio fallback would produce a number here — which is why - there is none. -``measured`` real edits, exactly counted, priced from the catalog and net of the - cache-write penalty. - -Only ``measured`` may put a dollar figure on the page. -""" - -from __future__ import annotations - -import logging -import time -from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence - -from ace.sidecar.levers.counter import resolve_counter -from ace.sidecar.levers.ledger import FIDELITY_MEASURED, price_all -from ace.sidecar.levers.protocol import MODE_OFF, LeverContext, TokenCounter -from ace.sidecar.levers.registry import discover, load_settings, propose_safely, resolve_modes -from ace.sidecar.levers.types import from_corpus_sessions - -__all__ = ["STATUS_NO_PACKAGE", "STATUS_ALL_OFF", "STATUS_NO_COUNTER", "STATUS_MEASURED", - "resolve_counter", "rail_payload", "refresh_measured"] - -log = logging.getLogger(__name__) - -STATUS_NO_PACKAGE = "no_package" -STATUS_ALL_OFF = "all_off" -STATUS_NO_COUNTER = "no_counter" -STATUS_MEASURED = "measured" - -_STATUS_NOTE = { - STATUS_NO_PACKAGE: "no lever package installed — this release measures headroom only", - STATUS_ALL_OFF: "levers installed, all off — enable one in ~/.ace/config.json", - STATUS_NO_COUNTER: ( - "no exact token counter configured — the ledger prices nothing it cannot count" - ), - STATUS_MEASURED: "measured on your own sessions, net of the cache-write penalty", -} - -# Discovery walks installed distribution metadata, which is stable for the life of the -# process and is on the cached dashboard path. Re-walking it per request buys nothing. -_DISCOVERED: Optional[Sequence[Any]] = None - - -def _levers() -> Sequence[Any]: - global _DISCOVERED - if _DISCOVERED is None: - _DISCOVERED = discover() - return _DISCOVERED - - -def _measured(store: Any, *, since: Optional[float] = None) -> Dict[str, Any]: - """Aggregated live results from the telemetry store, or ``{}``. - - Tolerant of a store that predates the ``lever_turns`` table, or of no store at all: this - is an optional column on a dashboard that has to render either way, and an old - ``~/.ace/telemetry.db`` is the common case immediately after an upgrade. - """ - if store is None or not hasattr(store, "lever_summary"): - return {} - try: - summary = store.lever_summary(since=since) - except Exception: - log.debug("[levers] lever_summary failed", exc_info=True) - return {} - return summary if summary.get("by_lever") else {} - - -def _measured_note(prior_status: Optional[str]) -> str: - """The measured note, qualified by what discovery currently says. - - Recorded results and installed packages are two independent facts, and they disagree in - an ordinary way: a developer measures a lever for a week, then uninstalls or disables it. - Reporting ``no_package`` and dropping the rows would hide a real measurement behind a - packaging detail; reporting them unqualified would imply the lever is still running. - Both facts get said. - """ - if prior_status == STATUS_NO_PACKAGE: - return ( - _STATUS_NOTE[STATUS_MEASURED] - + " — from turns already recorded; no lever package is installed now" - ) - if prior_status == STATUS_ALL_OFF: - return ( - _STATUS_NOTE[STATUS_MEASURED] - + " — from turns already recorded; every installed lever is now off" - ) - return _STATUS_NOTE[STATUS_MEASURED] - - -def refresh_measured( - payload: Mapping[str, Any], store: Any, *, since: Optional[float] = None -) -> Dict[str, Any]: - """A rail payload with its live half re-read from ``store``. - - ``insights._build_payload`` is memoised on a transcript fingerprint, which is exactly - right for the installed/modes half — that changes when a package is installed, not when a - turn is proxied. The measured half moves on every turn, so caching it with the rest would - freeze the one number on the rail that is supposed to be alive. Same treatment - ``build`` already gives ``live``. - - Returns a copy. The cached payload is shared, and writing the live keys into it is how a - per-request value ends up served to the next caller. - """ - out = dict(payload) - measured = _measured(store, since=since) - if not measured: - return out - out["measured"] = measured - out["turns_observed"] = measured.get("turns_observed", 0) - out["status"] = STATUS_MEASURED - out["note"] = _measured_note(out.get("status")) - return out - - -def rail_payload( - sessions: Sequence[Mapping[str, Any]], - *, - counter: Optional[TokenCounter] = None, - counter_note: str = "", - credential: Optional[str] = None, - scheme: str = "api_key", - store: Any = None, - since: Optional[float] = None, - config: Optional[Mapping[str, Any]] = None, -) -> Dict[str, Any]: - """What the dashboard needs to render the live half of the lever rail. - - Runs only levers resolved to a non-``off`` mode, over the sessions already scoped to the - dashboard's range and agent filter. Cheap and total when nothing is installed, which is - the common path: one entry-point lookup and an early return. - - ``store`` is the sidecar's :class:`~ace.gateway.local_store.LocalStore`. It carries the - measured half, recorded turn by turn as levers ran on the proxy path, and reading it is - what makes a measured result survive the request that produced it. - - ``credential``/``scheme`` are for a caller that holds one — the proxy path, which is the - only place a credential exists at all under ``no_key: true``. The dashboard renders - outside any request and passes neither, so it falls back to the environment and usually - reports :data:`STATUS_NO_COUNTER` with the reason attached. That is the honest state, not - a degraded one: this half of the rail is measured on proxied turns. - """ - t0 = time.monotonic() - found = _levers() - base: Dict[str, Any] = { - "installed": [ - {"id": r.id, "label": getattr(r.lever, "label", r.id), - "risk": getattr(r.lever, "risk", ""), "dist": r.dist, - "requires_content": bool(getattr(r.lever, "requires_content", False))} - for r in found - ], - "modes": {}, - "by_lever": {}, - "entries": [], - "counter": counter_note, - "measured": {}, - "turns_observed": 0, - "elapsed_ms": 0.0, - } - # Read once, up front: measured rows are recorded history and stay true regardless of what - # is installed or enabled *right now*. Deciding `no_package` before looking would hide a - # real measurement behind a packaging detail. - measured_rows = _measured(store, since=since) - - def _terminal(status: str) -> Dict[str, Any]: - base.update(status=status, note=_STATUS_NOTE[status]) - if measured_rows: - base["measured"] = measured_rows - base["turns_observed"] = measured_rows.get("turns_observed", 0) - base.update(status=STATUS_MEASURED, note=_measured_note(status)) - base["elapsed_ms"] = (time.monotonic() - t0) * 1000.0 - return base - - if not found: - return _terminal(STATUS_NO_PACKAGE) - - modes = resolve_modes(found, config=config) - base["modes"] = modes - active = [r for r in found if modes.get(r.id, MODE_OFF) != MODE_OFF] - if not active: - return _terminal(STATUS_ALL_OFF) - - # The measured half is READ, not recomputed. Levers run once, against the real request - # body, in the proxy's background task (`levers.shadow`); this reads what they recorded. - # - # It cannot be derived here instead. The dashboard has transcripts — hashes and sizes, - # no text — and an exact token delta needs the bytes. Recomputing over history would - # force a byte-ratio fallback, which is the one thing the ledger refuses to do. - if measured_rows: - base["measured"] = measured_rows - base["turns_observed"] = measured_rows.get("turns_observed", 0) - base.update(status=STATUS_MEASURED, note=_STATUS_NOTE[STATUS_MEASURED]) - base["elapsed_ms"] = (time.monotonic() - t0) * 1000.0 - return base - - if counter is None: - counter, counter_note = resolve_counter(credential, scheme) - base["counter"] = counter_note - if counter is None: - base.update( - status=STATUS_NO_COUNTER, - note=f"{_STATUS_NOTE[STATUS_NO_COUNTER]} ({counter_note})", - ) - return base - - # The measurement path holds no tool-result bytes, so a lever needing them is refused by - # `propose_safely` rather than allowed to half-run. That is why a content-requiring lever - # can be installed, enabled, and still contribute nothing here: it needs the proxy or a - # hook to supply the text. - typed = from_corpus_sessions(sessions) - now = time.time() - pairs = [] - for reg in active: - ctx = LeverContext( - count_tokens=counter, - mode=modes[reg.id], - now=now, - settings=load_settings(reg.id, config=config), - ) - for s in typed: - proposal = propose_safely(reg, s, ctx) - if proposal is not None: - pairs.append((s, proposal)) - - # Priced under a context of its own rather than whichever lever's `ctx` the loop above - # happened to exit with. The ledger reads only `count_tokens`, so the leaked binding was - # harmless today — but it silently attributed one lever's `settings` and `mode` to every - # other lever's pricing, which is exactly the kind of thing that stops being harmless the - # first time the ledger reads one more field. - pricing_ctx = LeverContext(count_tokens=counter, now=now) - report = price_all(pairs, pricing_ctx) if pairs else None - if report is not None: - base["by_lever"] = report.by_lever() - base["entries"] = [ - {"lever": e.lever, "session": e.session_id, "agent": e.agent, - "fidelity": e.fidelity, "net_usd": e.net_usd, - "gross_usd": e.gross_saving_usd, "penalty_usd": e.cache_write_penalty_usd, - "removed_tokens": e.removed_tokens, "note": e.note} - for e in report.entries - ] - measured = any(e.fidelity == FIDELITY_MEASURED and e.edits for e in report.entries) - else: - measured = False - - base["elapsed_ms"] = (time.monotonic() - t0) * 1000.0 - if measured: - base.update(status=STATUS_MEASURED, note=_STATUS_NOTE[STATUS_MEASURED]) - else: - base.update( - status=STATUS_NO_COUNTER, - note="levers ran but produced no measurable edit on these sessions", - ) - return base diff --git a/src/ace/sidecar/levers/registry.py b/src/ace/sidecar/levers/registry.py deleted file mode 100644 index bdff265..0000000 --- a/src/ace/sidecar/levers/registry.py +++ /dev/null @@ -1,232 +0,0 @@ -"""ace.sidecar.levers.registry — discovery, mode resolution, and failure isolation. - -Why discovery is by entry point -------------------------------- -This package defines what a lever *is*; it deliberately contains none. Implementations ship -in a separate distribution and register themselves:: - - [project.entry-points."ace.sidecar.levers"] - trajectory_compaction = "ace_skills.compaction:TrajectoryCompaction" - -Nothing here names a lever, imports one, or fails without one. That is the property being -bought: this repository stays a measurement product that gains optimizations when a package -providing them is present, and the package providing them needs no change here to land. - -A missing lever package is the ordinary case, not an error state. :func:`discover` returns -an empty tuple and every caller keeps working — the dashboard renders its measured rail -exactly as it does today. - -The rule this module enforces ------------------------------ -**Presence supplies availability. It never supplies consent.** An installed lever resolves -to ``off`` unless the developer's own ``~/.ace/config.json`` says otherwise, and ``on`` -must be typed per lever. Nothing here defaults a lever to acting on a live session, and no -future default should: the sidecar sits in front of a real coding session, and the cost of -being wrong is a silent corruption three turns later that the developer has no way to -attribute. -""" - -from __future__ import annotations - -import json -import logging -import os -from dataclasses import dataclass -from typing import Any, Dict, Iterable, Mapping, Optional, Tuple - -from ace.sidecar.levers.protocol import ( - MODE_OFF, - MODE_SHADOW, - MODES, - Lever, - LeverContext, - Proposal, -) -from ace.sidecar.levers.types import Session - -__all__ = [ - "ENTRY_POINT_GROUP", - "CONFIG_PATH", - "RegisteredLever", - "discover", - "resolve_modes", - "load_settings", - "propose_safely", -] - -log = logging.getLogger(__name__) - -ENTRY_POINT_GROUP = "ace.sidecar.levers" - -# The same file ``ace.cli`` reads. Kept as a literal rather than imported: ``cli`` builds the -# app and would import this package, and a cycle for one path string is a poor trade. -CONFIG_PATH = os.path.expanduser("~/.ace/config.json") - - -@dataclass(frozen=True, slots=True) -class RegisteredLever: - """A discovered lever plus where it came from. - - ``dist`` is carried so the dashboard can say which package supplied a lever. A developer - seeing an optimization act on their session is entitled to know what installed it. - """ - - lever: Lever - dist: str = "" - - @property - def id(self) -> str: - return getattr(self.lever, "id", "") - - -def discover(group: str = ENTRY_POINT_GROUP) -> Tuple[RegisteredLever, ...]: - """Every lever advertised by an installed distribution. - - Each entry point is loaded independently and a broken one is skipped with a log line - rather than raised: one bad third-party package must not stop ``ace up``. The same - reasoning applies to an object that loads but does not satisfy :class:`Lever` — it is - dropped here, where the message can name the entry point, instead of failing later at a - call site that cannot. - """ - try: - from importlib.metadata import entry_points - except Exception: # pragma: no cover - importlib.metadata is stdlib on 3.12 - return () - - found: list[RegisteredLever] = [] - seen: set[str] = set() - try: - eps: Iterable[Any] = entry_points(group=group) - except Exception: - log.debug("lever entry-point lookup failed", exc_info=True) - return () - - for ep in eps: - try: - obj = ep.load() - except Exception: - log.warning("lever entry point %r failed to load; skipping", ep.name) - continue - # Both a class and a ready instance are accepted. A stateless lever is naturally a - # class; one holding a loaded model artifact is naturally an instance already built - # by the providing package. - try: - candidate = obj() if isinstance(obj, type) else obj - except Exception: - log.warning("lever %r failed to instantiate; skipping", ep.name) - continue - if not isinstance(candidate, Lever): - log.warning("lever %r does not satisfy the Lever protocol; skipping", ep.name) - continue - lever_id = getattr(candidate, "id", "") or ep.name - if lever_id in seen: - log.warning("duplicate lever id %r; keeping the first", lever_id) - continue - seen.add(lever_id) - dist = "" - try: - dist = ep.dist.name if ep.dist is not None else "" - except Exception: - pass - found.append(RegisteredLever(lever=candidate, dist=dist)) - return tuple(found) - - -def _read_config(path: Optional[str] = None) -> Mapping[str, Any]: - """``~/.ace/config.json``, or an empty mapping. A missing or unreadable file is not an error.""" - try: - with open(path or CONFIG_PATH, "r", encoding="utf-8") as fh: - data = json.load(fh) - return data if isinstance(data, Mapping) else {} - except Exception: - return {} - - -def _lever_config(config: Mapping[str, Any]) -> Mapping[str, Any]: - raw = config.get("levers") - return raw if isinstance(raw, Mapping) else {} - - -def resolve_modes( - levers: Iterable[RegisteredLever], - *, - config: Optional[Mapping[str, Any]] = None, - config_path: Optional[str] = None, -) -> Dict[str, str]: - """``{lever_id: mode}`` for every discovered lever. - - Two spellings are accepted under ``levers`` in the config, because the short one is what - people actually type:: - - {"levers": {"trajectory_compaction": "shadow"}} - {"levers": {"trajectory_compaction": {"mode": "on", "keep_recent_images": 2}}} - - An unrecognised mode string resolves to ``off``, not to the default. A typo'd ``"On"`` - that silently became ``shadow`` would be tolerable; one that silently became ``on`` would - not, and the only rule that cannot get that backwards is to refuse the value outright. - """ - cfg = config if config is not None else _read_config(config_path) - per_lever = _lever_config(cfg) - out: Dict[str, str] = {} - for reg in levers: - entry = per_lever.get(reg.id) - if isinstance(entry, Mapping): - raw = entry.get("mode", MODE_OFF) - elif isinstance(entry, str): - raw = entry - elif entry is True: - # A bare `true` is an enablement, and the safe reading of "enabled" is the mode - # that changes nothing about the request. - raw = MODE_SHADOW - else: - raw = MODE_OFF - mode = str(raw).lower() - if mode not in MODES: - log.warning("lever %r has unknown mode %r; treating as off", reg.id, raw) - mode = MODE_OFF - out[reg.id] = mode - return out - - -def load_settings( - lever_id: str, - *, - config: Optional[Mapping[str, Any]] = None, - config_path: Optional[str] = None, -) -> Mapping[str, Any]: - """This lever's own configuration block, minus ``mode``, ready for :class:`LeverContext`.""" - cfg = config if config is not None else _read_config(config_path) - entry = _lever_config(cfg).get(lever_id) - if not isinstance(entry, Mapping): - return {} - return {k: v for k, v in entry.items() if k != "mode"} - - -def propose_safely( - reg: RegisteredLever, session: Session, ctx: LeverContext -) -> Optional[Proposal]: - """Run one lever, returning ``None`` where it raised. - - The measurement path calls every lever over a developer's entire history, which is the - widest input any of this code sees and the likeliest place for a third-party lever to - meet a session shape it did not expect. One such session must cost that lever's row, not - the dashboard. - - A lever declaring ``requires_content`` is refused outright on a measure-only session - rather than allowed to raise :class:`~ace.sidecar.levers.types.ContentUnavailable` part - way through — a partial proposal is worse than none, because the ledger cannot tell it - from a complete one. - """ - if getattr(reg.lever, "requires_content", False) and not any( - call.has_content for _, _, call in session.iter_calls() - ): - return None - try: - proposal = reg.lever.propose(session, ctx) - except Exception: - log.warning("lever %r raised on session %r; skipping", reg.id, session.id, exc_info=True) - return None - if not isinstance(proposal, Proposal): - log.warning("lever %r returned %s; skipping", reg.id, type(proposal)) - return None - return proposal diff --git a/src/ace/sidecar/levers/shadow.py b/src/ace/sidecar/levers/shadow.py deleted file mode 100644 index 88b8758..0000000 --- a/src/ace/sidecar/levers/shadow.py +++ /dev/null @@ -1,736 +0,0 @@ -"""ace.sidecar.levers.shadow — running levers for real, on a live proxied turn. - -This is where "measured" stops being a promise. Everything else in this package either scores -transcripts (hashes and sizes, no text) or defines the contract; here the actual tool-result -bytes are in hand, the credential to count them is in flight, and the provider has just -reported what the turn really cost. - -The three things this path has that the transcript path does not ---------------------------------------------------------------- -1. **The bytes.** A ``/v1/messages`` request body carries every tool result verbatim. That is - what lets a ``requires_content`` lever run at all, and what lets the token delta be - *counted* rather than inferred from ``result_bytes / 4``. -2. **A credential.** Under ``no_key: true`` — the sidecar's own default — the OAuth token the - proxy is about to relay is the only credential in the building. See - :mod:`ace.sidecar.levers.counter`. -3. **Ground-truth usage.** The response says exactly how this turn's prompt was billed across - fresh input, cache reads and cache writes. That split is what turns a token delta into a - dollar figure without assuming anything. - -Shadow means shadow -------------------- -The relayed bytes are never touched. Levers run against a **copy**, the counterfactual is -counted, priced and recorded, and the request the developer's agent actually made goes -upstream byte-for-byte — the invariant ``ace.gateway.messages`` exists to enforce ("parse to -decide, never parse to forward"). A lever resolved to ``on`` still does not mutate the request -here; actuation is a separate seam and deliberately not this one. - -It also runs **after** the response, in a worker thread, so it costs the developer's turn -nothing. Counting is a network round trip and the counter is a synchronous client; doing -either on the hot path would trade a real latency regression for a number nobody asked to -wait for. - -What is measured, and what is honestly not ------------------------------------------- -Measured: the token delta between the real request body and the counterfactual, both counted -the same way through the provider's own counter. A constant offset in either count cancels; -the difference is exact. - -Priced: that delta against **this turn's own usage split**, newest-bucket-first — see -:func:`price_delta`. No projection over future turns. A transcript-driven lever multiplies its -saving by the turns that carried the bytes, because it can see how the session ended; a live -turn cannot, and inventing that multiplier is how a one-turn saving becomes a headline number -that never arrives. - -Not priced: ``expire`` edits, which change no bytes and therefore no prompt (see -``protocol.Edit``), and the cache-write penalty of an edit that rewrites already-cached -history, which is only observable on the *following* turn. Both are recorded and labelled -rather than guessed at. -""" - -from __future__ import annotations - -import json -import logging -import time -from dataclasses import dataclass, field -from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple - -from ace.sidecar.levers.protocol import MODE_OFF, Edit, LeverContext, Proposal -from ace.sidecar.levers.registry import ( - RegisteredLever, - discover, - load_settings, - propose_safely, - resolve_modes, -) -from ace.sidecar.levers.types import ContentRef, Session, ToolCall, Turn, Usage - -__all__ = [ - "TOOL_RESULT", - "TOOL_USE", - "Anchor", - "LiveEdit", - "TurnMeasurement", - "body_to_session", - "apply_edits", - "price_delta", - "ShadowRunner", -] - -log = logging.getLogger(__name__) - -TOOL_USE = "tool_use" -TOOL_RESULT = "tool_result" - -_MTOK = 1_000_000.0 - -# What a dropped result is replaced by. Matches ``ledger._POINTER_TOKENS`` in intent: a -# reference, not nothing, because the applied lever does leave a marker behind and claiming -# otherwise would overstate the saving by exactly the marker. -_POINTER_TEXT = "[tool result elided by an ACE lever]" - - -@dataclass(frozen=True, slots=True) -class Anchor: - """Where one ``ToolCall`` lives in the request body, so an edit can be applied back. - - The typed model addresses calls by ``(turn_index, call_index)`` — stable and - agent-neutral — while the body addresses them by ``messages[m]["content"][b]``. Levers - only ever see the former, so something has to hold the mapping; keeping it beside the - session rather than inside it is what stops the wire format leaking into the contract - every lever compiles against. - """ - - msg_index: int - block_index: int - # True for the newest result in the body. Its bytes have not yet been written to the - # provider's cache, so editing it is prefix-safe and carries no invalidation penalty. - is_tail: bool = False - - -@dataclass(frozen=True, slots=True) -class LiveEdit: - """One edit, priced against the turn it would have changed.""" - - lever: str - kind: str - reason: str - turn_index: int - call_index: int - prefix_safe: bool = True - applied: bool = True - note: str = "" - - -@dataclass(frozen=True, slots=True) -class TurnMeasurement: - """One lever's counterfactual for one live turn. The row that gets persisted. - - ``removed_tokens`` is the whole point and is exact. ``usd`` is exact *for this turn* and - deliberately carries no forward projection. - """ - - lever: str - mode: str - model: str - request_id: str = "" - session_id: Optional[str] = None - ts: float = 0.0 - - baseline_tokens: int = 0 - counterfactual_tokens: int = 0 - removed_tokens: int = 0 - - # How the removed tokens were allocated against this turn's real usage buckets. - from_cache_write: int = 0 - from_input: int = 0 - from_cache_read: int = 0 - - usd: float = 0.0 - priced: bool = True - # The provider's own reported prompt size, kept purely as a cross-check on the baseline - # count. Never used as an operand — see ``counter.count_body``. - reported_prompt_tokens: int = 0 - - edits: Tuple[LiveEdit, ...] = () - diagnostics: Mapping[str, Any] = field(default_factory=dict) - note: str = "" - elapsed_ms: float = 0.0 - - @property - def measured(self) -> bool: - return self.priced and self.removed_tokens > 0 - - -# --------------------------------------------------------------------------------------- -# Wire format -> typed model -# --------------------------------------------------------------------------------------- - - -def _blocks(content: Any) -> List[Any]: - """A message's content as a block list. A bare string is one implicit text block.""" - if isinstance(content, list): - return content - return [] if content is None else [content] - - -def _result_text(content: Any) -> str: - """A tool result's textual payload, images excluded. - - Mirrors ``insights._digest``'s treatment: an image's base64 is never part of the text a - truncation lever reasons about, and two screenshots of the same page are never - byte-identical anyway. - """ - if isinstance(content, str): - return content - if not isinstance(content, list): - return "" if content is None else json.dumps(content, default=str) - parts: List[str] = [] - for blk in content: - if isinstance(blk, str): - parts.append(blk) - elif isinstance(blk, dict): - if blk.get("type") == "image": - continue - t = blk.get("text") - parts.append(t if isinstance(t, str) else json.dumps(blk, default=str)) - return "".join(parts) - - -def body_to_session( - body: Mapping[str, Any], - *, - session_id: str = "", - agent: str = "claude", - usage: Optional[Usage] = None, -) -> Tuple[Session, Dict[Tuple[int, int], Anchor]]: - """Adapt one ``/v1/messages`` request body into the model levers read. - - Sibling of ``types.from_corpus_session``, and deliberately not a replacement for it: that - one adapts a finished transcript (hashes and sizes, every turn's usage known), this one - adapts a request in flight (real bytes, only the current turn's usage knowable). Levers - cannot tell the difference, which is the property that lets one lever serve both paths. - - A "turn" here is an assistant message that made tool calls, paired with the results that - came back in the following user message. Historical turns carry an empty :class:`Usage` - — the body does not record what they were billed — so the live path prices with - :func:`price_delta` rather than the transcript ledger, which needs those numbers. - - ``sig``/``target``/``digest`` are computed by ``insights``' own helpers rather than - reimplemented. Two hashing conventions for the same quantity is precisely how a lever - ends up scoring one thing on transcripts and a different thing live. - """ - # Lazy: `insights` is a 2,700-line dashboard module and this is reached from the gateway. - # One-time cost, off the hot path, and it buys a single definition of these hashes. - from ace.sidecar.insights import _digest, _measure, _sig, _target - - messages = body.get("messages") - messages = messages if isinstance(messages, list) else [] - - # tool_use id -> where its result landed, so a call can be joined to its result across - # the message boundary. Anthropic pairs them by id, which is reliable here (unlike in - # some transcript formats, where position is the only anchor available). - results: Dict[str, Tuple[int, int, Any]] = {} - for mi, msg in enumerate(messages): - if not isinstance(msg, dict) or msg.get("role") != "user": - continue - for bi, blk in enumerate(_blocks(msg.get("content"))): - if isinstance(blk, dict) and blk.get("type") == TOOL_RESULT: - tid = blk.get("tool_use_id") - if isinstance(tid, str): - results[tid] = (mi, bi, blk.get("content")) - - # The newest result in the body is the one this turn just produced: it has not yet been - # written to the provider's cache, so an edit to it is free of invalidation cost. - tail_id = "" - tail_pos = (-1, -1) - for tid, (mi, bi, _) in results.items(): - if (mi, bi) > tail_pos: - tail_pos, tail_id = (mi, bi), tid - - turns: List[Turn] = [] - anchors: Dict[Tuple[int, int], Anchor] = {} - - for msg in messages: - if not isinstance(msg, dict) or msg.get("role") != "assistant": - continue - calls: List[ToolCall] = [] - ti = len(turns) - for blk in _blocks(msg.get("content")): - if not (isinstance(blk, dict) and blk.get("type") == TOOL_USE): - continue - name = str(blk.get("name") or "?") - tool_input = blk.get("input") - tool_input = tool_input if isinstance(tool_input, dict) else {} - tid = blk.get("tool_use_id") or blk.get("id") - - found = results.get(tid) if isinstance(tid, str) else None - ci = len(calls) - if found is None: - # A call whose result is not in this body — the in-flight one, typically. - # Recorded so positions stay stable, but with nothing to edit. - calls.append( - ToolCall(name=name, sig=_sig(name, tool_input), - target=_target(tool_input), call_id=tid) - ) - continue - - mi, bi, content = found - anchors[(ti, ci)] = Anchor(mi, bi, is_tail=(tid == tail_id)) - calls.append( - ToolCall( - name=name, - sig=_sig(name, tool_input), - target=_target(tool_input), - digest=_digest(content), - result_bytes=_measure(content), - call_id=tid, - # THE difference from the transcript path: real bytes, bound lazily so a - # lever that only reads sizes never materializes them. - content=ContentRef(lambda c=content: c), - ) - ) - # Historical turns carry an empty Usage deliberately: the request body does not - # record what they were billed, and fabricating a plausible number is exactly the - # kind of input that would make the transcript ledger's arithmetic silently wrong. - turns.append( - Turn(index=ti, model=str(body.get("model") or ""), calls=tuple(calls)) - ) - - # The one turn whose billing IS known is this one, and it belongs to the tail. - if turns and usage is not None: - last = turns[-1] - turns[-1] = Turn(index=last.index, model=last.model, ts=last.ts, - stop_reason=last.stop_reason, usage=usage, calls=last.calls) - - return Session(id=session_id, agent=agent, turns=tuple(turns)), anchors - - -# --------------------------------------------------------------------------------------- -# Building the counterfactual -# --------------------------------------------------------------------------------------- - - -def _truncate_content(content: Any, keep_bytes: int) -> Any: - """A tool result kept to its first ``keep_bytes`` of text, structure preserved. - - Truncation is applied to the *text*, block by block, so a result that is a list of parts - stays a list of parts and an image inside it is dropped rather than sliced into - corruption. A lever asking to keep 4 KB of a screenshot is asking for something - meaningless; returning the blocks it can honour is better than returning bytes that no - longer parse. - """ - if keep_bytes <= 0: - return _POINTER_TEXT - if isinstance(content, str): - return content if len(content) <= keep_bytes else content[:keep_bytes] - if not isinstance(content, list): - return content - - out: List[Any] = [] - budget = keep_bytes - for blk in content: - if budget <= 0: - break - if isinstance(blk, str): - out.append(blk[:budget]) - budget -= min(len(blk), budget) - elif isinstance(blk, dict) and blk.get("type") == "image": - continue # cannot be partially kept; keeping it whole would defeat the cap - elif isinstance(blk, dict): - t = blk.get("text") - if isinstance(t, str): - nb = dict(blk) - nb["text"] = t[:budget] - out.append(nb) - budget -= min(len(t), budget) - else: - out.append(blk) - else: - out.append(blk) - return out or _POINTER_TEXT - - -def apply_edits( - body: Mapping[str, Any], - edits: Sequence[Edit], - anchors: Mapping[Tuple[int, int], Anchor], -) -> Tuple[Dict[str, Any], List[LiveEdit], int]: - """The counterfactual body, plus what actually landed in it. - - Copy-on-write down the edited path only. A deep copy would duplicate a multi-megabyte - agent prompt once per lever per turn, for the sake of changing a handful of strings. - - Returns ``(new_body, applied, skipped)``. An edit that addresses a call with no result in - this body, or that is an ``expire``, changes no bytes and is reported rather than dropped - — a lever whose every edit was skipped must not look like a lever that found nothing. - """ - out = dict(body) - messages = list(body.get("messages") or []) - touched_msgs: Dict[int, Dict[str, Any]] = {} - applied: List[LiveEdit] = [] - skipped = 0 - - for edit in edits: - anchor = anchors.get((edit.turn_index, edit.call_index)) - if anchor is None: - skipped += 1 - applied.append(LiveEdit( - lever="", kind=edit.kind, reason=edit.reason, - turn_index=edit.turn_index, call_index=edit.call_index, - applied=False, note="no tool result at this position in the request body", - )) - continue - if edit.kind == "expire": - # Changes residency, not bytes. Volume levers and accounting levers do not share - # a number here for the same reason the ledger keeps them apart. - skipped += 1 - applied.append(LiveEdit( - lever="", kind=edit.kind, reason=edit.reason, - turn_index=edit.turn_index, call_index=edit.call_index, - prefix_safe=anchor.is_tail, applied=False, - note="expire changes residency, not prompt bytes — not priced on this path", - )) - continue - - msg = touched_msgs.get(anchor.msg_index) - if msg is None: - src = messages[anchor.msg_index] - msg = dict(src) - msg["content"] = list(_blocks(src.get("content"))) - touched_msgs[anchor.msg_index] = msg - messages[anchor.msg_index] = msg - - blocks = msg["content"] - if not (0 <= anchor.block_index < len(blocks)): - skipped += 1 - continue - blk = blocks[anchor.block_index] - if not isinstance(blk, dict): - skipped += 1 - continue - - nb = dict(blk) - if edit.kind == "truncate": - if edit.keep_bytes is None: - skipped += 1 - continue - nb["content"] = _truncate_content(blk.get("content"), edit.keep_bytes) - elif edit.kind == "drop": - nb["content"] = _POINTER_TEXT - elif edit.kind == "replace": - nb["content"] = edit.replacement if edit.replacement is not None else _POINTER_TEXT - else: - skipped += 1 - continue - - blocks[anchor.block_index] = nb - applied.append(LiveEdit( - lever="", kind=edit.kind, reason=edit.reason, - turn_index=edit.turn_index, call_index=edit.call_index, - prefix_safe=anchor.is_tail, applied=True, - )) - - out["messages"] = messages - return out, applied, skipped - - -# --------------------------------------------------------------------------------------- -# Pricing one live turn -# --------------------------------------------------------------------------------------- - - -def price_delta(removed: int, usage: Usage, rates) -> Tuple[float, int, int, int]: - """Value ``removed`` tokens against the turn's own billed buckets. Newest first. - - Returns ``(usd, from_cache_write, from_input, from_cache_read)``. - - The allocation is the whole argument, so it is worth stating plainly. A tool result that - a lever trims sits at the **end** of the prompt, and the end of an agent prompt is the - part that was not served from cache: it is either fresh input or the content being - written to the cache for the next turn. The cached prefix in front of it is older - material the edit never touches. So removed tokens are drawn from - ``cache_write -> input -> cache_read``, in that order, and each bucket is valued at the - rate the provider actually charged for it. - - This matters by an order of magnitude and in the direction that flatters the product, - which is why it is derived rather than assumed. Valuing everything at the cache-read rate - (~0.1x) understates a tail truncation roughly tenfold; valuing everything at the write - rate (1.25x) overstates a history rewrite by about the same. Both numbers are wrong. The - turn's own usage split is the only thing here that is ground truth, and it is free. - - Falls back to the cache-read rate — the conservative end — once the newer buckets are - exhausted, which is what happens when an edit really does reach into cached history. - """ - if removed <= 0 or rates is None: - return 0.0, 0, 0, 0 - - left = removed - from_write = min(left, max(0, usage.cache_write_tokens)) - left -= from_write - from_input = min(left, max(0, usage.input_tokens)) - left -= from_input - from_read = max(0, left) - - ttl = "5m" - if usage.cache_write_by_ttl: - ttl = max(usage.cache_write_by_ttl.items(), key=lambda kv: kv[1])[0] - - usd = ( - from_write / _MTOK * rates.cache_write_per_mtok(ttl) - + from_input / _MTOK * rates.input_per_mtok - + from_read / _MTOK * rates.cache_read_per_mtok - ) - return usd, from_write, from_input, from_read - - -# --------------------------------------------------------------------------------------- -# Orchestration -# --------------------------------------------------------------------------------------- - - -class ShadowRunner: - """Runs enabled levers against live turns and hands the results to a sink. - - Holds the discovered levers and the counter for the life of the process. Discovery walks - installed distribution metadata, which cannot change under a running process, and the - counter carries the one fact worth remembering across turns — whether its credential is - accepted at all. - - Cheap and total when nothing is installed. That is the ordinary state for the open-source - sidecar, and it must cost a proxied turn nothing measurable: :meth:`enabled` is one cached - entry-point lookup and a dict comparison. - """ - - def __init__( - self, - *, - config: Optional[Mapping[str, Any]] = None, - sink=None, - counter=None, - agent: str = "claude", - ) -> None: - self._config = config - self._sink = sink - self._counter = counter - self._agent = agent - self._levers: Optional[Tuple[RegisteredLever, ...]] = None - self._modes: Optional[Dict[str, str]] = None - - # -- wiring ------------------------------------------------------------------------ - - def _discovered(self) -> Tuple[RegisteredLever, ...]: - if self._levers is None: - try: - self._levers = discover() - except Exception: # pragma: no cover - discovery isolates its own failures - log.debug("[levers] discovery failed", exc_info=True) - self._levers = () - return self._levers - - def modes(self) -> Dict[str, str]: - if self._modes is None: - self._modes = resolve_modes(self._discovered(), config=self._config) - return self._modes - - def active(self) -> List[RegisteredLever]: - modes = self.modes() - return [r for r in self._discovered() if modes.get(r.id, MODE_OFF) != MODE_OFF] - - @property - def enabled(self) -> bool: - """Whether any lever would run. The early-out every proxied turn hits first.""" - return bool(self._discovered()) and bool(self.active()) - - def set_counter(self, counter) -> None: - """Adopt a counter built from the in-flight credential, once. - - Kept for the life of the process rather than rebuilt per turn: a rebuilt counter - would forget that the credential had already been refused and re-ask the counting - endpoint on every single turn. - """ - if self._counter is None and counter is not None: - self._counter = counter - - @property - def counter(self): - return self._counter - - # -- the measurement --------------------------------------------------------------- - - def observe( - self, - body: Mapping[str, Any], - usage: Usage, - *, - model: str = "", - request_id: str = "", - session_id: Optional[str] = None, - rates=None, - ) -> List[TurnMeasurement]: - """Measure every enabled lever against one completed turn. Blocking; call off-thread. - - Never raises. This runs after a response has already been served, and there is no - failure here worth converting into a developer-visible error. - """ - out: List[TurnMeasurement] = [] - levers = self.active() - if not levers or self._counter is None: - return out - - t0 = time.monotonic() - try: - session, anchors = body_to_session( - body, session_id=session_id or "", agent=self._agent, usage=usage - ) - except Exception: - log.debug("[levers] could not adapt request body", exc_info=True) - return out - - if rates is None: - try: - from ace.gateway.pricing import rates_for - - rates = rates_for(model or str(body.get("model") or "")) - except Exception: - rates = None - - # One baseline count, shared by every lever. Counting it per lever would multiply the - # network cost by the number installed for an answer that cannot differ. - try: - baseline = self._counter.count_body(body) - except Exception as exc: - log.debug("[levers] baseline count failed: %s", exc) - return out - - modes = self.modes() - for reg in levers: - m = self._measure_one( - reg, session, anchors, body, usage, baseline, modes.get(reg.id, MODE_OFF), - model=model or str(body.get("model") or ""), - request_id=request_id, session_id=session_id, rates=rates, - ) - if m is not None: - out.append(m) - - if out: - log.debug( - "[levers] measured %d lever(s) in %.0fms", - len(out), (time.monotonic() - t0) * 1000.0, - ) - return out - - def _measure_one( - self, - reg: RegisteredLever, - session: Session, - anchors: Mapping[Tuple[int, int], Anchor], - body: Mapping[str, Any], - usage: Usage, - baseline: int, - mode: str, - *, - model: str, - request_id: str, - session_id: Optional[str], - rates, - ) -> Optional[TurnMeasurement]: - t0 = time.monotonic() - ctx = LeverContext( - count_tokens=self._counter, - mode=mode, - now=time.time(), - settings=load_settings(reg.id, config=self._config), - ) - proposal = propose_safely(reg, session, ctx) - if proposal is None: - return None - - base_row = dict( - lever=reg.id, mode=mode, model=model, request_id=request_id, - session_id=session_id, ts=time.time(), - baseline_tokens=baseline, reported_prompt_tokens=usage.prompt_tokens, - diagnostics=dict(proposal.diagnostics), - ) - - if not proposal.edits: - # A real and useful result, not a failure — a loop guardrail's entire output is - # its diagnostics. Recorded so the rail can show the lever ran and found nothing. - return TurnMeasurement( - **base_row, counterfactual_tokens=baseline, - note="lever proposed no edits", - elapsed_ms=(time.monotonic() - t0) * 1000.0, - ) - - try: - new_body, applied, skipped = apply_edits(body, proposal.edits, anchors) - except Exception: - log.debug("[levers] %r: counterfactual body failed", reg.id, exc_info=True) - return None - - edits = tuple( - LiveEdit(lever=reg.id, kind=e.kind, reason=e.reason, turn_index=e.turn_index, - call_index=e.call_index, prefix_safe=e.prefix_safe, - applied=e.applied, note=e.note) - for e in applied - ) - if not any(e.applied for e in edits): - return TurnMeasurement( - **base_row, counterfactual_tokens=baseline, edits=edits, - note="lever proposed edits, none of which changed prompt bytes", - elapsed_ms=(time.monotonic() - t0) * 1000.0, - ) - - try: - counterfactual = self._counter.count_body(new_body) - except Exception as exc: - return TurnMeasurement( - **base_row, counterfactual_tokens=0, edits=edits, priced=False, - note=f"counterfactual could not be counted: {exc}", - elapsed_ms=(time.monotonic() - t0) * 1000.0, - ) - - removed = max(0, baseline - counterfactual) - usd, w, i, r = price_delta(removed, usage, rates) - note = "" if rates is not None else "model has no catalog entry — unpriced, not free" - if any(e.applied and not e.prefix_safe for e in edits): - # The invalidation cost lands on the NEXT turn's cache write, which has not - # happened yet. Saying so is the difference between a net figure and a gross one - # wearing a net figure's label. - note = (note + "; " if note else "") + ( - "touches already-cached history — the cache-write penalty falls on the next " - "turn and is not netted here" - ) - - return TurnMeasurement( - **base_row, - counterfactual_tokens=counterfactual, - removed_tokens=removed, - from_cache_write=w, from_input=i, from_cache_read=r, - usd=usd, priced=rates is not None, - edits=edits, note=note, - elapsed_ms=(time.monotonic() - t0) * 1000.0, - ) - - # -- the async entry point the proxy uses ------------------------------------------ - - async def observe_async(self, *args, **kwargs) -> List[TurnMeasurement]: - """:meth:`observe` on a worker thread, results handed to the sink. - - The counter is a synchronous HTTP client; awaiting it directly would block the event - loop that is serving every other turn. Runs detached, after the response. - """ - import asyncio - - try: - rows = await asyncio.to_thread(self.observe, *args, **kwargs) - except Exception: # pragma: no cover - a shadow run never surfaces - log.debug("[levers] shadow run failed", exc_info=True) - return [] - if rows and self._sink is not None: - try: - self._sink(rows) - except Exception: - log.debug("[levers] shadow sink failed", exc_info=True) - return rows diff --git a/src/ace/sidecar/levers/types.py b/src/ace/sidecar/levers/types.py deleted file mode 100644 index 176a40a..0000000 --- a/src/ace/sidecar/levers/types.py +++ /dev/null @@ -1,278 +0,0 @@ -"""ace.sidecar.levers.types — the normalized session model every lever reads. - -Why this layer exists ---------------------- -``insights._scan``, ``_scan_antigravity`` and ``_scan_codex`` already emit one shape — -"corpus-shaped sessions" — from three completely different on-disk formats. That shape is -the only agent-agnostic thing in the codebase, and it is what makes one lever work for -Claude Code, Antigravity and Codex without knowing which produced the session. - -So a lever is defined against **this** model and never against a provider's wire format. A -lever that took an Anthropic ``messages[]`` body would work for exactly one of the three -agents and would have to be rewritten for the fourth. Adding an agent is then a scanner, -not a lever change. - -Hashes and sizes, not content ------------------------------ -The corpus is deliberately "numbers and hashes only": ``target`` and ``digest`` are -truncated SHA-256, ``result_bytes`` is a size. That is a privacy property worth keeping — -the dashboard reads a developer's whole transcript history and nothing about it needs the -text. - -It is also sufficient for the entire measurement half. Every lever in ``strategies.py`` -(read de-dup, supersede, age-out, truncate) decides purely on ``sig``/``digest``/ -``result_bytes``, which is why they can be scored on transcripts alone. Only *actuation* -needs bytes — you cannot truncate text you do not have — so content arrives through the -optional :class:`ContentRef` seam, resolved lazily and only in the proxy/hook path where -the bytes are in hand anyway. ``ContentRef`` is ``None`` in measure-only mode, and a lever -that declares ``requires_content`` is simply not offered there. - -Provider neutrality in the usage record ---------------------------------------- -:class:`Usage` carries a total ``cache_write_tokens`` plus a ``by_ttl`` breakdown rather -than Anthropic's ``ephemeral_5m``/``ephemeral_1h`` field names. The cache-write premium is -a *provider* property — Anthropic charges 1.25x at the 5-minute TTL and 2x at one hour; -other providers price prefix reuse differently and some charge no write premium at all. -Pricing that difference is the ledger's job (see ``ace.gateway.pricing``); the lever must -never see it, or a lever tuned against one provider's cache economics will quietly give -wrong answers on another. -""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Tuple - -__all__ = [ - "ContentUnavailable", - "ContentRef", - "ToolCall", - "Usage", - "Turn", - "Session", - "from_corpus_session", - "from_corpus_sessions", -] - - -class ContentUnavailable(RuntimeError): - """Raised when a lever asks for bytes that this run does not have. - - Reaching this is a wiring bug, not a runtime condition: a lever declaring - ``requires_content = True`` must never be handed a measure-only session. The registry - filters on that flag, so this exception exists to make a filtering mistake loud rather - than to be caught. - """ - - -@dataclass(frozen=True, slots=True) -class ContentRef: - """A lazy handle to a tool result's actual bytes. - - Deliberately not the bytes themselves. A session can hold thousands of tool results and - the measurement path wants none of them; materializing every result to run a lever that - only reads sizes would turn a transcript scan into a memory problem for no gain. - """ - - _resolve: Optional[Callable[[], Any]] = None - - @property - def available(self) -> bool: - return self._resolve is not None - - def resolve(self) -> Any: - """The result body, in whatever shape the agent recorded it (str or list-of-parts).""" - if self._resolve is None: - raise ContentUnavailable( - "tool result content is not available in measure-only mode" - ) - return self._resolve() - - -@dataclass(frozen=True, slots=True) -class ToolCall: - """One tool invocation and the result it put into context. - - ``sig`` vs ``target`` is the distinction that decides whether a de-dup lever is worth - anything. ``target`` hashes the primary path argument alone, so three disjoint slices - of one file share it; ``sig`` hashes the *whole* input, so ``offset``/``limit`` - participate. Keying de-dup on ``target`` measured $36.88 of headroom on the reference - corpus where keying on ``sig`` + ``digest`` measures $0.33 — the same lever, two orders - of magnitude apart. Prefer ``sig``, and require ``digest`` equality before claiming - bytes are redundant. - """ - - name: str - sig: str - target: Optional[str] = None - digest: Optional[str] = None - result_bytes: int = 0 - call_id: Optional[str] = None - content: Optional[ContentRef] = None - - @property - def has_content(self) -> bool: - return self.content is not None and self.content.available - - -@dataclass(frozen=True, slots=True) -class Usage: - """One turn's billed token counts, as the provider reported them. - - These are ground truth — read off the transcript, not derived — which is what lets a - counterfactual be stated against a real bill instead of against an estimate. Anything a - lever *proposes* has to be counted separately and exactly (see - ``protocol.TokenCounter``); never infer the counterfactual by scaling these. - """ - - input_tokens: int = 0 - output_tokens: int = 0 - cache_read_tokens: int = 0 - cache_write_tokens: int = 0 - # TTL label -> tokens written at that TTL, e.g. {"5m": 12000, "1h": 0}. Empty when the - # provider reports no breakdown; the total above still stands on its own. - cache_write_by_ttl: Mapping[str, int] = field(default_factory=dict) - - @property - def prompt_tokens(self) -> int: - """Everything that was in the prompt this turn, cached or not. - - ``input_tokens`` already EXCLUDES the cached buckets on Anthropic, so this is a sum - and not a max. Subtracting ``cache_read`` from ``input`` to "correct" it under-reports - prompt volume — an easy bug with no visible symptom. - """ - return self.input_tokens + self.cache_read_tokens + self.cache_write_tokens - - -@dataclass(frozen=True, slots=True) -class Turn: - """One API request. Not one transcript record. - - Claude Code writes one record per content block and repeats the whole ``usage`` object - on each; the scanners join on message id before emitting. Counting per record instead - inflates prompt volume 1.95x and output 2.34x. A lever receives turns already joined and - must not try to re-derive them. - """ - - index: int - model: str = "" - ts: Optional[float] = None # epoch seconds - stop_reason: Optional[str] = None - usage: Usage = field(default_factory=Usage) - calls: Tuple[ToolCall, ...] = () - - -@dataclass(frozen=True, slots=True) -class Session: - """One agent session, normalized. The unit a lever reasons over. - - ``agent`` is one of ``insights.AGENTS`` ("claude", "antigravity", "codex"). A lever may - read it — some tool names are agent-specific — but must not require a particular value: - the whole point of this model is that a lever written today keeps working when a fourth - scanner lands. - """ - - id: str - agent: str - kind: str = "main" # "main" | "subagent" - parent: Optional[str] = None - turns: Tuple[Turn, ...] = () - - @property - def n_turns(self) -> int: - return len(self.turns) - - def iter_calls(self): - """``(turn_index, call_index, ToolCall)`` over the whole session, in order.""" - for t in self.turns: - for ci, call in enumerate(t.calls): - yield t.index, ci, call - - -def _usage_from_corpus(t: Mapping[str, Any]) -> Usage: - by_ttl: Dict[str, int] = {} - for label, key in (("5m", "ephemeral_5m_input_tokens"), ("1h", "ephemeral_1h_input_tokens")): - v = int(t.get(key) or 0) - if v: - by_ttl[label] = v - return Usage( - input_tokens=int(t.get("input_tokens") or 0), - output_tokens=int(t.get("output_tokens") or 0), - cache_read_tokens=int(t.get("cache_read_input_tokens") or 0), - cache_write_tokens=int(t.get("cache_creation_input_tokens") or 0), - cache_write_by_ttl=by_ttl, - ) - - -def from_corpus_session( - raw: Mapping[str, Any], - *, - content_for: Optional[Callable[[int, int], Optional[Callable[[], Any]]]] = None, -) -> Session: - """Adapt one ``insights`` session dict into the typed model. - - This function is the entire cost of supporting a new agent: write a scanner that emits - the corpus shape and every lever works on it unchanged. - - ``content_for(turn_index, call_index)`` is the actuation hook. It returns a zero-arg - callable producing that result's body, or ``None`` where the bytes are not held. Omit it - entirely for the measurement path — which is every transcript-driven caller — and every - ``ToolCall.content`` is ``None``. - """ - turns = [] - for i, t in enumerate(raw.get("turns") or []): - calls = [] - for ci, c in enumerate(t.get("calls") or []): - resolver = content_for(i, ci) if content_for is not None else None - calls.append( - ToolCall( - name=str(c.get("name") or "?"), - sig=str(c.get("sig") or ""), - target=c.get("target"), - digest=c.get("digest"), - result_bytes=int(c.get("result_bytes") or 0), - call_id=c.get("id"), - content=ContentRef(resolver) if resolver is not None else None, - ) - ) - turns.append( - Turn( - index=i, - model=str(t.get("model") or ""), - ts=_epoch(t.get("ts")), - stop_reason=t.get("stop_reason"), - usage=_usage_from_corpus(t), - calls=tuple(calls), - ) - ) - return Session( - id=str(raw.get("session") or ""), - agent=str(raw.get("agent_type") or ""), - kind=str(raw.get("kind") or "main"), - parent=raw.get("parent"), - turns=tuple(turns), - ) - - -def from_corpus_sessions(rows: Sequence[Mapping[str, Any]]) -> Tuple[Session, ...]: - """Measure-only adaptation of a whole scan. The common case.""" - return tuple(from_corpus_session(r) for r in rows) - - -def _epoch(v: Any) -> Optional[float]: - """Corpus timestamps are ISO strings; the model wants seconds. - - Kept local rather than imported from ``insights``: the dependency runs from insights - into this package, and reversing it for a six-line helper would make the two mutually - importable. - """ - if isinstance(v, (int, float)): - return float(v) - if not isinstance(v, str) or not v: - return None - import datetime - - try: - return datetime.datetime.fromisoformat(v.replace("Z", "+00:00")).timestamp() - except ValueError: - return None diff --git a/src/ace/sidecar/strategies.py b/src/ace/sidecar/strategies.py index 0d9f259..41aeab5 100644 --- a/src/ace/sidecar/strategies.py +++ b/src/ace/sidecar/strategies.py @@ -30,36 +30,7 @@ from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple -# Byte-equivalents per token, for converting the byte-turns `simulate` counts into the -# tokens and dollars `score` reports. -# -# Measured, not assumed. Against 4,512 text-only, single-tool-call results drawn from the -# local Claude Code corpus, the observed characters-per-token distribution is: -# -# p10 1.51 p25 1.84 median 2.16 p75 2.41 p90 2.64 p95 2.82 p99 4.34 -# -# The previous value of 4.0 sat at the **99th percentile** of that distribution — not a -# central estimate but its extreme tail. 4.0 is the familiar figure for English prose; agent -# tool output is code, JSON, logs, diffs and file paths, which tokenize far denser. The same -# mistake, in the same direction, is recorded in ``ace.gateway.tokenizer``: a long-context -# gate computed at ~1.33 tokens/word on content that is really up to 3.3. -# -# The measurement is derived per turn as ``(prompt[i+1] - prompt[i]) - output[i]``, which is -# ground truth on both sides but is contaminated upward on the token leg by anything else -# that entered the prompt between the two turns (injected reminders, re-read context). That -# contamination can only *reduce* the observed ratio, so the honest estimate lives in the -# upper tail rather than at the median. 2.8 is that tail (~p95). -# -# Direction of the correction matters: `score` divides by this constant, so a LOWER value -# reports MORE tokens and more dollars. Moving 4.0 -> 2.8 raises every byte-turn headroom -# figure on the rail by ~1.43x. That is the direction that warrants caution, which is why -# the value chosen is the conservative end of the measured band and not its median. -# -# COUPLED: ``insights._CHARS_PER_TOKEN`` must hold the same value. `_measure` converts an -# image's known token count *into* byte-equivalents by multiplying by it, and the division -# here converts back — so images round-trip exactly when the two agree and are mispriced by -# their ratio when they do not. See :func:`_check_image_bridge`. -BYTES_PER_TOKEN = 2.8 +BYTES_PER_TOKEN = 4.0 POINTER_BYTES = 120 WRITE_TOOLS = ("Edit", "Write", "NotebookEdit") TTL_SECONDS = 3600.0 @@ -385,42 +356,6 @@ def accounting(sessions: List[Dict[str, Any]], rates_for) -> Dict[str, float]: return dict(out) -_BRIDGE_CHECKED = False - - -def _check_image_bridge() -> None: - """Warn once if ``insights._CHARS_PER_TOKEN`` has drifted from :data:`BYTES_PER_TOKEN`. - - The two constants are a matched pair, not two opinions about the same quantity. - ``insights._measure`` prices an image at its real token count and multiplies by - ``_CHARS_PER_TOKEN`` purely to keep one unit flowing through the pipeline; the division in - :func:`score` undoes it. Any value works so long as both sides use the SAME one — and when - they diverge, every image-bearing result is mispriced by exactly their ratio, silently, - with no error and no visible symptom beyond a lever's number moving. - - Checked lazily rather than at import: ``insights`` imports this module, so a module-level - import here would close the cycle. - """ - global _BRIDGE_CHECKED - if _BRIDGE_CHECKED: - return - _BRIDGE_CHECKED = True - try: - from ace.sidecar.insights import _CHARS_PER_TOKEN - except Exception: - return - if abs(float(_CHARS_PER_TOKEN) - BYTES_PER_TOKEN) > 1e-9: - import logging - - logging.getLogger(__name__).warning( - "[strategies] insights._CHARS_PER_TOKEN=%s != BYTES_PER_TOKEN=%s — image-bearing " - "tool results are mispriced by %.2fx. Set them to the same value.", - _CHARS_PER_TOKEN, - BYTES_PER_TOKEN, - float(_CHARS_PER_TOKEN) / BYTES_PER_TOKEN, - ) - - def score( sessions: List[Dict[str, Any]], s: Strategy, @@ -439,7 +374,6 @@ def score( rate = r.cache_read_per_mtok break for lever, byte_turns in simulate(sess, s).items(): - _check_image_bridge() tok = byte_turns / BYTES_PER_TOKEN tokens[lever] += tok usd[lever] += tok / 1e6 * rate diff --git a/tests/test_lever_ledger.py b/tests/test_lever_ledger.py deleted file mode 100644 index 9bc04b4..0000000 --- a/tests/test_lever_ledger.py +++ /dev/null @@ -1,160 +0,0 @@ -"""The ledger's arithmetic — the place a lever's proposal becomes money. - -Every rate here is a stand-in, so these tests assert the *arithmetic* and not the shipped -price list. The counter is ``len(text) // 4`` for the same reason: exact expected token counts -are hand-checkable, which is what makes a wrong answer legible rather than merely red. - -The properties under test are the three ways this arithmetic has already gone wrong in this -codebase: ignoring the cache-write penalty, summing levers that overlap, and rendering an -unpriced model as free. -""" - -from __future__ import annotations - -import pytest - -from ace.gateway.pricing import Rates -from ace.sidecar import levers as L - -RATES = Rates(model="m", input_per_mtok=3.0, output_per_mtok=15.0, - cache_read_per_mtok=0.30, source="test", as_of="2026-08-27") -LOOKUP = lambda m: RATES if m == "m" else None # noqa: E731 - -CTX = L.LeverContext(count_tokens=lambda t, *, model: len(t) // 4, mode=L.MODE_SHADOW) - -BIG = 40_000 -DUMP = "x" * BIG # 10,000 tokens at 4 bytes/token - -TRUNCATE = L.Proposal("truncate_dumps", ( - L.Edit(turn_index=0, call_index=0, kind="truncate", reason="dump", keep_bytes=4_000), -)) -EXPIRE_AT_5 = L.Proposal("compaction", ( - L.Edit(turn_index=0, call_index=0, kind="expire", reason="stale", live_until=5), -)) - - -def session(n_turns=10, model="m", prompt=50_000, cache_read=45_000, with_content=True): - """`n` turns with one big Bash dump created at turn 0, prompt growing each turn.""" - turns = [] - for i in range(n_turns): - calls = ( - [{"name": "Bash", "sig": "s0", "digest": "d0", "result_bytes": BIG}] - if i == 0 else [] - ) - turns.append({ - "model": model, "ts": None, "input_tokens": 100, "output_tokens": 50, - "cache_read_input_tokens": 0 if i == 0 else cache_read + i * 2_000, - "cache_creation_input_tokens": prompt if i == 0 else 2_000, - "calls": calls, - }) - raw = {"session": "s1", "agent_type": "claude", "kind": "main", "turns": turns} - content = (lambda ti, ci: (lambda: DUMP)) if with_content else None - return L.from_corpus_session(raw, content_for=content) - - -# -- the prefix-safe case: a fresh dump, trimmed before it is ever cached ------------------ - - -def test_prefix_safe_truncation_is_measured_and_free_of_penalty(): - e = L.price_proposal(session(10), TRUNCATE, CTX, rates_lookup=LOOKUP) - assert e.fidelity == L.FIDELITY_MEASURED - c = e.edits[0] - assert c.removed_tokens == 10_000 - 1_000 # exact, counted, not inferred - assert c.apply_at == 1 and c.turns_carried == 9 - assert c.prefix_safe and c.invalidated_tokens == 0 - assert c.cache_write_penalty_usd == pytest.approx(0.0) - # Priced at the CACHE-READ rate: these tokens were resident in a cached prefix and - # re-read each turn at ~0.1x. Valuing them at the input rate would inflate the lever 10x. - assert c.gross_saving_usd == pytest.approx((9_000 / 1e6) * 0.30 * 9) - assert c.net_usd == pytest.approx(c.gross_saving_usd) - assert c.break_even_turn is None # in profit immediately - assert e.rate_sources == (("m", "test", "2026-08-27"),) - - -# -- the history-rewriting case: the penalty that makes a "saving" cost money -------------- - - -def test_editing_cached_history_carries_a_penalty_and_can_be_a_net_loss(): - c = L.price_proposal(session(10), EXPIRE_AT_5, CTX, rates_lookup=LOOKUP).edits[0] - assert c.apply_at == 6 and c.turns_carried == 4 - assert not c.prefix_safe, "editing cached history is never prefix-safe" - assert c.invalidated_tokens > 0 - assert c.cache_write_penalty_usd > 0.0 - assert c.gross_saving_usd == pytest.approx((10_000 / 1e6) * 0.30 * 4) - if c.net_usd < 0: - assert c.break_even_turn is not None and c.break_even_turn > 10, ( - "a losing edit must break even beyond the session it ran in" - ) - - -def test_the_ttl_is_read_from_the_turn_not_assumed(): - """`strategies.TTL_SECONDS` hard-codes one hour while Claude Code defaults to 5m, and the - two carry different write premiums (2x vs 1.25x).""" - s = L.from_corpus_session({ - "session": "s2", "agent_type": "claude", "turns": [ - dict(model="m", input_tokens=100, cache_read_input_tokens=0, - cache_creation_input_tokens=50_000, ephemeral_1h_input_tokens=50_000, - output_tokens=50, - calls=[{"name": "Bash", "sig": "s", "result_bytes": BIG}]), - *[dict(model="m", input_tokens=100, cache_read_input_tokens=45_000, - cache_creation_input_tokens=0, output_tokens=50, calls=[]) - for _ in range(9)], - ]}, content_for=lambda ti, ci: (lambda: DUMP)) - c = L.price_proposal(s, EXPIRE_AT_5, CTX, rates_lookup=LOOKUP).edits[0] - assert c.cache_write_ttl in ("5m", "1h") - - -# -- the three refusals ------------------------------------------------------------------- - - -def test_no_content_means_no_dollars_and_it_says_why(): - """The refusal that keeps a measured claim measured: no `result_bytes / 4` fallback.""" - e = L.price_proposal( - session(10, with_content=False), TRUNCATE, CTX, rates_lookup=LOOKUP - ) - assert e.fidelity == L.FIDELITY_UNMEASURABLE - assert e.edits == () - assert "cannot be counted exactly" in e.note - assert e.net_usd == pytest.approx(0.0) - - -def test_an_unpriced_model_reports_real_tokens_and_absent_dollars(): - """A silent $0.00 looks like a cost win. Tokens are real; dollars are absent.""" - e = L.price_proposal( - session(10, model="unknown-model"), TRUNCATE, CTX, rates_lookup=LOOKUP - ) - assert e.fidelity == L.FIDELITY_UNPRICED - assert not e.priced - assert e.removed_tokens == 9_000 - assert e.net_usd == 0.0 - assert "not free" in e.note - - -def test_a_lever_firing_on_the_last_turn_saves_nothing_and_says_so(): - e = L.price_proposal(session(1), TRUNCATE, CTX, rates_lookup=LOOKUP) - assert e.edits[0].turns_carried == 0 - assert e.edits[0].gross_saving_usd == pytest.approx(0.0) - - -def test_an_edit_free_proposal_is_measured_not_an_error(): - e = L.price_proposal( - session(10), L.Proposal("loop_guard", (), {"loops_detected": 3}), CTX, - rates_lookup=LOOKUP, - ) - assert e.fidelity == L.FIDELITY_MEASURED - assert e.diagnostics == {"loops_detected": 3} - assert not e.priced and e.note == "lever proposed no edits" - - -# -- the report ranks, and refuses to total ----------------------------------------------- - - -def test_the_report_ranks_and_has_no_total(): - """Two levers can target the same bytes; scored alone their figures overlap, so adding - them produces a number larger than anything they could jointly deliver.""" - s = session(10) - rep = L.price_all([(s, TRUNCATE), (s, EXPIRE_AT_5)], CTX, rates_lookup=LOOKUP) - assert not hasattr(rep, "total_usd") - ranked = rep.ranked() - assert ranked[0][1] >= ranked[1][1] - assert rep.unmeasured() == () diff --git a/tests/test_lever_shadow.py b/tests/test_lever_shadow.py deleted file mode 100644 index 81ff522..0000000 --- a/tests/test_lever_shadow.py +++ /dev/null @@ -1,470 +0,0 @@ -"""The live path: counting with the in-flight credential, and measuring a proxied turn. - -This is the half that makes "measured" true rather than simulated, so the tests are mostly -about the two things that could quietly make it false: - -* the relayed request must go upstream byte-for-byte, whatever a lever proposes; -* a token delta must be *counted*, and a credential the counting endpoint refuses must - produce an explanation rather than a blank. - -No live provider call is made anywhere here. The upstream relay and the counting endpoint are -both driven through ``httpx.MockTransport``, which is the same discipline the rest of this -route's suite uses. -""" - -from __future__ import annotations - -import asyncio -import json -import sqlite3 - -import httpx -import pytest -from fastapi import FastAPI - -from ace.gateway.local_store import LocalStore -from ace.gateway.messages import MessagesConfig, install_messages_route -from ace.gateway.messages_auth import MODE_LOOPBACK, AuthConfig -from ace.sidecar import levers as L -from ace.sidecar.levers import rail -from ace.sidecar.levers.counter import AnthropicCounter, resolve_counter -from ace.sidecar.levers.shadow import ( - ShadowRunner, - apply_edits, - body_to_session, - price_delta, -) -from ace.sidecar.levers.types import Usage - -DUMP = "ERROR line\n" * 3000 - - -def body(dump=DUMP): - """A Claude Code shaped turn: one tool call, one big result, a system prompt and tools.""" - return { - "model": "claude-sonnet-5", "max_tokens": 1024, "system": "You are a coding agent.", - "tools": [{"name": "Bash", "description": "run", "input_schema": {"type": "object"}}], - "messages": [ - {"role": "user", "content": "find the bug"}, - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "tu_1", "name": "Bash", - "input": {"command": "cat big.log"}}]}, - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "tu_1", "content": dump}]}, - ], - } - - -class Truncate: - id, label = "truncate_dumps", "Truncate large tool dumps" - risk, requires_content = L.RISK_LOW, False - - def propose(self, session, ctx): - cap = int(ctx.settings.get("keep_bytes", 2048)) - return L.Proposal(self.id, tuple( - L.Edit(turn_index=t, call_index=c, kind="truncate", reason="over cap", - keep_bytes=cap) - for t, c, x in session.iter_calls() if x.result_bytes > cap - ), {"scanned": session.n_turns}) - - -class LoopGuard: - id, label = "loop_guard", "Loop guardrail" - risk, requires_content = L.RISK_NONE, False - - def propose(self, session, ctx): - return L.Proposal(self.id, (), {"loops_detected": 2, "detail": "dropped: not a number"}) - - -def counting_transport(calls=None): - """A stand-in counting endpoint: tokens == serialized bytes / 4.""" - def handler(req): - if calls is not None: - calls.append(json.loads(req.content)) - return httpx.Response(200, json={"input_tokens": len(req.content) // 4}) - return httpx.MockTransport(handler) - - -def upstream_transport(seen=None): - def handler(req): - if seen is not None: - seen["body"] = req.content - seen["headers"] = dict(req.headers) - return httpx.Response(200, json={ - "id": "msg_1", "type": "message", "role": "assistant", - "model": "claude-sonnet-5", "content": [{"type": "text", "text": "ok"}], - "usage": {"input_tokens": 1200, "output_tokens": 90, - "cache_read_input_tokens": 30000, - "cache_creation_input_tokens": 12000}}) - return httpx.MockTransport(handler) - - -# -- the counter -------------------------------------------------------------------------- - - -def test_no_credential_is_an_ordinary_outcome_with_a_reason(): - """`no_key: true` is this sidecar's own default, so this is the common state.""" - counter, note = resolve_counter(None) - if counter is None: # no key exported in this environment - assert "no credential available" in note - else: # a developer who did export one - assert "count_tokens" in note - - -def test_an_oauth_token_is_presented_as_bearer_with_the_required_beta(): - """An OAuth token sent as `x-api-key` is rejected, and /v1/messages needs the beta.""" - seen = [] - def handler(req): - seen.append(dict(req.headers)) - return httpx.Response(200, json={"input_tokens": 7}) - c = AnthropicCounter("sk-ant-oat-tok", "bearer", - client=httpx.Client(transport=httpx.MockTransport(handler))) - assert c("hello", model="claude-sonnet-5") == 7 - assert "authorization" in seen[0] - assert "x-api-key" not in seen[0] - assert seen[0]["anthropic-beta"] == "oauth-2025-04-20" - - -def test_count_body_strips_fields_the_endpoint_rejects(): - calls = [] - c = AnthropicCounter("k", "api_key", client=httpx.Client(transport=counting_transport(calls))) - c.count_body({"model": "m", "messages": [], "system": "S", "tools": [], - "stream": True, "max_tokens": 5, "temperature": 0.7, "metadata": {}}) - assert sorted(calls[0]) == ["messages", "model", "system", "tools"] - - -def test_a_refused_credential_latches_and_explains(): - """The OAuth-vs-API-key question is settled here, as a side effect of the first count — - there is no preflight probe. A refusal must be remembered, not re-asked every turn.""" - n = {"i": 0} - def deny(req): - n["i"] += 1 - return httpx.Response(401, json={"error": "unauthorized"}) - c = AnthropicCounter("sk-ant-oat-tok", "bearer", - client=httpx.Client(transport=httpx.MockTransport(deny))) - for _ in range(3): - with pytest.raises(RuntimeError): - c("x", model="m") - assert n["i"] == 1, "a definitive refusal must stop the calling" - assert not c.usable - assert "OAuth token" in c.note and "401" in c.note - - -def test_a_throttle_does_not_disable_counting(): - """A 429 says nothing about the credential. Latching on it would look exactly like the - feature not working.""" - n = {"i": 0} - def throttle(req): - n["i"] += 1 - return httpx.Response(429, json={}) - c = AnthropicCounter("k", "api_key", - client=httpx.Client(transport=httpx.MockTransport(throttle))) - for _ in range(3): - with pytest.raises(Exception): - c("x", model="m") - assert n["i"] == 3 - assert c.usable - - -# -- wire format -> typed model ----------------------------------------------------------- - - -def test_body_adapts_to_the_same_model_a_transcript_does(): - s, anchors = body_to_session(body(), session_id="s1") - assert s.agent == "claude" and s.n_turns == 1 - (ti, ci, call), = list(s.iter_calls()) - assert (ti, ci) == (0, 0) and call.name == "Bash" - assert call.result_bytes == len(DUMP) - # THE difference from the transcript path: real bytes are in hand. - assert call.has_content and call.content.resolve() == DUMP - assert anchors[(0, 0)].is_tail - - -def test_the_newest_result_is_the_prefix_safe_one(): - b = body() - b["messages"] += [ - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "tu_2", "name": "Read", "input": {"file_path": "/a"}}]}, - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "tu_2", "content": "short"}]}, - ] - _, anchors = body_to_session(b) - assert not anchors[(0, 0)].is_tail, "already cached history" - assert anchors[(1, 0)].is_tail, "produced this turn, not yet written to cache" - - -def test_historical_turns_carry_no_fabricated_usage(): - """The body does not record what earlier turns were billed, and inventing a plausible - number is what would make the ledger's arithmetic silently wrong.""" - u = Usage(input_tokens=1200, cache_read_tokens=30000, cache_write_tokens=12000) - b = body() - b["messages"] += [ - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "tu_2", "name": "Read", "input": {"file_path": "/a"}}]}, - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "tu_2", "content": "short"}]}, - ] - s, _ = body_to_session(b, usage=u) - assert s.turns[0].usage.prompt_tokens == 0 - assert s.turns[-1].usage.prompt_tokens == u.prompt_tokens - - -# -- the counterfactual -------------------------------------------------------------------- - - -def test_apply_edits_never_mutates_the_request(): - """Shadow means shadow. The relayed body and the counterfactual share no mutated object.""" - b = body() - _, anchors = body_to_session(b) - new, applied, skipped = apply_edits( - b, [L.Edit(turn_index=0, call_index=0, kind="truncate", reason="r", keep_bytes=2048)], - anchors, - ) - assert len(b["messages"][2]["content"][0]["content"]) == len(DUMP) - assert len(new["messages"][2]["content"][0]["content"]) == 2048 - assert [e.applied for e in applied] == [True] and skipped == 0 - - -def test_a_list_shaped_result_stays_a_list(): - b = body(dump=None) - b["messages"][2]["content"][0]["content"] = [{"type": "text", "text": "y" * 9000}] - _, anchors = body_to_session(b) - new, _, _ = apply_edits( - b, [L.Edit(turn_index=0, call_index=0, kind="truncate", reason="r", keep_bytes=1000)], - anchors, - ) - out = new["messages"][2]["content"][0]["content"] - assert isinstance(out, list) and len(out[0]["text"]) == 1000 - - -def test_expire_changes_no_bytes_and_says_so(): - """Volume levers and accounting levers must not share a number.""" - b = body() - _, anchors = body_to_session(b) - new, applied, skipped = apply_edits( - b, [L.Edit(turn_index=0, call_index=0, kind="expire", reason="stale", live_until=3)], - anchors, - ) - assert skipped == 1 - assert not applied[0].applied - assert "not priced on this path" in applied[0].note - assert new["messages"][2]["content"][0]["content"] == DUMP - - -# -- pricing one live turn ------------------------------------------------------------------ - - -def test_removed_tokens_are_drawn_newest_bucket_first(): - """The allocation IS the pricing argument: the end of an agent prompt is the part that - was not served from cache, and the same delta is worth ~12x more coming out of a write.""" - from ace.gateway.pricing import Rates - r = Rates(model="m", input_per_mtok=3.0, output_per_mtok=15.0, - cache_read_per_mtok=0.30, source="t", as_of="x") - u = Usage(input_tokens=1200, cache_read_tokens=30000, cache_write_tokens=12000, - cache_write_by_ttl={"5m": 12000}) - - usd, w, i, rd = price_delta(5_000, u, r) - assert (w, i, rd) == (5_000, 0, 0) - assert usd == pytest.approx(5_000 / 1e6 * r.cache_write_per_mtok("5m")) - - _, w, i, rd = price_delta(13_000, u, r) - assert (w, i, rd) == (12_000, 1_000, 0) - - _, w, i, rd = price_delta(20_000, u, r) - assert (w, i, rd) == (12_000, 1_200, 6_800), "the overflow falls back to the cache rate" - - -def test_an_unpriced_model_yields_no_dollars(): - assert price_delta(5_000, Usage(cache_write_tokens=9_000), None) == (0.0, 0, 0, 0) - - -# -- the whole path, through the real route -------------------------------------------------- - - -def make_app(store, *, seen=None, config=None, levers=(Truncate, LoopGuard)): - runner = ShadowRunner( - config=config or {"levers": {"truncate_dumps": "shadow", "loop_guard": "shadow"}}, - sink=store.record_lever_turns, - ) - runner._levers = tuple(L.RegisteredLever(lever=k(), dist="ace-skills") for k in levers) - runner.set_counter( - AnthropicCounter("k", "api_key", client=httpx.Client(transport=counting_transport())) - ) - app = FastAPI() - install_messages_route( - app, - config=MessagesConfig(base_url="https://upstream.test", timeout_s=5), - client=httpx.AsyncClient(transport=upstream_transport(seen)), - auth_config=AuthConfig(mode=MODE_LOOPBACK, local_api_key="k"), - accountant=store, shadow=runner, - ) - return app, runner - - -async def drive(app, n=1, payload=None): - sent = json.dumps(payload or body()).encode() - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), base_url="http://t" - ) as c: - for _ in range(n): - r = await c.post("/v1/messages", content=sent, - headers={"x-api-key": "k", "content-type": "application/json"}) - assert r.status_code == 200 - return sent - - -@pytest.fixture -def store(tmp_path): - return LocalStore(str(tmp_path / "telemetry.db")) - - -async def settle(store, rows): - for _ in range(100): - await asyncio.sleep(0.01) - if store.lever_summary()["rows"] >= rows: - return - raise AssertionError(f"shadow rows never reached {rows}") - - -def test_the_relayed_bytes_are_untouched_whatever_a_lever_proposes(store): - """THE fidelity invariant. A lever that would strip 90% of the prompt must still leave - the request that actually goes upstream byte-identical.""" - seen = {} - app, _ = make_app(store, seen=seen) - - async def main(): - sent = await drive(app) - await settle(store, 2) - return sent - - sent = asyncio.run(main()) - assert seen["body"] == sent - assert store.lever_summary()["by_lever"], "the lever did run" - - -def test_a_proxied_turn_is_measured_persisted_and_ranked(store): - app, _ = make_app(store) - - async def main(): - await drive(app, n=3) - await settle(store, 6) - - asyncio.run(main()) - s = store.lever_summary() - assert s["rows"] == 6 and s["turns_observed"] == 3 - by = {r["lever"]: r for r in s["by_lever"]} - - trunc = by["truncate_dumps"] - assert trunc["turns"] == 3 and trunc["removed_tokens"] > 0 and trunc["usd"] > 0 - assert trunc["edits_applied"] == 3 - # The dump is new this turn, so it comes out of the cache-write bucket. - assert trunc["from_cache_write"] == trunc["removed_tokens"] - - # An edit-free lever is recorded, not dropped: "it ran and found nothing" is a result. - assert by["loop_guard"]["turns"] == 3 - assert by["loop_guard"]["removed_tokens"] == 0 - - assert "total_usd" not in s, "levers overlap; a total would exceed what they can deliver" - - -def test_the_store_keeps_numbers_only(store): - """The one invariant of this store. Diagnostics are third-party authored and are the - only field that could carry a developer's own text.""" - app, _ = make_app(store) - - async def main(): - await drive(app) - await settle(store, 2) - - asyncio.run(main()) - got = { - d for (d,) in sqlite3.connect(store.path).execute( - "SELECT DISTINCT diagnostics FROM lever_turns" - ) - } - assert json.dumps({"loops_detected": 2}) in got - assert not any("dropped: not a number" in (d or "") for d in got) - - -def test_real_spend_is_recorded_alongside_the_counterfactual(store): - """`turns` and `lever_turns` are siblings: one is what was billed, the other is a prompt - that was never sent. Both have to be there for the saving to mean anything.""" - app, _ = make_app(store) - - async def main(): - await drive(app, n=2) - await settle(store, 4) - - asyncio.run(main()) - assert store.summary()["turns"] == 2 - assert store.summary()["cost_usd"] > 0 - - -def test_the_rail_reports_measured_only_once_something_was_measured(store): - app, _ = make_app(store) - before = rail.rail_payload([], store=store) - assert before["status"] in (rail.STATUS_NO_PACKAGE, rail.STATUS_ALL_OFF) - assert before["measured"] == {} - - async def main(): - await drive(app) - await settle(store, 2) - - asyncio.run(main()) - after = rail.rail_payload([], store=store) - assert after["status"] == rail.STATUS_MEASURED - assert after["turns_observed"] == 1 - assert {r["lever"] for r in after["measured"]["by_lever"]} == { - "truncate_dumps", "loop_guard" - } - - -@pytest.mark.parametrize( - "discovered,expected_qualifier", - [ - ((), "no lever package is installed now"), - ("installed-but-off", "every installed lever is now off"), - ], -) -def test_recorded_results_survive_the_lever_being_removed( - store, monkeypatch, discovered, expected_qualifier -): - """Recorded results and installed packages are independent facts, and they disagree in an - ordinary way: a developer measures a lever for a week, then uninstalls or disables it. - - Dropping the rows would hide a real measurement behind a packaging detail; reporting them - unqualified would imply the lever is still running. Both facts have to be said. - - Discovery is pinned rather than left to the environment — whether a lever package happens - to be installed in the venv running the suite is not what this test is about. - """ - app, _ = make_app(store) - - async def main(): - await drive(app) - await settle(store, 2) - - asyncio.run(main()) - - if discovered == "installed-but-off": - discovered = (L.RegisteredLever(lever=Truncate(), dist="ace-skills"),) - monkeypatch.setattr(rail, "_DISCOVERED", discovered) - - payload = rail.rail_payload([], store=store, config={}) - assert payload["status"] == rail.STATUS_MEASURED - assert expected_qualifier in payload["note"] - assert payload["measured"]["by_lever"], "the rows are still reported" - - -def test_nothing_enabled_costs_the_turn_nothing(store): - """The ordinary state for the open-source sidecar: one cached entry-point lookup.""" - app, runner = make_app(store, config={"levers": {}}) - assert not runner.enabled - - async def main(): - await drive(app) - await asyncio.sleep(0.05) - - asyncio.run(main()) - assert store.lever_summary()["rows"] == 0 - assert store.summary()["turns"] == 1, "accounting still happened" diff --git a/tests/test_levers.py b/tests/test_levers.py deleted file mode 100644 index cafc69d..0000000 --- a/tests/test_levers.py +++ /dev/null @@ -1,205 +0,0 @@ -"""The lever contract: the normalized model, mode resolution, and failure isolation. - -Nothing here installs a real lever. The point of ``ace.sidecar.levers`` is that it defines -what a lever *is* and contains none, so these tests stand in local implementations and assert -the properties the package promises third-party code — above all that presence never implies -consent, and that one bad lever costs its own row rather than the dashboard. -""" - -from __future__ import annotations - -import logging - -import pytest - -from ace.sidecar import levers as L - -# One session, three agents' worth of shape. `agent_type` is "codex" deliberately: the model -# is agent-neutral and a test that only ever exercises Claude Code would not prove it. -RAW = { - "session": "s1", "agent_type": "codex", "kind": "main", - "turns": [ - {"model": "gpt-5", "ts": "2026-08-27T10:00:00Z", "input_tokens": 100, - "output_tokens": 20, "cache_read_input_tokens": 900, - "cache_creation_input_tokens": 400, "ephemeral_5m_input_tokens": 400, - "calls": [{"id": "t1", "name": "Bash", "sig": "abc", "target": "deadbeef", - "digest": "d1", "result_bytes": 240000}]}, - {"model": "gpt-5", "ts": "2026-08-27T10:01:00Z", "input_tokens": 50, - "output_tokens": 10, "cache_read_input_tokens": 1400, - "cache_creation_input_tokens": 0, - "calls": [{"name": "Read", "sig": "xyz", "digest": "d1", "result_bytes": 1200}]}, - ], -} - - -class Truncator: - """Reads sizes only, so it scores from transcripts alone — the common case.""" - - id, label = "truncate_dumps", "Truncate large tool dumps" - risk, requires_content = L.RISK_LOW, False - - def propose(self, session, ctx): - cap = int(ctx.settings.get("keep_bytes", 4096)) - return L.Proposal( - lever=self.id, - edits=tuple( - L.Edit(turn_index=ti, call_index=ci, kind="truncate", - reason="dump over cap", sig=c.sig, keep_bytes=cap) - for ti, ci, c in session.iter_calls() if c.result_bytes > cap - ), - diagnostics={"scanned": session.n_turns}, - ) - - -class NeedsBytes(Truncator): - id, requires_content = "needs_bytes", True - - def propose(self, session, ctx): - body = session.turns[0].calls[0].content.resolve() - return L.Proposal(lever=self.id, diagnostics={"bytes": len(body)}) - - -class Boom(Truncator): - id = "boom" - - def propose(self, session, ctx): - raise ValueError("bad session") - - -@pytest.fixture -def ctx(): - return L.LeverContext( - count_tokens=lambda t, *, model: len(t) // 4, - mode=L.MODE_SHADOW, - settings={"keep_bytes": 4096}, - ) - - -@pytest.fixture -def session(): - return L.from_corpus_session(RAW) - - -@pytest.fixture -def registered(): - return L.RegisteredLever(lever=Truncator(), dist="ace-skills") - - -# -- the normalized model ---------------------------------------------------------------- - - -def test_one_adapter_serves_any_agent(session): - assert (session.agent, session.n_turns, session.kind) == ("codex", 2, "main") - assert session.turns[0].ts is not None - assert list(session.iter_calls())[1][:2] == (1, 0) - - -def test_prompt_tokens_sums_the_cached_buckets(session): - """`input_tokens` EXCLUDES the cached buckets, so this is a sum and not a max. - - Subtracting cache_read from input to "correct" it under-reports prompt volume — a bug - with no visible symptom. - """ - assert session.turns[0].usage.prompt_tokens == 100 + 900 + 400 - - -def test_ttl_breakdown_is_provider_neutral(session): - assert session.turns[0].usage.cache_write_by_ttl == {"5m": 400} - - -def test_measure_only_sessions_carry_no_content(session): - assert session.turns[0].calls[0].content is None - assert not session.turns[0].calls[0].has_content - - -# -- levers ------------------------------------------------------------------------------ - - -def test_a_lever_scores_on_hashes_and_sizes_alone(registered, session, ctx): - assert isinstance(registered.lever, L.Lever) - p = L.propose_safely(registered, session, ctx) - assert p is not None and len(p.edits) == 1 - assert p.edits[0].keep_bytes == 4096 - - -def test_edit_free_proposal_is_a_real_result(session, ctx): - """A loop guardrail's whole output is its diagnostics; `if proposal:` must not eat it.""" - p = L.Proposal("loop_guard", (), {"loops_detected": 3}) - assert not p.edits - assert p.diagnostics == {"loops_detected": 3} - - -# -- presence is not consent ------------------------------------------------------------- - - -@pytest.mark.parametrize( - "config,expected", - [ - ({}, L.MODE_OFF), - ({"levers": {"truncate_dumps": "shadow"}}, L.MODE_SHADOW), - ({"levers": {"truncate_dumps": {"mode": "on"}}}, L.MODE_ON), - # A bare `true` is an enablement, and the safe reading of "enabled" is the mode that - # changes nothing about the request. - ({"levers": {"truncate_dumps": True}}, L.MODE_SHADOW), - ], -) -def test_mode_resolution(registered, config, expected): - assert L.resolve_modes([registered], config=config) == {"truncate_dumps": expected} - - -def test_an_installed_lever_defaults_to_off(registered): - """The rule the registry exists to enforce. An unconfigured lever must never act.""" - assert L.resolve_modes([registered], config={}) == {"truncate_dumps": L.MODE_OFF} - - -def test_an_unknown_mode_resolves_to_off_not_to_the_default(registered, caplog): - """A typo becoming `shadow` is tolerable; a typo becoming `on` is not.""" - with caplog.at_level(logging.WARNING): - modes = L.resolve_modes([registered], config={"levers": {"truncate_dumps": "Bogus"}}) - assert modes == {"truncate_dumps": L.MODE_OFF} - - -def test_settings_exclude_mode(): - got = L.load_settings( - "truncate_dumps", - config={"levers": {"truncate_dumps": {"mode": "on", "keep_bytes": 99}}}, - ) - assert got == {"keep_bytes": 99} - - -# -- requires_content --------------------------------------------------------------------- - - -def test_content_requiring_lever_is_refused_not_raised(session, ctx): - """Refused outright rather than allowed to half-run: a partial proposal is worse than - none, because the ledger cannot tell it from a complete one.""" - nb = L.RegisteredLever(lever=NeedsBytes()) - assert L.propose_safely(nb, session, ctx) is None - - -def test_content_requiring_lever_runs_where_an_actuator_supplied_bytes(ctx): - s = L.from_corpus_session( - RAW, content_for=lambda ti, ci: (lambda: "x" * 240000) if ti == 0 else None - ) - assert s.turns[0].calls[0].has_content - assert not s.turns[1].calls[0].has_content - p = L.propose_safely(L.RegisteredLever(lever=NeedsBytes()), s, ctx) - assert p is not None and p.diagnostics == {"bytes": 240000} - - -def test_content_ref_refuses_in_measure_only_mode(): - with pytest.raises(L.ContentUnavailable): - L.ContentRef().resolve() - - -# -- failure isolation -------------------------------------------------------------------- - - -def test_a_throwing_lever_costs_its_own_row_and_nothing_else(session, ctx, caplog): - with caplog.at_level(logging.WARNING): - assert L.propose_safely(L.RegisteredLever(lever=Boom()), session, ctx) is None - - -def test_no_lever_package_is_the_ordinary_case(): - """An empty entry-point group is not an error state — it is the open-source default.""" - assert L.discover("ace.sidecar.levers.nonexistent") == () From ad0f6e6d9d2c323a5fa277013fdcf5cb9f3d9587 Mon Sep 17 00:00:00 2001 From: ACE Engineering Date: Thu, 27 Aug 2026 23:58:57 -0700 Subject: [PATCH 7/8] feat(quality): add coding task capability breakdown by domain (UI, Backend, Testing, Docs, Research) --- src/ace/sidecar/dashboard_render.py | 76 ++++++++++++- src/ace/sidecar/insights.py | 170 ++++++++++++++++++++++++++++ tests/test_quality_metrics.py | 146 ++++++++++++++++++++++++ 3 files changed, 391 insertions(+), 1 deletion(-) diff --git a/src/ace/sidecar/dashboard_render.py b/src/ace/sidecar/dashboard_render.py index 47df73e..14e5684 100644 --- a/src/ace/sidecar/dashboard_render.py +++ b/src/ace/sidecar/dashboard_render.py @@ -911,6 +911,79 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"" ) + category_table = "" + by_category = qm.get("by_category") or {} + category_rows = [] + for ck, c_info in by_category.items(): + c_sessions = c_info.get("sessions", 0) + if c_sessions == 0: + continue + c_label = c_info.get("label", ck.capitalize()) + c_icon = c_info.get("icon", "💻") + c_desc = c_info.get("desc", "") + c_score = c_info.get("quality_score", 100) + c_grade = c_info.get("grade", "A") + c_comp = c_info.get("task_completion_rate_pct", 100.0) + c_verif = c_info.get("verification_rate_pct", 100.0) + c_fsr = c_info.get("first_pass_success_rate_pct", 100.0) + c_thrash = c_info.get("thrashed_files_count", 0) + c_share = c_info.get("share_pct", 0.0) + best_agent = c_info.get("best_agent", "—") + best_model = c_info.get("best_model", "—") + + score_badge = ( + "color:var(--mint);border-color:#1d3b2e;background:#0F231A" + if c_score >= 80 + else ( + "color:var(--gold);border-color:#3d3014;background:#241D0E" + if c_score >= 60 + else "color:var(--crit);border-color:#4a1e17;background:#2a110e" + ) + ) + category_rows.append( + f"" + f"" + f"
    " + f"{c_icon}{escape(c_label)}" + f"({c_share}%)" + f"
    " + f"
    {escape(c_desc)}
    " + f"" + f"{c_score} ({c_grade})" + f"{c_comp}%" + f"{c_verif}%" + f"{c_fsr}%" + f"{'' + str(c_thrash) + '' if c_thrash > 0 else '0'}" + f"{c_sessions}" + f"" + f"
    {escape(best_agent)}
    " + f"
    {escape(best_model)}
    " + f"" + f"" + ) + + if category_rows: + category_table = ( + f"
    " + f"
    " + f"CAPABILITY & PERFORMANCE BY CODING TASK DOMAIN" + f"
    " + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"" + f"{''.join(category_rows)}" + f"
    TASK CATEGORYSCORECOMPLETIONVERIFICATIONFIRST-PASS SUCCESSTHRASH FILESSESSIONSBEST FIT ENGINE / MODEL
    " + f"
    " + ) + return ( f"
    {''.join(tiles)}
    " f"
    " @@ -924,7 +997,8 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"
    " f"{thrash_html}" f"{matrix_table}" - f"
    Measures how safely and stably coding agents operate in your repository. Correlates cost against first-pass tool correctness and test diligence.
    " + f"{category_table}" + f"
    Measures how safely and stably coding agents operate in your repository. Correlates cost against first-pass tool correctness, test diligence, and domain task fit.
    " f"" ) diff --git a/src/ace/sidecar/insights.py b/src/ace/sidecar/insights.py index 2973ab3..8c0598e 100644 --- a/src/ace/sidecar/insights.py +++ b/src/ace/sidecar/insights.py @@ -188,6 +188,109 @@ def _sig(name: str, tool_input: Dict[str, Any]) -> str: ".vue", ) +TASK_CAT_UI = "ui" +TASK_CAT_BACKEND = "backend" +TASK_CAT_TESTING = "testing" +TASK_CAT_DOCS = "docs" +TASK_CAT_RESEARCH = "research" + +TASK_CATEGORIES: Dict[str, Dict[str, str]] = { + TASK_CAT_UI: { + "label": "UI & Frontend", + "icon": "🎨", + "desc": "User interfaces, web layouts, components, templates, and styling.", + }, + TASK_CAT_BACKEND: { + "label": "Backend & Systems", + "icon": "⚙️", + "desc": "Server logic, database models, APIs, pipelines, and algorithms.", + }, + TASK_CAT_TESTING: { + "label": "Testing & QA", + "icon": "🧪", + "desc": "Unit/integration tests, test harnesses, assertions, and linter auto-fixes.", + }, + TASK_CAT_DOCS: { + "label": "Docs & Specs", + "icon": "📝", + "desc": "Architecture documentation, implementation plans, and walkthroughs.", + }, + TASK_CAT_RESEARCH: { + "label": "Codebase Research", + "icon": "🔍", + "desc": "Codebase navigation, symbol search, architecture analysis, and comprehension.", + }, +} + +_UI_EXTS = ( + ".tsx", + ".jsx", + ".vue", + ".svelte", + ".css", + ".scss", + ".sass", + ".html", + ".svg", +) +_DOC_EXTS = (".md", ".mdx", ".rst", ".txt", ".adoc") + + +def _classify_session_task_category(session: Dict[str, Any]) -> str: + """Classifies a coding session into a primary task domain based on tool calls and target files.""" + turns = session.get("turns") or [] + edits: List[str] = [] + has_test_runs = False + + for t in turns: + for c in t.get("calls") or []: + if c.get("is_edit"): + tgt = str(c.get("raw_target") or c.get("target") or "").lower() + edits.append(tgt) + if c.get("is_test_run"): + has_test_runs = True + + if not edits: + return TASK_CAT_RESEARCH + + ui_count = sum( + 1 + for e in edits + if any(e.endswith(ext) for ext in _UI_EXTS) + or any( + k in e + for k in ( + "/ui/", + "/frontend/", + "/components/", + "/views/", + "/styles/", + "/web/", + "/templates/", + "dashboard_render", + ) + ) + ) + test_count = sum(1 for e in edits if _TEST_FILE_RE.search(e)) + doc_count = sum( + 1 + for e in edits + if any(e.endswith(ext) for ext in _DOC_EXTS) + or any(k in e for k in ("doc", "readme", "walkthrough", "plan")) + ) + backend_count = len(edits) - ui_count - test_count - doc_count + + counts = { + TASK_CAT_UI: ui_count, + TASK_CAT_TESTING: test_count, + TASK_CAT_DOCS: doc_count, + TASK_CAT_BACKEND: max(0, backend_count), + } + top_cat = max(counts.items(), key=lambda kv: kv[1]) + if top_cat[1] > 0: + return top_cat[0] + return TASK_CAT_BACKEND + def _classify_call(name: str, tool_input: Dict[str, Any]) -> Dict[str, Any]: target_raw = None @@ -2204,11 +2307,13 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: Includes top-line metrics along with breakdowns: - by_agent: Quality scores partitioned per agent engine (Claude Code, Antigravity, Codex). - by_model: Quality scores partitioned per LLM model. + - by_category: Quality and capability metrics partitioned per coding task domain (UI, Backend, Testing, Docs, Research). """ overall = _calc_quality_block(sess) if not sess: overall["by_agent"] = {} overall["by_model"] = [] + overall["by_category"] = {} return overall # Group by agent @@ -2254,8 +2359,62 @@ def quality_metrics(sess: List[Dict[str, Any]]) -> Dict[str, Any]: } ) + # Group by task category + by_category: Dict[str, Any] = {} + category_groups: Dict[str, List[Dict[str, Any]]] = {} + for s in sess: + ck = _classify_session_task_category(s) + category_groups.setdefault(ck, []).append(s) + + for ck, cat_meta in TASK_CATEGORIES.items(): + c_sess = category_groups.get(ck) or [] + block = _calc_quality_block(c_sess) + + best_agent = "—" + best_agent_score = -1 + for ak in (AGENT_CLAUDE, AGENT_ANTIGRAVITY, AGENT_CODEX): + sub_ak = [s for s in c_sess if (s.get("agent_type") or AGENT_CLAUDE) == ak] + if sub_ak: + sc = _calc_quality_block(sub_ak)["quality_score"] + if sc > best_agent_score: + best_agent_score = sc + best_agent = AGENTS.get(ak, ak.capitalize()) + + best_model = "—" + best_model_score = -1 + cat_models = set( + t.get("model") + for s in c_sess + for t in s.get("turns", []) + if t.get("model") and not str(t.get("model")).startswith("<") + ) + for m in cat_models: + sub_m = [] + for s in c_sess: + proj = [t for t in s.get("turns", []) if t.get("model") == m] + if proj: + sub_m.append({"turns": proj, "events": s.get("events", [])}) + if sub_m: + sc = _calc_quality_block(sub_m)["quality_score"] + if sc > best_model_score: + best_model_score = sc + best_model = m + + by_category[ck] = { + "category": ck, + "label": cat_meta["label"], + "icon": cat_meta["icon"], + "desc": cat_meta["desc"], + "sessions": len(c_sess), + "share_pct": round(len(c_sess) / max(1, len(sess)) * 100.0, 1), + "best_agent": best_agent, + "best_model": best_model, + **block, + } + overall["by_agent"] = by_agent overall["by_model"] = by_model + overall["by_category"] = by_category return overall @@ -3210,6 +3369,17 @@ def format_prometheus_metrics(d: Dict[str, Any]) -> str: lines.append("# TYPE ace_quality_test_to_code_ratio gauge") lines.append(f'ace_quality_test_to_code_ratio {qm.get("test_to_code_ratio", 1.0)}') + # Task Category Performance Metrics + lines.append("# HELP ace_quality_category_score Quality score partitioned by task category.") + lines.append("# TYPE ace_quality_category_score gauge") + for cat_id, cat_info in (qm.get("by_category") or {}).items(): + lines.append(f'ace_quality_category_score{{category="{cat_id}"}} {cat_info.get("quality_score", 100)}') + + lines.append("# HELP ace_quality_category_completion_rate Task completion rate partitioned by task category.") + lines.append("# TYPE ace_quality_category_completion_rate gauge") + for cat_id, cat_info in (qm.get("by_category") or {}).items(): + lines.append(f'ace_quality_category_completion_rate{{category="{cat_id}"}} {cat_info.get("task_completion_rate", 1.0)}') + return "\n".join(lines) + "\n" diff --git a/tests/test_quality_metrics.py b/tests/test_quality_metrics.py index 02e6512..33b5db3 100644 --- a/tests/test_quality_metrics.py +++ b/tests/test_quality_metrics.py @@ -348,3 +348,149 @@ def test_quality_metrics_by_agent_and_model() -> None: assert "claude-sonnet-4-6" in html assert "gemini-3.6-flash" in html assert "ENGINE / MODEL" in html + + +def test_quality_metrics_by_task_category() -> None: + from ace.sidecar.insights import ( + _classify_session_task_category, + TASK_CAT_UI, + TASK_CAT_BACKEND, + TASK_CAT_TESTING, + TASK_CAT_DOCS, + TASK_CAT_RESEARCH, + ) + + # 1. UI session + s_ui = { + "session": "ui_sess", + "turns": [ + { + "input_tokens": 100, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "output_tokens": 50, + "model": "claude-sonnet-4-6", + "calls": [ + {"name": "write_to_file", "raw_target": "src/components/Header.tsx", "is_edit": True}, + {"name": "write_to_file", "raw_target": "src/styles/app.css", "is_edit": True}, + ], + } + ], + "events": [], + } + assert _classify_session_task_category(s_ui) == TASK_CAT_UI + + # 2. Testing session + s_test = { + "session": "test_sess", + "turns": [ + { + "input_tokens": 100, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "output_tokens": 50, + "model": "claude-sonnet-4-6", + "calls": [ + {"name": "write_to_file", "raw_target": "tests/test_api.py", "is_edit": True, "is_test_file": True}, + {"name": "Bash", "command": "pytest", "is_test_run": True}, + ], + } + ], + "events": [], + } + assert _classify_session_task_category(s_test) == TASK_CAT_TESTING + + # 3. Docs session + s_docs = { + "session": "docs_sess", + "turns": [ + { + "input_tokens": 100, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "output_tokens": 50, + "model": "claude-sonnet-4-6", + "calls": [ + {"name": "write_to_file", "raw_target": "docs/architecture.md", "is_edit": True}, + {"name": "replace_file_content", "raw_target": "README.md", "is_edit": True}, + ], + } + ], + "events": [], + } + assert _classify_session_task_category(s_docs) == TASK_CAT_DOCS + + # 4. Research session (read-only) + s_res = { + "session": "res_sess", + "turns": [ + { + "input_tokens": 100, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "output_tokens": 50, + "model": "gemini-3.6-flash", + "calls": [ + {"name": "view_file", "raw_target": "src/server.py", "is_view": True}, + {"name": "grep_search", "raw_target": "main", "is_view": False}, + ], + } + ], + "events": [], + } + assert _classify_session_task_category(s_res) == TASK_CAT_RESEARCH + + # 5. Backend session + s_back = { + "session": "back_sess", + "turns": [ + { + "input_tokens": 100, + "cache_read_input_tokens": 0, + "cache_creation_input_tokens": 0, + "ephemeral_5m_input_tokens": 0, + "ephemeral_1h_input_tokens": 0, + "output_tokens": 50, + "model": "gpt-5.3-codex", + "calls": [ + {"name": "replace_file_content", "raw_target": "src/ace/gateway/proxy.py", "is_edit": True, "is_src_file": True}, + ], + } + ], + "events": [], + } + assert _classify_session_task_category(s_back) == TASK_CAT_BACKEND + + # Multi-session aggregation in quality_metrics + sess_all = [s_ui, s_test, s_docs, s_res, s_back] + qm = quality_metrics(sess_all) + assert "by_category" in qm + assert TASK_CAT_UI in qm["by_category"] + assert TASK_CAT_BACKEND in qm["by_category"] + assert TASK_CAT_TESTING in qm["by_category"] + assert TASK_CAT_DOCS in qm["by_category"] + assert TASK_CAT_RESEARCH in qm["by_category"] + + ui_cat = qm["by_category"][TASK_CAT_UI] + assert ui_cat["sessions"] == 1 + assert ui_cat["label"] == "UI & Frontend" + + # Prometheus export + payload = _build_payload(sess_all, capture=None, range_key="all", agent="all", store_path=None) + prom_text = format_prometheus_metrics(payload) + assert 'ace_quality_category_score{category="ui"}' in prom_text + assert 'ace_quality_category_completion_rate{category="ui"}' in prom_text + + # Render dashboard + html = render(payload) + assert "CAPABILITY & PERFORMANCE BY CODING TASK DOMAIN" in html or "CAPABILITY & PERFORMANCE BY CODING TASK DOMAIN" in html + assert "UI & Frontend" in html or "UI & Frontend" in html + From c3298d6b623c5b5dc14d55ea40dd1f73b121b26f Mon Sep 17 00:00:00 2001 From: ACE Engineering Date: Fri, 28 Aug 2026 00:16:59 -0700 Subject: [PATCH 8/8] feat(quality): add turns/task, time/task, follow-up fixes, verbosity, and comment-to-code ratio --- src/ace/sidecar/dashboard_render.py | 114 +++++++++++++++----- src/ace/sidecar/insights.py | 155 ++++++++++++++++++++++++++++ tests/test_quality_metrics.py | 119 +++++++++++++++++++++ 3 files changed, 361 insertions(+), 27 deletions(-) diff --git a/src/ace/sidecar/dashboard_render.py b/src/ace/sidecar/dashboard_render.py index 14e5684..0caf082 100644 --- a/src/ace/sidecar/dashboard_render.py +++ b/src/ace/sidecar/dashboard_render.py @@ -733,11 +733,19 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: thrash_cnt = qm.get("thrashed_files_count", 0) recovery_turns = qm.get("avg_error_recovery_turns", 1.0) redundant_reads = qm.get("redundant_reads_count", 0) - test_code_ratio = qm.get("test_to_code_ratio", 1.0) sessions_edits = qm.get("sessions_with_edits", 0) sessions_tests = qm.get("sessions_with_tests", 0) clean_completed = qm.get("clean_completed_sessions", 0) + turns_task = qm.get("turns_per_completion_avg", 1.0) + time_task_min = qm.get("duration_minutes_per_completion_avg", 0.0) + followup_fixes = qm.get("followup_code_fixes_count", 0) + followup_rate = qm.get("followup_code_fix_rate_pct", 0.0) + comment_ratio = qm.get("comment_to_code_ratio", 0.0) + comment_density = qm.get("comment_density_pct", 0.0) + verbosity_tok = qm.get("verbosity_tokens_per_turn", 0.0) + verbosity_lvl = qm.get("verbosity_level", "Concise") + score_color = ( "var(--mint)" if score >= 80 @@ -765,12 +773,18 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: title="Percentage of sessions that resolved cleanly without trailing tool errors or unverified changes.", ), _st( - "verification_rate", - f"{v_rate}%", - f"{sessions_tests} of {sessions_edits} edit sessions", - delta="TEST HYGIENE", - dcls=v_cls, - title="Percentage of sessions containing file modifications that executed an automated test runner or linter (pytest, npm test, ruff, etc.).", + "turns_per_task", + f"{turns_task} turns", + "avg turns / completion", + delta="CONVERSATION EFFICIENCY", + title="Average number of conversation turns required to achieve a verified clean task completion.", + ), + _st( + "time_per_task", + f"{time_task_min} min", + "avg elapsed / completion", + delta="DELIVERY SPEED", + title="Average wall-clock duration in minutes from session start to verified resolution.", ), _st( "first_pass_success", @@ -780,6 +794,35 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: dcls=fsr_cls, title="Share of tool executions that succeeded on their first attempt without returning execution errors or non-zero exit codes.", ), + _st( + "verification_rate", + f"{v_rate}%", + f"{sessions_tests} of {sessions_edits} edit sessions", + delta="TEST HYGIENE", + dcls=v_cls, + title="Percentage of sessions containing file modifications that executed an automated test runner or linter (pytest, npm test, ruff, etc.).", + ), + _st( + "followup_fixes", + f"{followup_fixes}", + f"{followup_rate}% rework rate", + delta="FOLLOW-UP FIXES", + title="Follow-up code modifications and bugfixes applied to the same files in subsequent turns.", + ), + _st( + "comment_ratio", + f"{comment_ratio}x", + f"{comment_density}% comment density", + delta="CODE COMMENT DENSITY", + title="Ratio of inline comments to executable code lines in modifications.", + ), + _st( + "verbosity", + f"{verbosity_tok} tok", + f"{verbosity_lvl} explanation", + delta="VERBOSITY LEVEL", + title="Average generated output tokens per conversational turn.", + ), _st( "edit_thrash_files", f"{thrash_cnt}", @@ -788,13 +831,6 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: dcls=thrash_cls, title="Files edited 3 or more times within the same session, indicating thrashing or lack of convergence.", ), - _st( - "healing_latency", - f"{recovery_turns} turns", - f"{redundant_reads} redundant reads", - delta="ERROR HEALING", - title="Average number of conversation turns required for the agent to resolve a failed tool execution and resume forward progress.", - ), ] thrashed_files_list = qm.get("thrashed_files_list") or [] @@ -818,10 +854,13 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: a_score = a_info.get("quality_score", 100) a_grade = a_info.get("grade", "A") a_comp = a_info.get("task_completion_rate_pct", 100.0) + a_turns = a_info.get("turns_per_completion_avg", 1.0) + a_time = a_info.get("duration_minutes_per_completion_avg", 0.0) a_v_rate = a_info.get("verification_rate_pct", 100.0) a_fsr = a_info.get("first_pass_success_rate_pct", 100.0) - a_thrash = a_info.get("thrashed_files_count", 0) - a_rec = a_info.get("avg_error_recovery_turns", 1.0) + a_fixes = a_info.get("followup_code_fixes_count", 0) + a_comm = a_info.get("comment_to_code_ratio", 0.0) + a_verb = a_info.get("verbosity_tokens_per_turn", 0.0) a_sess = a_info.get("sessions", 0) badge_style = ( "color:var(--mint);border-color:#1d3b2e;background:#0F231A" @@ -846,10 +885,13 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"{escape(a_info.get('label', ak))}" f"{a_score} ({a_grade})" f"{a_comp}%" + f"{a_turns}" + f"{a_time}m" f"{a_v_rate}%" f"{a_fsr}%" - f"{'' + str(a_thrash) + '' if a_thrash > 0 else '0'}" - f"{a_rec} turns" + f"{a_fixes}" + f"{a_comm}x" + f"{a_verb} tok" f"{a_sess}" f"" ) @@ -861,10 +903,13 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: m_score = m_info.get("quality_score", 100) m_grade = m_info.get("grade", "A") m_comp = m_info.get("task_completion_rate_pct", 100.0) + m_turns = m_info.get("turns_per_completion_avg", 1.0) + m_time = m_info.get("duration_minutes_per_completion_avg", 0.0) m_v_rate = m_info.get("verification_rate_pct", 100.0) m_fsr = m_info.get("first_pass_success_rate_pct", 100.0) - m_thrash = m_info.get("thrashed_files_count", 0) - m_rec = m_info.get("avg_error_recovery_turns", 1.0) + m_fixes = m_info.get("followup_code_fixes_count", 0) + m_comm = m_info.get("comment_to_code_ratio", 0.0) + m_verb = m_info.get("verbosity_tokens_per_turn", 0.0) m_sess = m_info.get("sessions", 0) score_badge = ( "color:var(--mint);border-color:#1d3b2e;background:#0F231A" @@ -880,10 +925,13 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"{escape(m_name)}" f"{m_score} ({m_grade})" f"{m_comp}%" + f"{m_turns}" + f"{m_time}m" f"{m_v_rate}%" f"{m_fsr}%" - f"{'' + str(m_thrash) + '' if m_thrash > 0 else '0'}" - f"{m_rec} turns" + f"{m_fixes}" + f"{m_comm}x" + f"{m_verb} tok" f"{m_sess}" f"" ) @@ -900,10 +948,13 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"ENGINE / MODEL" f"SCORE" f"COMPLETION" + f"TURNS/TASK" + f"TIME/TASK" f"VERIFICATION" f"FIRST-PASS SUCCESS" - f"THRASH FILES" - f"HEALING TURNS" + f"FOLLOW-UP FIXES" + f"COMMENT RATIO" + f"VERBOSITY" f"SESSIONS" f"" f"{''.join(breakdown_rows)}" @@ -924,9 +975,12 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: c_score = c_info.get("quality_score", 100) c_grade = c_info.get("grade", "A") c_comp = c_info.get("task_completion_rate_pct", 100.0) + c_turns = c_info.get("turns_per_completion_avg", 1.0) + c_time = c_info.get("duration_minutes_per_completion_avg", 0.0) c_verif = c_info.get("verification_rate_pct", 100.0) c_fsr = c_info.get("first_pass_success_rate_pct", 100.0) - c_thrash = c_info.get("thrashed_files_count", 0) + c_fixes = c_info.get("followup_code_fixes_count", 0) + c_comm = c_info.get("comment_to_code_ratio", 0.0) c_share = c_info.get("share_pct", 0.0) best_agent = c_info.get("best_agent", "—") best_model = c_info.get("best_model", "—") @@ -951,9 +1005,12 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"" f"{c_score} ({c_grade})" f"{c_comp}%" + f"{c_turns}" + f"{c_time}m" f"{c_verif}%" f"{c_fsr}%" - f"{'' + str(c_thrash) + '' if c_thrash > 0 else '0'}" + f"{c_fixes}" + f"{c_comm}x" f"{c_sessions}" f"" f"
    {escape(best_agent)}
    " @@ -973,9 +1030,12 @@ def _quality(qm: Optional[Dict[str, Any]]) -> str: f"TASK CATEGORY" f"SCORE" f"COMPLETION" + f"TURNS/TASK" + f"TIME/TASK" f"VERIFICATION" f"FIRST-PASS SUCCESS" - f"THRASH FILES" + f"FOLLOW-UP FIXES" + f"COMMENT RATIO" f"SESSIONS" f"BEST FIT ENGINE / MODEL" f"" diff --git a/src/ace/sidecar/insights.py b/src/ace/sidecar/insights.py index 8c0598e..36f53c0 100644 --- a/src/ace/sidecar/insights.py +++ b/src/ace/sidecar/insights.py @@ -292,6 +292,24 @@ def _classify_session_task_category(session: Dict[str, Any]) -> str: return TASK_CAT_BACKEND +_COMMENT_LINE_RE = re.compile(r"^\s*(#|//|/\*|\*|\*/|\"\"\"|\'\'\'|--|