From c76ee3f9e326401925ae612ee0f50fceda578e8e Mon Sep 17 00:00:00 2001 From: Trask Stalnaker Date: Fri, 26 Jun 2026 11:58:22 -0700 Subject: [PATCH] Capture Copilot assistant usage events --- .../pull-request-dashboard/classification.py | 19 +++++++++++++++---- .../pull-request-dashboard/dashboard.py | 11 ++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/scripts/pull-request-dashboard/classification.py b/.github/scripts/pull-request-dashboard/classification.py index 738f57e..0dd1819 100644 --- a/.github/scripts/pull-request-dashboard/classification.py +++ b/.github/scripts/pull-request-dashboard/classification.py @@ -87,6 +87,9 @@ "input_tokens": ("input_tokens", "prompt_tokens", "promptTokens", "inputTokens"), "output_tokens": ("output_tokens", "completion_tokens", "completionTokens", "outputTokens"), "total_tokens": ("total_tokens", "totalTokens", "tokens"), + "cache_read_tokens": ("cache_read_tokens", "cacheReadTokens", "cached_tokens", "cachedTokens"), + "cache_write_tokens": ("cache_write_tokens", "cacheWriteTokens"), + "reasoning_tokens": ("reasoning_tokens", "reasoningTokens"), } @@ -116,12 +119,18 @@ def normalize_copilot_usage(usage: Any) -> dict[str, int]: input_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["input_tokens"]) output_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["output_tokens"]) total_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["total_tokens"]) + cache_read_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["cache_read_tokens"]) + cache_write_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["cache_write_tokens"]) + reasoning_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["reasoning_tokens"]) if not total_tokens and (input_tokens or output_tokens): total_tokens = input_tokens + output_tokens normalized = { "input_tokens": input_tokens, "output_tokens": output_tokens, "total_tokens": total_tokens, + "cache_read_tokens": cache_read_tokens, + "cache_write_tokens": cache_write_tokens, + "reasoning_tokens": reasoning_tokens, } return {key: value for key, value in normalized.items() if value} @@ -137,13 +146,15 @@ def parse_copilot_jsonl(s: str) -> tuple[str, dict[str, int]]: evt = json.loads(line) except json.JSONDecodeError: continue - if evt.get("type") == "assistant.message": - content = (evt.get("data") or {}).get("content") + event_type = evt.get("type") + data = evt.get("data") or {} + if event_type == "assistant.message": + content = data.get("content") if isinstance(content, str): parts.append(content) - event_usage = normalize_copilot_usage(evt.get("usage")) + event_usage = normalize_copilot_usage(data if event_type == "assistant.usage" else evt.get("usage")) if not event_usage: - event_usage = normalize_copilot_usage((evt.get("data") or {}).get("usage")) + event_usage = normalize_copilot_usage(data.get("usage")) for key, value in event_usage.items(): usage[key] = usage.get(key, 0) + value return "\n".join(parts), usage diff --git a/.github/scripts/pull-request-dashboard/dashboard.py b/.github/scripts/pull-request-dashboard/dashboard.py index 411272c..6956ff7 100644 --- a/.github/scripts/pull-request-dashboard/dashboard.py +++ b/.github/scripts/pull-request-dashboard/dashboard.py @@ -910,6 +910,9 @@ def empty_copilot_usage() -> dict[str, int]: "input_tokens": 0, "output_tokens": 0, "total_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "reasoning_tokens": 0, } @@ -928,6 +931,9 @@ def add_copilot_usage(aggregate: dict[str, int], result: dict[str, Any] | None) aggregate["input_tokens"] += int(usage.get("input_tokens") or 0) aggregate["output_tokens"] += int(usage.get("output_tokens") or 0) aggregate["total_tokens"] += int(usage.get("total_tokens") or 0) + aggregate["cache_read_tokens"] += int(usage.get("cache_read_tokens") or 0) + aggregate["cache_write_tokens"] += int(usage.get("cache_write_tokens") or 0) + aggregate["reasoning_tokens"] += int(usage.get("reasoning_tokens") or 0) def copilot_usage_from_results(results: dict[int, dict[str, Any]]) -> dict[str, int]: @@ -946,7 +952,10 @@ def print_copilot_usage_summary(repo: str, model: str, usage: dict[str, int]) -> f"missing_usage={usage.get('missing_usage_calls', 0)}, " f"input_tokens={usage.get('input_tokens', 0)}, " f"output_tokens={usage.get('output_tokens', 0)}, " - f"total_tokens={usage.get('total_tokens', 0)}", + f"total_tokens={usage.get('total_tokens', 0)}, " + f"cache_read_tokens={usage.get('cache_read_tokens', 0)}, " + f"cache_write_tokens={usage.get('cache_write_tokens', 0)}, " + f"reasoning_tokens={usage.get('reasoning_tokens', 0)}", file=sys.stderr, )