Skip to content

Capture Copilot assistant usage events #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
maryliag merged 1 commit into from
Jun 26, 2026
+25 −5
Merged

Capture Copilot assistant usage events #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions .github/scripts/pull-request-dashboard/classification.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,9 @@
"input_tokens": ("input_tokens", "prompt_tokens", "promptTokens", "inputTokens"),
"output_tokens": ("output_tokens", "completion_tokens", "completionTokens", "outputTokens"),
"total_tokens": ("total_tokens", "totalTokens", "tokens"),
"cache_read_tokens": ("cache_read_tokens", "cacheReadTokens", "cached_tokens", "cachedTokens"),
"cache_write_tokens": ("cache_write_tokens", "cacheWriteTokens"),
"reasoning_tokens": ("reasoning_tokens", "reasoningTokens"),
}


Expand Down Expand Up @@ -116,12 +119,18 @@ def normalize_copilot_usage(usage: Any) -> dict[str, int]:
input_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["input_tokens"])
output_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["output_tokens"])
total_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["total_tokens"])
cache_read_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["cache_read_tokens"])
cache_write_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["cache_write_tokens"])
reasoning_tokens = _first_usage_value(usage, _COPILOT_USAGE_FIELDS["reasoning_tokens"])
if not total_tokens and (input_tokens or output_tokens):
total_tokens = input_tokens + output_tokens
normalized = {
"input_tokens": input_tokens,
"output_tokens": output_tokens,
"total_tokens": total_tokens,
"cache_read_tokens": cache_read_tokens,
"cache_write_tokens": cache_write_tokens,
"reasoning_tokens": reasoning_tokens,
}
return {key: value for key, value in normalized.items() if value}

Expand All @@ -137,13 +146,15 @@ def parse_copilot_jsonl(s: str) -> tuple[str, dict[str, int]]:
evt = json.loads(line)
except json.JSONDecodeError:
continue
if evt.get("type") == "assistant.message":
content = (evt.get("data") or {}).get("content")
event_type = evt.get("type")
data = evt.get("data") or {}
if event_type == "assistant.message":
content = data.get("content")
if isinstance(content, str):
parts.append(content)
event_usage = normalize_copilot_usage(evt.get("usage"))
event_usage = normalize_copilot_usage(data if event_type == "assistant.usage" else evt.get("usage"))
if not event_usage:
event_usage = normalize_copilot_usage((evt.get("data") or {}).get("usage"))
event_usage = normalize_copilot_usage(data.get("usage"))
for key, value in event_usage.items():
usage[key] = usage.get(key, 0) + value
return "\n".join(parts), usage
Expand Down
11 changes: 10 additions & 1 deletion .github/scripts/pull-request-dashboard/dashboard.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -910,6 +910,9 @@ def empty_copilot_usage() -> dict[str, int]:
"input_tokens": 0,
"output_tokens": 0,
"total_tokens": 0,
"cache_read_tokens": 0,
"cache_write_tokens": 0,
"reasoning_tokens": 0,
}


Expand All @@ -928,6 +931,9 @@ def add_copilot_usage(aggregate: dict[str, int], result: dict[str, Any] | None)
aggregate["input_tokens"] += int(usage.get("input_tokens") or 0)
aggregate["output_tokens"] += int(usage.get("output_tokens") or 0)
aggregate["total_tokens"] += int(usage.get("total_tokens") or 0)
aggregate["cache_read_tokens"] += int(usage.get("cache_read_tokens") or 0)
aggregate["cache_write_tokens"] += int(usage.get("cache_write_tokens") or 0)
aggregate["reasoning_tokens"] += int(usage.get("reasoning_tokens") or 0)


def copilot_usage_from_results(results: dict[int, dict[str, Any]]) -> dict[str, int]:
Expand All @@ -946,7 +952,10 @@ def print_copilot_usage_summary(repo: str, model: str, usage: dict[str, int]) ->
f"missing_usage={usage.get('missing_usage_calls', 0)}, "
f"input_tokens={usage.get('input_tokens', 0)}, "
f"output_tokens={usage.get('output_tokens', 0)}, "
f"total_tokens={usage.get('total_tokens', 0)}",
f"total_tokens={usage.get('total_tokens', 0)}, "
f"cache_read_tokens={usage.get('cache_read_tokens', 0)}, "
f"cache_write_tokens={usage.get('cache_write_tokens', 0)}, "
f"reasoning_tokens={usage.get('reasoning_tokens', 0)}",
file=sys.stderr,
)

Expand Down