diff --git a/CLAUDE.md b/CLAUDE.md index 4fec922..10aab71 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,7 +94,7 @@ Optional daemon services (feature-gated via config): - **CCInfoTimerService**: Lazy-fetch background timer for Claude Code statusline data (cost, quota, git info), plus an always-on Anthropic usage sync (10-min interval, only while a `claude` process is running) that pushes quota to `/api/v1/anthropic-usage` without needing statusline activity - **CodexUsageSyncService**: Syncs Codex usage to `/api/v1/codex-usage` every 10 min, only while a `codex`/`chatgpt` process is running (both intervals share the `usageSyncInterval` const in `daemon/usage_sync.go`) - **SyncCircuitBreakerService**: Retry failed syncs with file-based persistence (`sync_pending.log`) and hourly recovery timer -- **AICodeOtelServer**: gRPC OTEL collector for AI coding CLI metrics/logs (Claude Code, Codex). In a Claude Code cloud session (`CLAUDE_CODE_REMOTE=true`, where the SessionStart hook starts the daemon) it adds the VM's `CLAUDE_CODE_REMOTE_SESSION_ID` as the `ccr.session.id` attribute to Claude Code records that lack it; the server marks those sessions as cloud sessions. `daemon/main_test.go` clears both variables so tests behave the same inside a cloud session +- **AICodeOtelServer**: gRPC OTEL collector for AI coding CLI metrics, logs and traces (Claude Code, Codex). Spans (`claude_code.interaction`, `llm_request`, `tool`, ...) go to the same `/api/v1/cc/otel` endpoint in `spans`; a span without `session.id` takes it from another span of its trace in the same export. `shelltime cc install` turns traces on (`CLAUDE_CODE_ENHANCED_TELEMETRY_BETA`, `OTEL_TRACES_EXPORTER=otlp`). In a Claude Code cloud session (`CLAUDE_CODE_REMOTE=true`, where the SessionStart hook starts the daemon) it adds the VM's `CLAUDE_CODE_REMOTE_SESSION_ID` as the `ccr.session.id` attribute to Claude Code records that lack it; the server marks those sessions as cloud sessions. `daemon/main_test.go` clears both variables so tests behave the same inside a cloud session - **HeartbeatResyncService**: Periodic resync of failed heartbeats (30-min interval) - **CleanupTimerService**: Periodic log file cleanup (24-hour interval) @@ -138,4 +138,4 @@ Releases use Release-Please (always-bump-patch) + Goreleaser. macOS builds inclu - Encryption requires daemon mode and a token with encryption capability - Shell hooks are platform-specific (bash, zsh, fish) - test on target shells - CC statusline quota display is macOS-only (requires Keychain access to Claude Code OAuth token) -- AICodeOtel feature enables AI coding CLI metrics/logs passthrough via gRPC (port 54027) - supports Claude Code, Codex, and other OTEL-compatible CLIs \ No newline at end of file +- AICodeOtel feature enables AI coding CLI metrics/logs/traces passthrough via gRPC (port 54027) - supports Claude Code, Codex, and other OTEL-compatible CLIs \ No newline at end of file diff --git a/commands/doctor_checks.go b/commands/doctor_checks.go index c381262..c51d944 100644 --- a/commands/doctor_checks.go +++ b/commands/doctor_checks.go @@ -768,7 +768,7 @@ func doctorCheckClaude(env *doctorEnv) []doctorResult { results = append(results, doctorResult{ID: "claude.otel", Status: doctorOK, Message: fmt.Sprintf("Claude Code reports usage to shelltime (%s).", svc.SettingsPath())}) // Settings written by an older `cc install` lack the newer keys (tool details, - // assistant responses, delta temporality, ...). + // assistant responses, delta temporality, traces, ...). if missing, err := svc.MissingManagedKeys(); err == nil && len(missing) > 0 { results = append(results, doctorResult{ ID: "claude.otel_keys", diff --git a/daemon/aicode_otel_processor.go b/daemon/aicode_otel_processor.go index 56fab13..7547c1d 100644 --- a/daemon/aicode_otel_processor.go +++ b/daemon/aicode_otel_processor.go @@ -22,10 +22,12 @@ import ( "github.com/malamtime/cli/model" collogsv1 "go.opentelemetry.io/proto/otlp/collector/logs/v1" collmetricsv1 "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + colltracev1 "go.opentelemetry.io/proto/otlp/collector/trace/v1" commonv1 "go.opentelemetry.io/proto/otlp/common/v1" logsv1 "go.opentelemetry.io/proto/otlp/logs/v1" metricsv1 "go.opentelemetry.io/proto/otlp/metrics/v1" resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/proto" ) @@ -257,6 +259,181 @@ func (p *AICodeOtelProcessor) ProcessLogs(ctx context.Context, req *collogsv1.Ex return &collogsv1.ExportLogsServiceResponse{}, nil } +// ProcessTraces receives OTEL spans and forwards them to the backend immediately +func (p *AICodeOtelProcessor) ProcessTraces(ctx context.Context, req *colltracev1.ExportTraceServiceRequest) (*colltracev1.ExportTraceServiceResponse, error) { + slog.Debug("AICodeOtel: Processing traces request", "resourceSpansCount", len(req.GetResourceSpans()), slog.Bool("debug", p.debug)) + + if p.debug { + p.writeDebugFile("aicode-otel-debug-traces.txt", req) + } + + for _, rs := range req.GetResourceSpans() { + resource := rs.GetResource() + + source := detectOtelSource(resource) + if source == "" { + slog.Debug("AICodeOtel: Skipping unknown resource") + continue + } + + attrs := extractResourceAttributes(resource) + var spans []model.AICodeOtelSpan + for _, ss := range rs.GetScopeSpans() { + for _, s := range ss.GetSpans() { + if span := parseSpan(s, source, attrs); span != nil { + spans = append(spans, *span) + } + } + } + if len(spans) == 0 { + continue + } + fillSpanSessionIDs(spans) + + p.send(ctx, &model.AICodeOtelRequest{ + Host: p.hostname, + Project: p.detectProject(resource, source), + Source: source, + Spans: spans, + }) + } + + return &colltracev1.ExportTraceServiceResponse{}, nil +} + +// spanIdentityAttributes are the account attributes Claude Code repeats on every span. They are +// the same for the whole session, so they are not forwarded with each span. +var spanIdentityAttributes = map[string]bool{ + "user.id": true, + "user.email": true, + "user.account_uuid": true, + "user.account_id": true, + "organization.id": true, +} + +// parseSpan converts an OTLP span into an AICodeOtelSpan. It returns nil for a span without +// valid trace and span ids. +func parseSpan(s *tracev1.Span, source string, res *model.AICodeOtelResourceAttributes) *model.AICodeOtelSpan { + traceID, spanID := otelID(s.GetTraceId(), 16), otelID(s.GetSpanId(), 8) + if traceID == "" || spanID == "" { + return nil + } + span := &model.AICodeOtelSpan{ + TraceID: traceID, + SpanID: spanID, + ParentSpanID: otelID(s.GetParentSpanId(), 8), + Name: model.CapAICodeText(s.GetName(), model.AICodeOtelMaxAttributeBytes), + Kind: spanKind(s.GetKind()), + StartTimeUnixNano: unixNano(s.GetStartTimeUnixNano()), + EndTimeUnixNano: unixNano(s.GetEndTimeUnixNano()), + StatusMessage: model.CapAICodeText(s.GetStatus().GetMessage(), model.AICodeOtelMaxAttributeBytes), + SessionID: firstNonEmpty(res.SessionID, res.ConversationID), + ClientType: source, + } + switch s.GetStatus().GetCode() { + case tracev1.Status_STATUS_CODE_OK: + span.StatusCode = "ok" + case tracev1.Status_STATUS_CODE_ERROR: + span.StatusCode = "error" + } + if span.EndTimeUnixNano < span.StartTimeUnixNano { + span.EndTimeUnixNano = span.StartTimeUnixNano + } + + var conversationID string + for _, attr := range s.GetAttributes() { + key := attr.GetKey() + switch { + // The session is kept in SessionID, not in the attributes. + case key == "session.id": + if id := anyString(attr.GetValue()); id != "" { + span.SessionID = id + } + case key == "conversation.id": + conversationID = anyString(attr.GetValue()) + case spanIdentityAttributes[key]: + // Dropped: the same for every span of the session. + default: + span.Attributes = setAttribute(span.Attributes, key, anyValueToGo(attr.GetValue())) + } + } + if span.SessionID == "" { + span.SessionID = conversationID + } + + for i, ev := range s.GetEvents() { + if i >= model.AICodeOtelMaxSpanEvents { + break + } + event := model.AICodeOtelSpanEvent{ + Name: model.CapAICodeText(ev.GetName(), model.AICodeOtelMaxAttributeBytes), + TimeUnixNano: unixNano(ev.GetTimeUnixNano()), + } + for _, attr := range ev.GetAttributes() { + event.Attributes = setAttribute(event.Attributes, attr.GetKey(), anyValueToGo(attr.GetValue())) + } + span.Events = append(span.Events, event) + } + + return span +} + +// fillSpanSessionIDs gives spans without a session id the one another span of the same trace +// carries. +func fillSpanSessionIDs(spans []model.AICodeOtelSpan) { + sessions := make(map[string]string) + for _, s := range spans { + if s.SessionID != "" { + if _, ok := sessions[s.TraceID]; !ok { + sessions[s.TraceID] = s.SessionID + } + } + } + for i := range spans { + if spans[i].SessionID == "" { + spans[i].SessionID = sessions[spans[i].TraceID] + } + } +} + +// otelID hex-encodes a trace or span id of the given byte length. A missing, malformed or +// all-zero id yields "". +func otelID(id []byte, size int) string { + if len(id) != size { + return "" + } + for _, b := range id { + if b != 0 { + return hex.EncodeToString(id) + } + } + return "" +} + +func spanKind(kind tracev1.Span_SpanKind) string { + switch kind { + case tracev1.Span_SPAN_KIND_INTERNAL: + return "internal" + case tracev1.Span_SPAN_KIND_SERVER: + return "server" + case tracev1.Span_SPAN_KIND_CLIENT: + return "client" + case tracev1.Span_SPAN_KIND_PRODUCER: + return "producer" + case tracev1.Span_SPAN_KIND_CONSUMER: + return "consumer" + } + return "" +} + +// unixNano converts an OTLP timestamp to int64, clamping values past the year 2262. +func unixNano(ts uint64) int64 { + if ts > math.MaxInt64 { + return math.MaxInt64 + } + return int64(ts) +} + // send forwards a request to the backend, split so that each body stays within maxRequestBytes. // Passthrough mode: failures are logged, not retried. func (p *AICodeOtelProcessor) send(ctx context.Context, req *model.AICodeOtelRequest) { @@ -267,10 +444,10 @@ func (p *AICodeOtelProcessor) send(ctx context.Context, req *model.AICodeOtelReq for _, chunk := range model.SplitAICodeOtelRequest(req, maxBytes) { resp, err := model.SendAICodeOtelData(ctx, chunk, p.endpoint) if err != nil { - slog.Error("AICodeOtel: Failed to send data to backend", "error", err, "events", len(chunk.Events), "metrics", len(chunk.Metrics)) + slog.Error("AICodeOtel: Failed to send data to backend", "error", err, "events", len(chunk.Events), "metrics", len(chunk.Metrics), "spans", len(chunk.Spans)) continue } - slog.Debug("AICodeOtel: Data sent to backend", "eventsProcessed", resp.EventsProcessed, "metricsProcessed", resp.MetricsProcessed) + slog.Debug("AICodeOtel: Data sent to backend", "eventsProcessed", resp.EventsProcessed, "metricsProcessed", resp.MetricsProcessed, "spansProcessed", resp.SpansProcessed) } } diff --git a/daemon/aicode_otel_server.go b/daemon/aicode_otel_server.go index 5db7cba..bd84715 100644 --- a/daemon/aicode_otel_server.go +++ b/daemon/aicode_otel_server.go @@ -8,6 +8,7 @@ import ( collogsv1 "go.opentelemetry.io/proto/otlp/collector/logs/v1" collmetricsv1 "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + colltracev1 "go.opentelemetry.io/proto/otlp/collector/trace/v1" "google.golang.org/grpc" ) @@ -45,6 +46,7 @@ func (s *AICodeOtelServer) Start() error { // Register OTEL collector services collmetricsv1.RegisterMetricsServiceServer(s.grpcServer, &metricsServiceServer{processor: s.processor}) collogsv1.RegisterLogsServiceServer(s.grpcServer, &logsServiceServer{processor: s.processor}) + colltracev1.RegisterTraceServiceServer(s.grpcServer, &traceServiceServer{processor: s.processor}) slog.Info("AICodeOtel gRPC server starting", "port", s.port) @@ -87,3 +89,14 @@ type logsServiceServer struct { func (s *logsServiceServer) Export(ctx context.Context, req *collogsv1.ExportLogsServiceRequest) (*collogsv1.ExportLogsServiceResponse, error) { return s.processor.ProcessLogs(ctx, req) } + +// traceServiceServer implements the OTEL TraceService +type traceServiceServer struct { + colltracev1.UnimplementedTraceServiceServer + processor *AICodeOtelProcessor +} + +// Export handles incoming trace export requests +func (s *traceServiceServer) Export(ctx context.Context, req *colltracev1.ExportTraceServiceRequest) (*colltracev1.ExportTraceServiceResponse, error) { + return s.processor.ProcessTraces(ctx, req) +} diff --git a/daemon/aicode_otel_traces_test.go b/daemon/aicode_otel_traces_test.go new file mode 100644 index 0000000..e238db8 --- /dev/null +++ b/daemon/aicode_otel_traces_test.go @@ -0,0 +1,273 @@ +package daemon + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/malamtime/cli/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + colltracev1 "go.opentelemetry.io/proto/otlp/collector/trace/v1" + commonv1 "go.opentelemetry.io/proto/otlp/common/v1" + resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" +) + +// Fixtures shaped after the Claude Code tracing reference (monitoring-usage.md, "Traces"): every +// span carries the standard attributes plus span.type, and tool spans have blocked_on_user and +// execution children. + +var ( + testTraceID = []byte{0x0a, 0xf7, 0x65, 0x19, 0x16, 0xcd, 0x43, 0xdd, 0x84, 0x48, 0xeb, 0x21, 0x1c, 0x80, 0x31, 0x9c} + testRootID = []byte{0xb7, 0xad, 0x6b, 0x71, 0x69, 0x20, 0x33, 0x31} + testLLMID = []byte{0x00, 0xf0, 0x67, 0xaa, 0x0b, 0xa9, 0x02, 0xb7} +) + +func claudeSpan(name string, spanID, parentID []byte, startOffset, duration time.Duration, attrs ...*commonv1.KeyValue) *tracev1.Span { + base := []*commonv1.KeyValue{ + kv("span.type", strVal(name)), + kv("session.id", strVal("sess-claude")), + kv("app.version", strVal("2.1.300")), + kv("organization.id", strVal("org-1")), + kv("user.account_uuid", strVal("acct-uuid")), + kv("user.id", strVal("anon-1")), + kv("user.email", strVal("dev@example.com")), + kv("terminal.type", strVal("iTerm.app")), + } + start := uint64(v2TestTimeNano) + uint64(startOffset) + return &tracev1.Span{ + TraceId: testTraceID, + SpanId: spanID, + ParentSpanId: parentID, + Name: name, + Kind: tracev1.Span_SPAN_KIND_INTERNAL, + StartTimeUnixNano: start, + EndTimeUnixNano: start + uint64(duration), + Attributes: append(base, attrs...), + } +} + +func parseTestSpan(t *testing.T, resource *resourcev1.Resource, s *tracev1.Span) *model.AICodeOtelSpan { + t.Helper() + source := detectOtelSource(resource) + require.NotEmpty(t, source) + return parseSpan(s, source, extractResourceAttributes(resource)) +} + +func TestParseSpan_ClaudeInteraction(t *testing.T) { + span := parseTestSpan(t, claudeTestResource(), claudeSpan("claude_code.interaction", testRootID, nil, 0, 4200*time.Millisecond, + kv("user_prompt", strVal("fix the failing test")), + kv("user_prompt_length", intVal(20)), + kv("interaction.sequence", intVal(3)), + kv("interaction.duration_ms", intVal(4200)), + )) + require.NotNil(t, span) + + assert.Equal(t, "0af7651916cd43dd8448eb211c80319c", span.TraceID) + assert.Equal(t, "b7ad6b7169203331", span.SpanID) + assert.Empty(t, span.ParentSpanID, "a root span has no parent") + assert.Equal(t, "claude_code.interaction", span.Name) + assert.Equal(t, "internal", span.Kind) + assert.Equal(t, int64(v2TestTimeNano), span.StartTimeUnixNano) + assert.Equal(t, int64(v2TestTimeNano)+int64(4200*time.Millisecond), span.EndTimeUnixNano) + assert.Empty(t, span.StatusCode, "unset status") + assert.Equal(t, "sess-claude", span.SessionID) + assert.Equal(t, model.AICodeOtelSourceClaudeCode, span.ClientType) + + assert.Equal(t, "fix the failing test", span.Attributes["user_prompt"]) + assert.Equal(t, int64(3), span.Attributes["interaction.sequence"]) + assert.Equal(t, "claude_code.interaction", span.Attributes["span.type"]) + assert.Equal(t, "2.1.300", span.Attributes["app.version"]) + assert.Equal(t, "iTerm.app", span.Attributes["terminal.type"]) + for _, key := range []string{"session.id", "user.email", "user.id", "user.account_uuid", "organization.id"} { + assert.NotContains(t, span.Attributes, key, "the session identity isn't repeated per span") + } +} + +func TestParseSpan_ClaudeFailedLLMRequestWithEvents(t *testing.T) { + s := claudeSpan("claude_code.llm_request", testLLMID, testRootID, 10*time.Millisecond, 900*time.Millisecond, + kv("model", strVal("claude-opus-5-5")), + kv("success", boolVal(false)), + kv("status_code", intVal(529)), + kv("error_class", strVal("server_overload")), + ) + s.Status = &tracev1.Status{Code: tracev1.Status_STATUS_CODE_ERROR, Message: "Overloaded"} + s.Kind = tracev1.Span_SPAN_KIND_CLIENT + s.Events = []*tracev1.Span_Event{ + {Name: "gen_ai.request.attempt", TimeUnixNano: uint64(v2TestTimeNano) + uint64(20*time.Millisecond), Attributes: []*commonv1.KeyValue{ + kv("attempt", intVal(2)), + kv("client_request_id", strVal("req-2")), + }}, + } + + span := parseTestSpan(t, claudeTestResource(), s) + require.NotNil(t, span) + assert.Equal(t, "b7ad6b7169203331", span.ParentSpanID) + assert.Equal(t, "client", span.Kind) + assert.Equal(t, "error", span.StatusCode) + assert.Equal(t, "Overloaded", span.StatusMessage) + assert.Equal(t, false, span.Attributes["success"]) + assert.Equal(t, int64(529), span.Attributes["status_code"]) + + require.Len(t, span.Events, 1) + assert.Equal(t, "gen_ai.request.attempt", span.Events[0].Name) + assert.Equal(t, int64(v2TestTimeNano)+int64(20*time.Millisecond), span.Events[0].TimeUnixNano) + assert.Equal(t, int64(2), span.Events[0].Attributes["attempt"]) + assert.Equal(t, "req-2", span.Events[0].Attributes["client_request_id"]) +} + +func TestParseSpan_InvalidIDsAndTimes(t *testing.T) { + res := claudeTestResource() + + noTrace := claudeSpan("claude_code.tool", testLLMID, nil, 0, time.Second) + noTrace.TraceId = make([]byte, 16) + assert.Nil(t, parseTestSpan(t, res, noTrace), "an all-zero trace id is invalid") + + shortSpan := claudeSpan("claude_code.tool", []byte{1, 2, 3}, nil, 0, time.Second) + assert.Nil(t, parseTestSpan(t, res, shortSpan), "a span id must be 8 bytes") + + zeroParent := claudeSpan("claude_code.tool", testLLMID, make([]byte, 8), 0, time.Second) + span := parseTestSpan(t, res, zeroParent) + require.NotNil(t, span) + assert.Empty(t, span.ParentSpanID, "an all-zero parent means no parent") + + backwards := claudeSpan("claude_code.tool", testLLMID, nil, time.Second, 0) + backwards.EndTimeUnixNano = backwards.StartTimeUnixNano - 1 + span = parseTestSpan(t, res, backwards) + require.NotNil(t, span) + assert.Equal(t, span.StartTimeUnixNano, span.EndTimeUnixNano, "an end before the start is clamped") +} + +func TestParseSpan_EventsAreCapped(t *testing.T) { + s := claudeSpan("claude_code.llm_request", testLLMID, testRootID, 0, time.Second) + for i := 0; i < model.AICodeOtelMaxSpanEvents+5; i++ { + s.Events = append(s.Events, &tracev1.Span_Event{Name: "gen_ai.request.attempt", TimeUnixNano: uint64(v2TestTimeNano)}) + } + span := parseTestSpan(t, claudeTestResource(), s) + require.NotNil(t, span) + assert.Len(t, span.Events, model.AICodeOtelMaxSpanEvents) +} + +func TestParseSpan_KindAndStatus(t *testing.T) { + testCases := []struct { + kind tracev1.Span_SpanKind + status tracev1.Status_StatusCode + wantKind string + wantStatus string + }{ + {tracev1.Span_SPAN_KIND_UNSPECIFIED, tracev1.Status_STATUS_CODE_UNSET, "", ""}, + {tracev1.Span_SPAN_KIND_INTERNAL, tracev1.Status_STATUS_CODE_OK, "internal", "ok"}, + {tracev1.Span_SPAN_KIND_SERVER, tracev1.Status_STATUS_CODE_ERROR, "server", "error"}, + {tracev1.Span_SPAN_KIND_CLIENT, tracev1.Status_STATUS_CODE_OK, "client", "ok"}, + {tracev1.Span_SPAN_KIND_PRODUCER, tracev1.Status_STATUS_CODE_UNSET, "producer", ""}, + {tracev1.Span_SPAN_KIND_CONSUMER, tracev1.Status_STATUS_CODE_UNSET, "consumer", ""}, + } + for _, tc := range testCases { + t.Run(tc.kind.String()+"/"+tc.status.String(), func(t *testing.T) { + s := claudeSpan("claude_code.tool", testLLMID, testRootID, 0, time.Second) + s.Kind = tc.kind + s.Status = &tracev1.Status{Code: tc.status} + span := parseTestSpan(t, claudeTestResource(), s) + require.NotNil(t, span) + assert.Equal(t, tc.wantKind, span.Kind) + assert.Equal(t, tc.wantStatus, span.StatusCode) + }) + } +} + +func TestParseSpan_NamesAreCapped(t *testing.T) { + long := strings.Repeat("n", model.AICodeOtelMaxAttributeBytes*3) + s := claudeSpan(long, testLLMID, testRootID, 0, time.Second) + s.Events = []*tracev1.Span_Event{{Name: long, TimeUnixNano: uint64(v2TestTimeNano)}} + + span := parseTestSpan(t, claudeTestResource(), s) + require.NotNil(t, span) + assert.Len(t, span.Name, model.AICodeOtelMaxAttributeBytes) + require.Len(t, span.Events, 1) + assert.Len(t, span.Events[0].Name, model.AICodeOtelMaxAttributeBytes) +} + +func TestParseSpan_CodexConversationID(t *testing.T) { + s := &tracev1.Span{ + TraceId: testTraceID, + SpanId: testRootID, + Name: "session_loop", + StartTimeUnixNano: uint64(v2TestTimeNano), + EndTimeUnixNano: uint64(v2TestTimeNano) + uint64(time.Second), + Attributes: []*commonv1.KeyValue{kv("conversation.id", strVal("conv-1"))}, + } + span := parseTestSpan(t, codexTestResource(), s) + require.NotNil(t, span) + assert.Equal(t, "conv-1", span.SessionID) + assert.Equal(t, model.AICodeOtelSourceCodex, span.ClientType) + assert.NotContains(t, span.Attributes, "conversation.id") +} + +func TestFillSpanSessionIDs(t *testing.T) { + spans := []model.AICodeOtelSpan{ + {TraceID: "t1", SpanID: "a"}, + {TraceID: "t1", SpanID: "b", SessionID: "sess-1"}, + {TraceID: "t2", SpanID: "c"}, + {TraceID: "t3", SpanID: "d", SessionID: "sess-3"}, + } + fillSpanSessionIDs(spans) + assert.Equal(t, "sess-1", spans[0].SessionID, "taken from a span of the same trace") + assert.Equal(t, "sess-1", spans[1].SessionID) + assert.Empty(t, spans[2].SessionID, "no span of the trace names a session") + assert.Equal(t, "sess-3", spans[3].SessionID) +} + +// TestAICodeOtelServer_TraceExportRoundTrip sends a Claude Code trace over gRPC and checks the +// JSON the backend receives. +func TestAICodeOtelServer_TraceExportRoundTrip(t *testing.T) { + backend := newRawBackend(t) + conn := startTestOtelServer(t, backend.server.URL) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + llm := claudeSpan("claude_code.llm_request", testLLMID, testRootID, 5*time.Millisecond, 800*time.Millisecond, + kv("model", strVal("claude-opus-5-5"))) + // Without its own session.id, it takes the interaction's. + attrs := llm.Attributes[:0] + for _, attr := range llm.Attributes { + if attr.GetKey() != "session.id" { + attrs = append(attrs, attr) + } + } + llm.Attributes = attrs + req := &colltracev1.ExportTraceServiceRequest{ResourceSpans: []*tracev1.ResourceSpans{ + { + Resource: claudeTestResource(), + ScopeSpans: []*tracev1.ScopeSpans{{Spans: []*tracev1.Span{ + claudeSpan("claude_code.interaction", testRootID, nil, 0, time.Second), + llm, + }}}, + }, + { + Resource: serviceResource("some-other-service"), + ScopeSpans: []*tracev1.ScopeSpans{{Spans: []*tracev1.Span{claudeSpan("other", testRootID, nil, 0, time.Second)}}}, + }, + }} + resp, err := colltracev1.NewTraceServiceClient(conn).Export(ctx, req) + require.NoError(t, err) + require.NotNil(t, resp) + + bodies := backend.captured() + require.Len(t, bodies, 1, "only the Claude Code resource is forwarded") + var got model.AICodeOtelRequest + require.NoError(t, json.Unmarshal([]byte(bodies[0]), &got)) + assert.Equal(t, model.AICodeOtelSourceClaudeCode, got.Source) + assert.Empty(t, got.Events) + assert.Empty(t, got.Metrics) + require.Len(t, got.Spans, 2) + assert.Equal(t, "claude_code.interaction", got.Spans[0].Name) + assert.Equal(t, "claude_code.llm_request", got.Spans[1].Name) + assert.Equal(t, "b7ad6b7169203331", got.Spans[1].ParentSpanID) + assert.Equal(t, "sess-claude", got.Spans[1].SessionID) + assert.Equal(t, "claude-opus-5-5", got.Spans[1].Attributes["model"]) + assert.Equal(t, int64(v2TestTimeNano)+int64(5*time.Millisecond), got.Spans[1].StartTimeUnixNano, "nanoseconds survive JSON") +} diff --git a/model/aicode_otel_claude_settings.go b/model/aicode_otel_claude_settings.go index b73edae..a392bbb 100644 --- a/model/aicode_otel_claude_settings.go +++ b/model/aicode_otel_claude_settings.go @@ -43,6 +43,9 @@ func claudeSettingsOtelEnvVars() []claudeSettingsEnvVar { {"CLAUDE_CODE_ENABLE_TELEMETRY", "1"}, {"OTEL_METRICS_EXPORTER", "otlp"}, {"OTEL_LOGS_EXPORTER", "otlp"}, + // Traces (claude_code.interaction, llm_request, tool, ... spans) need both of these. + {"CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", "1"}, + {"OTEL_TRACES_EXPORTER", "otlp"}, {"OTEL_EXPORTER_OTLP_PROTOCOL", "grpc"}, {"OTEL_EXPORTER_OTLP_ENDPOINT", AICodeOtelEndpoint}, {"OTEL_METRIC_EXPORT_INTERVAL", "10000"}, diff --git a/model/aicode_otel_claude_settings_test.go b/model/aicode_otel_claude_settings_test.go index 5982959..e6a0371 100644 --- a/model/aicode_otel_claude_settings_test.go +++ b/model/aicode_otel_claude_settings_test.go @@ -281,6 +281,8 @@ func TestClaudeSettingsOtel_InstallEnablesDetailsAndDeltaTemporality(t *testing. assert.Equal(t, "true", env["OTEL_METRICS_INCLUDE_ENTRYPOINT"]) assert.Equal(t, "true", env["OTEL_METRICS_INCLUDE_REPOSITORY"]) assert.Equal(t, "delta", env["OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE"]) + assert.Equal(t, "1", env["CLAUDE_CODE_ENHANCED_TELEMETRY_BETA"]) + assert.Equal(t, "otlp", env["OTEL_TRACES_EXPORTER"]) } func TestClaudeSettingsOtel_MissingManagedKeys(t *testing.T) { @@ -305,6 +307,8 @@ func TestClaudeSettingsOtel_MissingManagedKeys(t *testing.T) { missing, err = svc.MissingManagedKeys() require.NoError(t, err) assert.Equal(t, []string{ + "CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", + "OTEL_TRACES_EXPORTER", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_ASSISTANT_RESPONSES", "OTEL_METRICS_INCLUDE_ENTRYPOINT", diff --git a/model/aicode_otel_types.go b/model/aicode_otel_types.go index 99bd8cf..de57976 100644 --- a/model/aicode_otel_types.go +++ b/model/aicode_otel_types.go @@ -10,6 +10,7 @@ type AICodeOtelRequest struct { Source string `json:"source,omitempty"` // "claude-code" or "codex" - identifies the CLI source Events []AICodeOtelEvent `json:"events,omitempty"` Metrics []AICodeOtelMetric `json:"metrics,omitempty"` + Spans []AICodeOtelSpan `json:"spans,omitempty"` } // AICodeOtelResourceAttributes contains common resource-level attributes @@ -192,11 +193,45 @@ type AICodeOtelMetric struct { ClientType string `json:"clientType"` // claude_code, codex (defaults to claude_code) } +// AICodeOtelSpan is one finished span from Claude Code traces (OTEL_TRACES_EXPORTER), such as +// claude_code.interaction, claude_code.llm_request or claude_code.tool. +type AICodeOtelSpan struct { + TraceID string `json:"traceId"` // 32 hex characters + SpanID string `json:"spanId"` // 16 hex characters + ParentSpanID string `json:"parentSpanId,omitempty"` // empty for a root span + Name string `json:"name"` + Kind string `json:"kind,omitempty"` // internal, server, client, producer, consumer + + StartTimeUnixNano int64 `json:"startTimeUnixNano"` + EndTimeUnixNano int64 `json:"endTimeUnixNano"` + + StatusCode string `json:"statusCode,omitempty"` // "ok" or "error"; empty when unset + StatusMessage string `json:"statusMessage,omitempty"` + + // SessionID is the span's session.id (Codex: conversation.id). Spans of a trace that carry + // none take it from another span of the same trace in the export. + SessionID string `json:"sessionId,omitempty"` + + // Attributes holds the span's own attributes without the session identity ones. + Attributes map[string]any `json:"attributes,omitempty"` + Events []AICodeOtelSpanEvent `json:"events,omitempty"` + + ClientType string `json:"clientType"` // claude-code, codex +} + +// AICodeOtelSpanEvent is a timed event inside a span (gen_ai.request.attempt, tool.output). +type AICodeOtelSpanEvent struct { + Name string `json:"name"` + TimeUnixNano int64 `json:"timeUnixNano"` + Attributes map[string]any `json:"attributes,omitempty"` +} + // AICodeOtelResponse is the response from POST /api/v1/cc/otel type AICodeOtelResponse struct { Success bool `json:"success"` EventsProcessed int `json:"eventsProcessed"` MetricsProcessed int `json:"metricsProcessed"` + SpansProcessed int `json:"spansProcessed"` Message string `json:"message,omitempty"` } @@ -214,6 +249,8 @@ const ( AICodeOtelMaxAttributes = 64 // AICodeOtelMaxRequestBytes bounds the JSON body of one POST /api/v1/cc/otel request. AICodeOtelMaxRequestBytes = 8 << 20 + // AICodeOtelMaxSpanEvents caps the events kept per span. + AICodeOtelMaxSpanEvents = 32 ) // AICodeOtelDroppedEvents lists event names (without the claude_code./codex. prefix) that the diff --git a/model/api_aicode_otel.go b/model/api_aicode_otel.go index 3e8279c..9068536 100644 --- a/model/api_aicode_otel.go +++ b/model/api_aicode_otel.go @@ -32,21 +32,24 @@ func SendAICodeOtelData(ctx context.Context, req *AICodeOtelRequest, endpoint En } // SplitAICodeOtelRequest splits req into requests whose JSON bodies stay within maxBytes, keeping -// events and metrics in their original order. Like PackBackfillBatches it estimates each item by +// events, metrics and spans in their original order. Like PackBackfillBatches it estimates each item by // encoding it on its own; an item larger than the budget goes alone in its request. func SplitAICodeOtelRequest(req *AICodeOtelRequest, maxBytes int) []*AICodeOtelRequest { if req == nil { return nil } envelope := AICodeOtelRequest{Host: req.Host, Project: req.Project, Source: req.Source} - // `,"events":[]` and `,"metrics":[]` are not part of the empty envelope's encoding. - budget := maxBytes - aiCodeOtelJSONSize(envelope) - len(`,"events":[],"metrics":[]`) + // `,"events":[]`, `,"metrics":[]` and `,"spans":[]` are not part of the empty envelope's encoding. + budget := maxBytes - aiCodeOtelJSONSize(envelope) - len(`,"events":[],"metrics":[],"spans":[]`) var out []*AICodeOtelRequest cur := envelope curBytes := 0 + empty := func() bool { + return len(cur.Events) == 0 && len(cur.Metrics) == 0 && len(cur.Spans) == 0 + } flush := func() { - if len(cur.Events) > 0 || len(cur.Metrics) > 0 { + if !empty() { chunk := cur out = append(out, &chunk) } @@ -54,7 +57,7 @@ func SplitAICodeOtelRequest(req *AICodeOtelRequest, maxBytes int) []*AICodeOtelR curBytes = 0 } fits := func(size int) bool { - return (len(cur.Events) == 0 && len(cur.Metrics) == 0) || curBytes+size <= budget + return empty() || curBytes+size <= budget } for _, e := range req.Events { @@ -73,6 +76,14 @@ func SplitAICodeOtelRequest(req *AICodeOtelRequest, maxBytes int) []*AICodeOtelR cur.Metrics = append(cur.Metrics, m) curBytes += size } + for _, sp := range req.Spans { + size := aiCodeOtelJSONSize(sp) + 1 + if !fits(size) { + flush() + } + cur.Spans = append(cur.Spans, sp) + curBytes += size + } flush() return out } diff --git a/model/api_aicode_otel_test.go b/model/api_aicode_otel_test.go index ede6ed8..e6145db 100644 --- a/model/api_aicode_otel_test.go +++ b/model/api_aicode_otel_test.go @@ -54,6 +54,38 @@ func TestSplitAICodeOtelRequest(t *testing.T) { } } +func TestSplitAICodeOtelRequest_Spans(t *testing.T) { + req := &AICodeOtelRequest{Host: "host", Project: "/repo", Source: AICodeOtelSourceClaudeCode} + for i := 0; i < 40; i++ { + req.Spans = append(req.Spans, AICodeOtelSpan{ + TraceID: "0af7651916cd43dd8448eb211c80319c", + SpanID: fmt.Sprintf("%016x", i+1), + Name: "claude_code.tool", + Attributes: map[string]any{"full_command": strings.Repeat("x", 500)}, + }) + } + + const maxBytes = 4 << 10 + chunks := SplitAICodeOtelRequest(req, maxBytes) + require.Greater(t, len(chunks), 1) + + var spanIDs []string + for _, chunk := range chunks { + body, err := json.Marshal(chunk) + require.NoError(t, err) + assert.LessOrEqual(t, len(body), maxBytes, "each body stays within the limit") + assert.Empty(t, chunk.Events) + assert.Empty(t, chunk.Metrics) + for _, s := range chunk.Spans { + spanIDs = append(spanIDs, s.SpanID) + } + } + require.Len(t, spanIDs, 40) + for i, id := range spanIDs { + assert.Equal(t, fmt.Sprintf("%016x", i+1), id, "order is kept") + } +} + func TestSplitAICodeOtelRequest_SmallRequestIsOneChunk(t *testing.T) { req := &AICodeOtelRequest{Host: "h", Project: "p", Events: []AICodeOtelEvent{{EventID: "a"}, {EventID: "b"}}} chunks := SplitAICodeOtelRequest(req, AICodeOtelMaxRequestBytes)