From 702eee703f33ebfeda9a25ab00ba212673656e89 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 02:18:16 +0000 Subject: [PATCH 1/2] feat(daemon): receive Claude Code OTEL traces and forward spans The AICodeOtel gRPC server now registers the OTLP TraceService. Spans are parsed into the new `spans` array of POST /api/v1/cc/otel: trace, span and parent ids as hex, start/end in nanoseconds, kind, status, the session id (session.id, else conversation.id, else another span of the same trace in the export) and the span's own attributes and events, without the per-span account identity. Large batches are split like events and metrics. `shelltime cc install` now sets CLAUDE_CODE_ENHANCED_TELEMETRY_BETA=1 and OTEL_TRACES_EXPORTER=otlp, so `shelltime doctor` asks existing installs to run it again. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ebmd3PkNoem9ubKNVA9K2K --- CLAUDE.md | 4 +- commands/doctor_checks.go | 2 +- daemon/aicode_otel_processor.go | 180 ++++++++++++++++- daemon/aicode_otel_server.go | 13 ++ daemon/aicode_otel_traces_test.go | 233 ++++++++++++++++++++++ model/aicode_otel_claude_settings.go | 3 + model/aicode_otel_claude_settings_test.go | 4 + model/aicode_otel_types.go | 37 ++++ model/api_aicode_otel.go | 21 +- model/api_aicode_otel_test.go | 32 +++ 10 files changed, 519 insertions(+), 10 deletions(-) create mode 100644 daemon/aicode_otel_traces_test.go diff --git a/CLAUDE.md b/CLAUDE.md index 4fec922..10aab71 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,7 +94,7 @@ Optional daemon services (feature-gated via config): - **CCInfoTimerService**: Lazy-fetch background timer for Claude Code statusline data (cost, quota, git info), plus an always-on Anthropic usage sync (10-min interval, only while a `claude` process is running) that pushes quota to `/api/v1/anthropic-usage` without needing statusline activity - **CodexUsageSyncService**: Syncs Codex usage to `/api/v1/codex-usage` every 10 min, only while a `codex`/`chatgpt` process is running (both intervals share the `usageSyncInterval` const in `daemon/usage_sync.go`) - **SyncCircuitBreakerService**: Retry failed syncs with file-based persistence (`sync_pending.log`) and hourly recovery timer -- **AICodeOtelServer**: gRPC OTEL collector for AI coding CLI metrics/logs (Claude Code, Codex). In a Claude Code cloud session (`CLAUDE_CODE_REMOTE=true`, where the SessionStart hook starts the daemon) it adds the VM's `CLAUDE_CODE_REMOTE_SESSION_ID` as the `ccr.session.id` attribute to Claude Code records that lack it; the server marks those sessions as cloud sessions. `daemon/main_test.go` clears both variables so tests behave the same inside a cloud session +- **AICodeOtelServer**: gRPC OTEL collector for AI coding CLI metrics, logs and traces (Claude Code, Codex). Spans (`claude_code.interaction`, `llm_request`, `tool`, ...) go to the same `/api/v1/cc/otel` endpoint in `spans`; a span without `session.id` takes it from another span of its trace in the same export. `shelltime cc install` turns traces on (`CLAUDE_CODE_ENHANCED_TELEMETRY_BETA`, `OTEL_TRACES_EXPORTER=otlp`). In a Claude Code cloud session (`CLAUDE_CODE_REMOTE=true`, where the SessionStart hook starts the daemon) it adds the VM's `CLAUDE_CODE_REMOTE_SESSION_ID` as the `ccr.session.id` attribute to Claude Code records that lack it; the server marks those sessions as cloud sessions. `daemon/main_test.go` clears both variables so tests behave the same inside a cloud session - **HeartbeatResyncService**: Periodic resync of failed heartbeats (30-min interval) - **CleanupTimerService**: Periodic log file cleanup (24-hour interval) @@ -138,4 +138,4 @@ Releases use Release-Please (always-bump-patch) + Goreleaser. macOS builds inclu - Encryption requires daemon mode and a token with encryption capability - Shell hooks are platform-specific (bash, zsh, fish) - test on target shells - CC statusline quota display is macOS-only (requires Keychain access to Claude Code OAuth token) -- AICodeOtel feature enables AI coding CLI metrics/logs passthrough via gRPC (port 54027) - supports Claude Code, Codex, and other OTEL-compatible CLIs \ No newline at end of file +- AICodeOtel feature enables AI coding CLI metrics/logs/traces passthrough via gRPC (port 54027) - supports Claude Code, Codex, and other OTEL-compatible CLIs \ No newline at end of file diff --git a/commands/doctor_checks.go b/commands/doctor_checks.go index c381262..c51d944 100644 --- a/commands/doctor_checks.go +++ b/commands/doctor_checks.go @@ -768,7 +768,7 @@ func doctorCheckClaude(env *doctorEnv) []doctorResult { results = append(results, doctorResult{ID: "claude.otel", Status: doctorOK, Message: fmt.Sprintf("Claude Code reports usage to shelltime (%s).", svc.SettingsPath())}) // Settings written by an older `cc install` lack the newer keys (tool details, - // assistant responses, delta temporality, ...). + // assistant responses, delta temporality, traces, ...). if missing, err := svc.MissingManagedKeys(); err == nil && len(missing) > 0 { results = append(results, doctorResult{ ID: "claude.otel_keys", diff --git a/daemon/aicode_otel_processor.go b/daemon/aicode_otel_processor.go index 56fab13..e747a02 100644 --- a/daemon/aicode_otel_processor.go +++ b/daemon/aicode_otel_processor.go @@ -22,10 +22,12 @@ import ( "github.com/malamtime/cli/model" collogsv1 "go.opentelemetry.io/proto/otlp/collector/logs/v1" collmetricsv1 "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + colltracev1 "go.opentelemetry.io/proto/otlp/collector/trace/v1" commonv1 "go.opentelemetry.io/proto/otlp/common/v1" logsv1 "go.opentelemetry.io/proto/otlp/logs/v1" metricsv1 "go.opentelemetry.io/proto/otlp/metrics/v1" resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" "google.golang.org/protobuf/proto" ) @@ -257,6 +259,180 @@ func (p *AICodeOtelProcessor) ProcessLogs(ctx context.Context, req *collogsv1.Ex return &collogsv1.ExportLogsServiceResponse{}, nil } +// ProcessTraces receives OTEL spans and forwards them to the backend immediately +func (p *AICodeOtelProcessor) ProcessTraces(ctx context.Context, req *colltracev1.ExportTraceServiceRequest) (*colltracev1.ExportTraceServiceResponse, error) { + slog.Debug("AICodeOtel: Processing traces request", "resourceSpansCount", len(req.GetResourceSpans()), slog.Bool("debug", p.debug)) + + if p.debug { + p.writeDebugFile("aicode-otel-debug-traces.txt", req) + } + + for _, rs := range req.GetResourceSpans() { + resource := rs.GetResource() + + source := detectOtelSource(resource) + if source == "" { + slog.Debug("AICodeOtel: Skipping unknown resource") + continue + } + + attrs := extractResourceAttributes(resource) + var spans []model.AICodeOtelSpan + for _, ss := range rs.GetScopeSpans() { + for _, s := range ss.GetSpans() { + if span := parseSpan(s, source, attrs); span != nil { + spans = append(spans, *span) + } + } + } + if len(spans) == 0 { + continue + } + fillSpanSessionIDs(spans) + + p.send(ctx, &model.AICodeOtelRequest{ + Host: p.hostname, + Project: p.detectProject(resource, source), + Source: source, + Spans: spans, + }) + } + + return &colltracev1.ExportTraceServiceResponse{}, nil +} + +// spanIdentityAttributes are the account attributes Claude Code repeats on every span. They are +// the same for the whole session, so they are not forwarded with each span. +var spanIdentityAttributes = map[string]bool{ + "user.id": true, + "user.email": true, + "user.account_uuid": true, + "user.account_id": true, + "organization.id": true, +} + +// parseSpan converts an OTLP span into an AICodeOtelSpan. It returns nil for a span without +// valid trace and span ids. +func parseSpan(s *tracev1.Span, source string, res *model.AICodeOtelResourceAttributes) *model.AICodeOtelSpan { + traceID, spanID := otelID(s.GetTraceId(), 16), otelID(s.GetSpanId(), 8) + if traceID == "" || spanID == "" { + return nil + } + span := &model.AICodeOtelSpan{ + TraceID: traceID, + SpanID: spanID, + ParentSpanID: otelID(s.GetParentSpanId(), 8), + Name: s.GetName(), + Kind: spanKind(s.GetKind()), + StartTimeUnixNano: unixNano(s.GetStartTimeUnixNano()), + EndTimeUnixNano: unixNano(s.GetEndTimeUnixNano()), + StatusMessage: model.CapAICodeText(s.GetStatus().GetMessage(), model.AICodeOtelMaxAttributeBytes), + SessionID: firstNonEmpty(res.SessionID, res.ConversationID), + ClientType: source, + } + switch s.GetStatus().GetCode() { + case tracev1.Status_STATUS_CODE_OK: + span.StatusCode = "ok" + case tracev1.Status_STATUS_CODE_ERROR: + span.StatusCode = "error" + } + if span.EndTimeUnixNano < span.StartTimeUnixNano { + span.EndTimeUnixNano = span.StartTimeUnixNano + } + + var conversationID string + for _, attr := range s.GetAttributes() { + key := attr.GetKey() + switch { + // The session is kept in SessionID, not in the attributes. + case key == "session.id": + if id := anyString(attr.GetValue()); id != "" { + span.SessionID = id + } + case key == "conversation.id": + conversationID = anyString(attr.GetValue()) + case spanIdentityAttributes[key]: + default: + span.Attributes = setAttribute(span.Attributes, key, anyValueToGo(attr.GetValue())) + } + } + if span.SessionID == "" { + span.SessionID = conversationID + } + + for i, ev := range s.GetEvents() { + if i >= model.AICodeOtelMaxSpanEvents { + break + } + event := model.AICodeOtelSpanEvent{ + Name: ev.GetName(), + TimeUnixNano: unixNano(ev.GetTimeUnixNano()), + } + for _, attr := range ev.GetAttributes() { + event.Attributes = setAttribute(event.Attributes, attr.GetKey(), anyValueToGo(attr.GetValue())) + } + span.Events = append(span.Events, event) + } + + return span +} + +// fillSpanSessionIDs gives spans without a session id the one another span of the same trace +// carries. +func fillSpanSessionIDs(spans []model.AICodeOtelSpan) { + sessions := make(map[string]string) + for _, s := range spans { + if s.SessionID != "" { + if _, ok := sessions[s.TraceID]; !ok { + sessions[s.TraceID] = s.SessionID + } + } + } + for i := range spans { + if spans[i].SessionID == "" { + spans[i].SessionID = sessions[spans[i].TraceID] + } + } +} + +// otelID hex-encodes a trace or span id of the given byte length. A missing, malformed or +// all-zero id yields "". +func otelID(id []byte, size int) string { + if len(id) != size { + return "" + } + for _, b := range id { + if b != 0 { + return hex.EncodeToString(id) + } + } + return "" +} + +func spanKind(kind tracev1.Span_SpanKind) string { + switch kind { + case tracev1.Span_SPAN_KIND_INTERNAL: + return "internal" + case tracev1.Span_SPAN_KIND_SERVER: + return "server" + case tracev1.Span_SPAN_KIND_CLIENT: + return "client" + case tracev1.Span_SPAN_KIND_PRODUCER: + return "producer" + case tracev1.Span_SPAN_KIND_CONSUMER: + return "consumer" + } + return "" +} + +// unixNano converts an OTLP timestamp to int64, clamping values past the year 2262. +func unixNano(ts uint64) int64 { + if ts > math.MaxInt64 { + return math.MaxInt64 + } + return int64(ts) +} + // send forwards a request to the backend, split so that each body stays within maxRequestBytes. // Passthrough mode: failures are logged, not retried. func (p *AICodeOtelProcessor) send(ctx context.Context, req *model.AICodeOtelRequest) { @@ -267,10 +443,10 @@ func (p *AICodeOtelProcessor) send(ctx context.Context, req *model.AICodeOtelReq for _, chunk := range model.SplitAICodeOtelRequest(req, maxBytes) { resp, err := model.SendAICodeOtelData(ctx, chunk, p.endpoint) if err != nil { - slog.Error("AICodeOtel: Failed to send data to backend", "error", err, "events", len(chunk.Events), "metrics", len(chunk.Metrics)) + slog.Error("AICodeOtel: Failed to send data to backend", "error", err, "events", len(chunk.Events), "metrics", len(chunk.Metrics), "spans", len(chunk.Spans)) continue } - slog.Debug("AICodeOtel: Data sent to backend", "eventsProcessed", resp.EventsProcessed, "metricsProcessed", resp.MetricsProcessed) + slog.Debug("AICodeOtel: Data sent to backend", "eventsProcessed", resp.EventsProcessed, "metricsProcessed", resp.MetricsProcessed, "spansProcessed", resp.SpansProcessed) } } diff --git a/daemon/aicode_otel_server.go b/daemon/aicode_otel_server.go index 5db7cba..bd84715 100644 --- a/daemon/aicode_otel_server.go +++ b/daemon/aicode_otel_server.go @@ -8,6 +8,7 @@ import ( collogsv1 "go.opentelemetry.io/proto/otlp/collector/logs/v1" collmetricsv1 "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + colltracev1 "go.opentelemetry.io/proto/otlp/collector/trace/v1" "google.golang.org/grpc" ) @@ -45,6 +46,7 @@ func (s *AICodeOtelServer) Start() error { // Register OTEL collector services collmetricsv1.RegisterMetricsServiceServer(s.grpcServer, &metricsServiceServer{processor: s.processor}) collogsv1.RegisterLogsServiceServer(s.grpcServer, &logsServiceServer{processor: s.processor}) + colltracev1.RegisterTraceServiceServer(s.grpcServer, &traceServiceServer{processor: s.processor}) slog.Info("AICodeOtel gRPC server starting", "port", s.port) @@ -87,3 +89,14 @@ type logsServiceServer struct { func (s *logsServiceServer) Export(ctx context.Context, req *collogsv1.ExportLogsServiceRequest) (*collogsv1.ExportLogsServiceResponse, error) { return s.processor.ProcessLogs(ctx, req) } + +// traceServiceServer implements the OTEL TraceService +type traceServiceServer struct { + colltracev1.UnimplementedTraceServiceServer + processor *AICodeOtelProcessor +} + +// Export handles incoming trace export requests +func (s *traceServiceServer) Export(ctx context.Context, req *colltracev1.ExportTraceServiceRequest) (*colltracev1.ExportTraceServiceResponse, error) { + return s.processor.ProcessTraces(ctx, req) +} diff --git a/daemon/aicode_otel_traces_test.go b/daemon/aicode_otel_traces_test.go new file mode 100644 index 0000000..4d400ae --- /dev/null +++ b/daemon/aicode_otel_traces_test.go @@ -0,0 +1,233 @@ +package daemon + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/malamtime/cli/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + colltracev1 "go.opentelemetry.io/proto/otlp/collector/trace/v1" + commonv1 "go.opentelemetry.io/proto/otlp/common/v1" + resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" +) + +// Fixtures shaped after the Claude Code tracing reference (monitoring-usage.md, "Traces"): every +// span carries the standard attributes plus span.type, and tool spans have blocked_on_user and +// execution children. + +var ( + testTraceID = []byte{0x0a, 0xf7, 0x65, 0x19, 0x16, 0xcd, 0x43, 0xdd, 0x84, 0x48, 0xeb, 0x21, 0x1c, 0x80, 0x31, 0x9c} + testRootID = []byte{0xb7, 0xad, 0x6b, 0x71, 0x69, 0x20, 0x33, 0x31} + testLLMID = []byte{0x00, 0xf0, 0x67, 0xaa, 0x0b, 0xa9, 0x02, 0xb7} +) + +func claudeSpan(name string, spanID, parentID []byte, startOffset, duration time.Duration, attrs ...*commonv1.KeyValue) *tracev1.Span { + base := []*commonv1.KeyValue{ + kv("span.type", strVal(name)), + kv("session.id", strVal("sess-claude")), + kv("app.version", strVal("2.1.300")), + kv("organization.id", strVal("org-1")), + kv("user.account_uuid", strVal("acct-uuid")), + kv("user.id", strVal("anon-1")), + kv("user.email", strVal("dev@example.com")), + kv("terminal.type", strVal("iTerm.app")), + } + start := uint64(v2TestTimeNano) + uint64(startOffset) + return &tracev1.Span{ + TraceId: testTraceID, + SpanId: spanID, + ParentSpanId: parentID, + Name: name, + Kind: tracev1.Span_SPAN_KIND_INTERNAL, + StartTimeUnixNano: start, + EndTimeUnixNano: start + uint64(duration), + Attributes: append(base, attrs...), + } +} + +func parseTestSpan(t *testing.T, resource *resourcev1.Resource, s *tracev1.Span) *model.AICodeOtelSpan { + t.Helper() + source := detectOtelSource(resource) + require.NotEmpty(t, source) + return parseSpan(s, source, extractResourceAttributes(resource)) +} + +func TestParseSpan_ClaudeInteraction(t *testing.T) { + span := parseTestSpan(t, claudeTestResource(), claudeSpan("claude_code.interaction", testRootID, nil, 0, 4200*time.Millisecond, + kv("user_prompt", strVal("fix the failing test")), + kv("user_prompt_length", intVal(20)), + kv("interaction.sequence", intVal(3)), + kv("interaction.duration_ms", intVal(4200)), + )) + require.NotNil(t, span) + + assert.Equal(t, "0af7651916cd43dd8448eb211c80319c", span.TraceID) + assert.Equal(t, "b7ad6b7169203331", span.SpanID) + assert.Empty(t, span.ParentSpanID, "a root span has no parent") + assert.Equal(t, "claude_code.interaction", span.Name) + assert.Equal(t, "internal", span.Kind) + assert.Equal(t, int64(v2TestTimeNano), span.StartTimeUnixNano) + assert.Equal(t, int64(v2TestTimeNano)+int64(4200*time.Millisecond), span.EndTimeUnixNano) + assert.Empty(t, span.StatusCode, "unset status") + assert.Equal(t, "sess-claude", span.SessionID) + assert.Equal(t, model.AICodeOtelSourceClaudeCode, span.ClientType) + + assert.Equal(t, "fix the failing test", span.Attributes["user_prompt"]) + assert.Equal(t, int64(3), span.Attributes["interaction.sequence"]) + assert.Equal(t, "claude_code.interaction", span.Attributes["span.type"]) + assert.Equal(t, "2.1.300", span.Attributes["app.version"]) + assert.Equal(t, "iTerm.app", span.Attributes["terminal.type"]) + for _, key := range []string{"session.id", "user.email", "user.id", "user.account_uuid", "organization.id"} { + assert.NotContains(t, span.Attributes, key, "the session identity isn't repeated per span") + } +} + +func TestParseSpan_ClaudeFailedLLMRequestWithEvents(t *testing.T) { + s := claudeSpan("claude_code.llm_request", testLLMID, testRootID, 10*time.Millisecond, 900*time.Millisecond, + kv("model", strVal("claude-opus-5-5")), + kv("success", boolVal(false)), + kv("status_code", intVal(529)), + kv("error_class", strVal("server_overload")), + ) + s.Status = &tracev1.Status{Code: tracev1.Status_STATUS_CODE_ERROR, Message: "Overloaded"} + s.Kind = tracev1.Span_SPAN_KIND_CLIENT + s.Events = []*tracev1.Span_Event{ + {Name: "gen_ai.request.attempt", TimeUnixNano: uint64(v2TestTimeNano) + uint64(20*time.Millisecond), Attributes: []*commonv1.KeyValue{ + kv("attempt", intVal(2)), + kv("client_request_id", strVal("req-2")), + }}, + } + + span := parseTestSpan(t, claudeTestResource(), s) + require.NotNil(t, span) + assert.Equal(t, "b7ad6b7169203331", span.ParentSpanID) + assert.Equal(t, "client", span.Kind) + assert.Equal(t, "error", span.StatusCode) + assert.Equal(t, "Overloaded", span.StatusMessage) + assert.Equal(t, false, span.Attributes["success"]) + assert.Equal(t, int64(529), span.Attributes["status_code"]) + + require.Len(t, span.Events, 1) + assert.Equal(t, "gen_ai.request.attempt", span.Events[0].Name) + assert.Equal(t, int64(v2TestTimeNano)+int64(20*time.Millisecond), span.Events[0].TimeUnixNano) + assert.Equal(t, int64(2), span.Events[0].Attributes["attempt"]) + assert.Equal(t, "req-2", span.Events[0].Attributes["client_request_id"]) +} + +func TestParseSpan_InvalidIDsAndTimes(t *testing.T) { + res := claudeTestResource() + + noTrace := claudeSpan("claude_code.tool", testLLMID, nil, 0, time.Second) + noTrace.TraceId = make([]byte, 16) + assert.Nil(t, parseTestSpan(t, res, noTrace), "an all-zero trace id is invalid") + + shortSpan := claudeSpan("claude_code.tool", []byte{1, 2, 3}, nil, 0, time.Second) + assert.Nil(t, parseTestSpan(t, res, shortSpan), "a span id must be 8 bytes") + + zeroParent := claudeSpan("claude_code.tool", testLLMID, make([]byte, 8), 0, time.Second) + span := parseTestSpan(t, res, zeroParent) + require.NotNil(t, span) + assert.Empty(t, span.ParentSpanID, "an all-zero parent means no parent") + + backwards := claudeSpan("claude_code.tool", testLLMID, nil, time.Second, 0) + backwards.EndTimeUnixNano = backwards.StartTimeUnixNano - 1 + span = parseTestSpan(t, res, backwards) + require.NotNil(t, span) + assert.Equal(t, span.StartTimeUnixNano, span.EndTimeUnixNano, "an end before the start is clamped") +} + +func TestParseSpan_EventsAreCapped(t *testing.T) { + s := claudeSpan("claude_code.llm_request", testLLMID, testRootID, 0, time.Second) + for i := 0; i < model.AICodeOtelMaxSpanEvents+5; i++ { + s.Events = append(s.Events, &tracev1.Span_Event{Name: "gen_ai.request.attempt", TimeUnixNano: uint64(v2TestTimeNano)}) + } + span := parseTestSpan(t, claudeTestResource(), s) + require.NotNil(t, span) + assert.Len(t, span.Events, model.AICodeOtelMaxSpanEvents) +} + +func TestParseSpan_CodexConversationID(t *testing.T) { + s := &tracev1.Span{ + TraceId: testTraceID, + SpanId: testRootID, + Name: "session_loop", + StartTimeUnixNano: uint64(v2TestTimeNano), + EndTimeUnixNano: uint64(v2TestTimeNano) + uint64(time.Second), + Attributes: []*commonv1.KeyValue{kv("conversation.id", strVal("conv-1"))}, + } + span := parseTestSpan(t, codexTestResource(), s) + require.NotNil(t, span) + assert.Equal(t, "conv-1", span.SessionID) + assert.Equal(t, model.AICodeOtelSourceCodex, span.ClientType) + assert.NotContains(t, span.Attributes, "conversation.id") +} + +func TestFillSpanSessionIDs(t *testing.T) { + spans := []model.AICodeOtelSpan{ + {TraceID: "t1", SpanID: "a"}, + {TraceID: "t1", SpanID: "b", SessionID: "sess-1"}, + {TraceID: "t2", SpanID: "c"}, + {TraceID: "t3", SpanID: "d", SessionID: "sess-3"}, + } + fillSpanSessionIDs(spans) + assert.Equal(t, "sess-1", spans[0].SessionID, "taken from a span of the same trace") + assert.Equal(t, "sess-1", spans[1].SessionID) + assert.Empty(t, spans[2].SessionID, "no span of the trace names a session") + assert.Equal(t, "sess-3", spans[3].SessionID) +} + +// TestAICodeOtelServer_TraceExportRoundTrip sends a Claude Code trace over gRPC and checks the +// JSON the backend receives. +func TestAICodeOtelServer_TraceExportRoundTrip(t *testing.T) { + backend := newRawBackend(t) + conn := startTestOtelServer(t, backend.server.URL) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + llm := claudeSpan("claude_code.llm_request", testLLMID, testRootID, 5*time.Millisecond, 800*time.Millisecond, + kv("model", strVal("claude-opus-5-5"))) + // Without its own session.id, it takes the interaction's. + attrs := llm.Attributes[:0] + for _, attr := range llm.Attributes { + if attr.GetKey() != "session.id" { + attrs = append(attrs, attr) + } + } + llm.Attributes = attrs + req := &colltracev1.ExportTraceServiceRequest{ResourceSpans: []*tracev1.ResourceSpans{ + { + Resource: claudeTestResource(), + ScopeSpans: []*tracev1.ScopeSpans{{Spans: []*tracev1.Span{ + claudeSpan("claude_code.interaction", testRootID, nil, 0, time.Second), + llm, + }}}, + }, + { + Resource: serviceResource("some-other-service"), + ScopeSpans: []*tracev1.ScopeSpans{{Spans: []*tracev1.Span{claudeSpan("other", testRootID, nil, 0, time.Second)}}}, + }, + }} + resp, err := colltracev1.NewTraceServiceClient(conn).Export(ctx, req) + require.NoError(t, err) + require.NotNil(t, resp) + + bodies := backend.captured() + require.Len(t, bodies, 1, "only the Claude Code resource is forwarded") + var got model.AICodeOtelRequest + require.NoError(t, json.Unmarshal([]byte(bodies[0]), &got)) + assert.Equal(t, model.AICodeOtelSourceClaudeCode, got.Source) + assert.Empty(t, got.Events) + assert.Empty(t, got.Metrics) + require.Len(t, got.Spans, 2) + assert.Equal(t, "claude_code.interaction", got.Spans[0].Name) + assert.Equal(t, "claude_code.llm_request", got.Spans[1].Name) + assert.Equal(t, "b7ad6b7169203331", got.Spans[1].ParentSpanID) + assert.Equal(t, "sess-claude", got.Spans[1].SessionID) + assert.Equal(t, "claude-opus-5-5", got.Spans[1].Attributes["model"]) + assert.Equal(t, int64(v2TestTimeNano)+int64(5*time.Millisecond), got.Spans[1].StartTimeUnixNano, "nanoseconds survive JSON") +} diff --git a/model/aicode_otel_claude_settings.go b/model/aicode_otel_claude_settings.go index b73edae..a392bbb 100644 --- a/model/aicode_otel_claude_settings.go +++ b/model/aicode_otel_claude_settings.go @@ -43,6 +43,9 @@ func claudeSettingsOtelEnvVars() []claudeSettingsEnvVar { {"CLAUDE_CODE_ENABLE_TELEMETRY", "1"}, {"OTEL_METRICS_EXPORTER", "otlp"}, {"OTEL_LOGS_EXPORTER", "otlp"}, + // Traces (claude_code.interaction, llm_request, tool, ... spans) need both of these. + {"CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", "1"}, + {"OTEL_TRACES_EXPORTER", "otlp"}, {"OTEL_EXPORTER_OTLP_PROTOCOL", "grpc"}, {"OTEL_EXPORTER_OTLP_ENDPOINT", AICodeOtelEndpoint}, {"OTEL_METRIC_EXPORT_INTERVAL", "10000"}, diff --git a/model/aicode_otel_claude_settings_test.go b/model/aicode_otel_claude_settings_test.go index 5982959..e6a0371 100644 --- a/model/aicode_otel_claude_settings_test.go +++ b/model/aicode_otel_claude_settings_test.go @@ -281,6 +281,8 @@ func TestClaudeSettingsOtel_InstallEnablesDetailsAndDeltaTemporality(t *testing. assert.Equal(t, "true", env["OTEL_METRICS_INCLUDE_ENTRYPOINT"]) assert.Equal(t, "true", env["OTEL_METRICS_INCLUDE_REPOSITORY"]) assert.Equal(t, "delta", env["OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE"]) + assert.Equal(t, "1", env["CLAUDE_CODE_ENHANCED_TELEMETRY_BETA"]) + assert.Equal(t, "otlp", env["OTEL_TRACES_EXPORTER"]) } func TestClaudeSettingsOtel_MissingManagedKeys(t *testing.T) { @@ -305,6 +307,8 @@ func TestClaudeSettingsOtel_MissingManagedKeys(t *testing.T) { missing, err = svc.MissingManagedKeys() require.NoError(t, err) assert.Equal(t, []string{ + "CLAUDE_CODE_ENHANCED_TELEMETRY_BETA", + "OTEL_TRACES_EXPORTER", "OTEL_LOG_TOOL_DETAILS", "OTEL_LOG_ASSISTANT_RESPONSES", "OTEL_METRICS_INCLUDE_ENTRYPOINT", diff --git a/model/aicode_otel_types.go b/model/aicode_otel_types.go index 99bd8cf..de57976 100644 --- a/model/aicode_otel_types.go +++ b/model/aicode_otel_types.go @@ -10,6 +10,7 @@ type AICodeOtelRequest struct { Source string `json:"source,omitempty"` // "claude-code" or "codex" - identifies the CLI source Events []AICodeOtelEvent `json:"events,omitempty"` Metrics []AICodeOtelMetric `json:"metrics,omitempty"` + Spans []AICodeOtelSpan `json:"spans,omitempty"` } // AICodeOtelResourceAttributes contains common resource-level attributes @@ -192,11 +193,45 @@ type AICodeOtelMetric struct { ClientType string `json:"clientType"` // claude_code, codex (defaults to claude_code) } +// AICodeOtelSpan is one finished span from Claude Code traces (OTEL_TRACES_EXPORTER), such as +// claude_code.interaction, claude_code.llm_request or claude_code.tool. +type AICodeOtelSpan struct { + TraceID string `json:"traceId"` // 32 hex characters + SpanID string `json:"spanId"` // 16 hex characters + ParentSpanID string `json:"parentSpanId,omitempty"` // empty for a root span + Name string `json:"name"` + Kind string `json:"kind,omitempty"` // internal, server, client, producer, consumer + + StartTimeUnixNano int64 `json:"startTimeUnixNano"` + EndTimeUnixNano int64 `json:"endTimeUnixNano"` + + StatusCode string `json:"statusCode,omitempty"` // "ok" or "error"; empty when unset + StatusMessage string `json:"statusMessage,omitempty"` + + // SessionID is the span's session.id (Codex: conversation.id). Spans of a trace that carry + // none take it from another span of the same trace in the export. + SessionID string `json:"sessionId,omitempty"` + + // Attributes holds the span's own attributes without the session identity ones. + Attributes map[string]any `json:"attributes,omitempty"` + Events []AICodeOtelSpanEvent `json:"events,omitempty"` + + ClientType string `json:"clientType"` // claude-code, codex +} + +// AICodeOtelSpanEvent is a timed event inside a span (gen_ai.request.attempt, tool.output). +type AICodeOtelSpanEvent struct { + Name string `json:"name"` + TimeUnixNano int64 `json:"timeUnixNano"` + Attributes map[string]any `json:"attributes,omitempty"` +} + // AICodeOtelResponse is the response from POST /api/v1/cc/otel type AICodeOtelResponse struct { Success bool `json:"success"` EventsProcessed int `json:"eventsProcessed"` MetricsProcessed int `json:"metricsProcessed"` + SpansProcessed int `json:"spansProcessed"` Message string `json:"message,omitempty"` } @@ -214,6 +249,8 @@ const ( AICodeOtelMaxAttributes = 64 // AICodeOtelMaxRequestBytes bounds the JSON body of one POST /api/v1/cc/otel request. AICodeOtelMaxRequestBytes = 8 << 20 + // AICodeOtelMaxSpanEvents caps the events kept per span. + AICodeOtelMaxSpanEvents = 32 ) // AICodeOtelDroppedEvents lists event names (without the claude_code./codex. prefix) that the diff --git a/model/api_aicode_otel.go b/model/api_aicode_otel.go index 3e8279c..9068536 100644 --- a/model/api_aicode_otel.go +++ b/model/api_aicode_otel.go @@ -32,21 +32,24 @@ func SendAICodeOtelData(ctx context.Context, req *AICodeOtelRequest, endpoint En } // SplitAICodeOtelRequest splits req into requests whose JSON bodies stay within maxBytes, keeping -// events and metrics in their original order. Like PackBackfillBatches it estimates each item by +// events, metrics and spans in their original order. Like PackBackfillBatches it estimates each item by // encoding it on its own; an item larger than the budget goes alone in its request. func SplitAICodeOtelRequest(req *AICodeOtelRequest, maxBytes int) []*AICodeOtelRequest { if req == nil { return nil } envelope := AICodeOtelRequest{Host: req.Host, Project: req.Project, Source: req.Source} - // `,"events":[]` and `,"metrics":[]` are not part of the empty envelope's encoding. - budget := maxBytes - aiCodeOtelJSONSize(envelope) - len(`,"events":[],"metrics":[]`) + // `,"events":[]`, `,"metrics":[]` and `,"spans":[]` are not part of the empty envelope's encoding. + budget := maxBytes - aiCodeOtelJSONSize(envelope) - len(`,"events":[],"metrics":[],"spans":[]`) var out []*AICodeOtelRequest cur := envelope curBytes := 0 + empty := func() bool { + return len(cur.Events) == 0 && len(cur.Metrics) == 0 && len(cur.Spans) == 0 + } flush := func() { - if len(cur.Events) > 0 || len(cur.Metrics) > 0 { + if !empty() { chunk := cur out = append(out, &chunk) } @@ -54,7 +57,7 @@ func SplitAICodeOtelRequest(req *AICodeOtelRequest, maxBytes int) []*AICodeOtelR curBytes = 0 } fits := func(size int) bool { - return (len(cur.Events) == 0 && len(cur.Metrics) == 0) || curBytes+size <= budget + return empty() || curBytes+size <= budget } for _, e := range req.Events { @@ -73,6 +76,14 @@ func SplitAICodeOtelRequest(req *AICodeOtelRequest, maxBytes int) []*AICodeOtelR cur.Metrics = append(cur.Metrics, m) curBytes += size } + for _, sp := range req.Spans { + size := aiCodeOtelJSONSize(sp) + 1 + if !fits(size) { + flush() + } + cur.Spans = append(cur.Spans, sp) + curBytes += size + } flush() return out } diff --git a/model/api_aicode_otel_test.go b/model/api_aicode_otel_test.go index ede6ed8..e6145db 100644 --- a/model/api_aicode_otel_test.go +++ b/model/api_aicode_otel_test.go @@ -54,6 +54,38 @@ func TestSplitAICodeOtelRequest(t *testing.T) { } } +func TestSplitAICodeOtelRequest_Spans(t *testing.T) { + req := &AICodeOtelRequest{Host: "host", Project: "/repo", Source: AICodeOtelSourceClaudeCode} + for i := 0; i < 40; i++ { + req.Spans = append(req.Spans, AICodeOtelSpan{ + TraceID: "0af7651916cd43dd8448eb211c80319c", + SpanID: fmt.Sprintf("%016x", i+1), + Name: "claude_code.tool", + Attributes: map[string]any{"full_command": strings.Repeat("x", 500)}, + }) + } + + const maxBytes = 4 << 10 + chunks := SplitAICodeOtelRequest(req, maxBytes) + require.Greater(t, len(chunks), 1) + + var spanIDs []string + for _, chunk := range chunks { + body, err := json.Marshal(chunk) + require.NoError(t, err) + assert.LessOrEqual(t, len(body), maxBytes, "each body stays within the limit") + assert.Empty(t, chunk.Events) + assert.Empty(t, chunk.Metrics) + for _, s := range chunk.Spans { + spanIDs = append(spanIDs, s.SpanID) + } + } + require.Len(t, spanIDs, 40) + for i, id := range spanIDs { + assert.Equal(t, fmt.Sprintf("%016x", i+1), id, "order is kept") + } +} + func TestSplitAICodeOtelRequest_SmallRequestIsOneChunk(t *testing.T) { req := &AICodeOtelRequest{Host: "h", Project: "p", Events: []AICodeOtelEvent{{EventID: "a"}, {EventID: "b"}}} chunks := SplitAICodeOtelRequest(req, AICodeOtelMaxRequestBytes) From 8ed6568f926cf551734caf39c05d96a832b5bd6e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 23 Oct 2026 03:17:58 +0000 Subject: [PATCH 2/2] fix(daemon): cap span and span event names Review follow-ups: span and event names are capped like the other forwarded text (2 KB), the dropped identity attributes are commented, and span kinds and status codes get a table test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Ebmd3PkNoem9ubKNVA9K2K --- daemon/aicode_otel_processor.go | 5 ++-- daemon/aicode_otel_traces_test.go | 40 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/daemon/aicode_otel_processor.go b/daemon/aicode_otel_processor.go index e747a02..7547c1d 100644 --- a/daemon/aicode_otel_processor.go +++ b/daemon/aicode_otel_processor.go @@ -322,7 +322,7 @@ func parseSpan(s *tracev1.Span, source string, res *model.AICodeOtelResourceAttr TraceID: traceID, SpanID: spanID, ParentSpanID: otelID(s.GetParentSpanId(), 8), - Name: s.GetName(), + Name: model.CapAICodeText(s.GetName(), model.AICodeOtelMaxAttributeBytes), Kind: spanKind(s.GetKind()), StartTimeUnixNano: unixNano(s.GetStartTimeUnixNano()), EndTimeUnixNano: unixNano(s.GetEndTimeUnixNano()), @@ -352,6 +352,7 @@ func parseSpan(s *tracev1.Span, source string, res *model.AICodeOtelResourceAttr case key == "conversation.id": conversationID = anyString(attr.GetValue()) case spanIdentityAttributes[key]: + // Dropped: the same for every span of the session. default: span.Attributes = setAttribute(span.Attributes, key, anyValueToGo(attr.GetValue())) } @@ -365,7 +366,7 @@ func parseSpan(s *tracev1.Span, source string, res *model.AICodeOtelResourceAttr break } event := model.AICodeOtelSpanEvent{ - Name: ev.GetName(), + Name: model.CapAICodeText(ev.GetName(), model.AICodeOtelMaxAttributeBytes), TimeUnixNano: unixNano(ev.GetTimeUnixNano()), } for _, attr := range ev.GetAttributes() { diff --git a/daemon/aicode_otel_traces_test.go b/daemon/aicode_otel_traces_test.go index 4d400ae..e238db8 100644 --- a/daemon/aicode_otel_traces_test.go +++ b/daemon/aicode_otel_traces_test.go @@ -3,6 +3,7 @@ package daemon import ( "context" "encoding/json" + "strings" "testing" "time" @@ -150,6 +151,45 @@ func TestParseSpan_EventsAreCapped(t *testing.T) { assert.Len(t, span.Events, model.AICodeOtelMaxSpanEvents) } +func TestParseSpan_KindAndStatus(t *testing.T) { + testCases := []struct { + kind tracev1.Span_SpanKind + status tracev1.Status_StatusCode + wantKind string + wantStatus string + }{ + {tracev1.Span_SPAN_KIND_UNSPECIFIED, tracev1.Status_STATUS_CODE_UNSET, "", ""}, + {tracev1.Span_SPAN_KIND_INTERNAL, tracev1.Status_STATUS_CODE_OK, "internal", "ok"}, + {tracev1.Span_SPAN_KIND_SERVER, tracev1.Status_STATUS_CODE_ERROR, "server", "error"}, + {tracev1.Span_SPAN_KIND_CLIENT, tracev1.Status_STATUS_CODE_OK, "client", "ok"}, + {tracev1.Span_SPAN_KIND_PRODUCER, tracev1.Status_STATUS_CODE_UNSET, "producer", ""}, + {tracev1.Span_SPAN_KIND_CONSUMER, tracev1.Status_STATUS_CODE_UNSET, "consumer", ""}, + } + for _, tc := range testCases { + t.Run(tc.kind.String()+"/"+tc.status.String(), func(t *testing.T) { + s := claudeSpan("claude_code.tool", testLLMID, testRootID, 0, time.Second) + s.Kind = tc.kind + s.Status = &tracev1.Status{Code: tc.status} + span := parseTestSpan(t, claudeTestResource(), s) + require.NotNil(t, span) + assert.Equal(t, tc.wantKind, span.Kind) + assert.Equal(t, tc.wantStatus, span.StatusCode) + }) + } +} + +func TestParseSpan_NamesAreCapped(t *testing.T) { + long := strings.Repeat("n", model.AICodeOtelMaxAttributeBytes*3) + s := claudeSpan(long, testLLMID, testRootID, 0, time.Second) + s.Events = []*tracev1.Span_Event{{Name: long, TimeUnixNano: uint64(v2TestTimeNano)}} + + span := parseTestSpan(t, claudeTestResource(), s) + require.NotNil(t, span) + assert.Len(t, span.Name, model.AICodeOtelMaxAttributeBytes) + require.Len(t, span.Events, 1) + assert.Len(t, span.Events[0].Name, model.AICodeOtelMaxAttributeBytes) +} + func TestParseSpan_CodexConversationID(t *testing.T) { s := &tracev1.Span{ TraceId: testTraceID,