Skip to content

Commit 5864347

Browse files
committed
feat(gax): implement operation-level client request spans (T3)
This PR implements operation-level client request spans (T3) as the parent span for individual attempt spans (T4) in OpenTelemetryTracingTracer.
1 parent 4bc219a commit 5864347

3 files changed

Lines changed: 260 additions & 36 deletions

File tree

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java‎

Lines changed: 151 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@
3535
import io.opentelemetry.api.trace.Span;
3636
import io.opentelemetry.api.trace.SpanBuilder;
3737
import io.opentelemetry.api.trace.SpanKind;
38+
import io.opentelemetry.api.trace.StatusCode;
3839
import io.opentelemetry.api.trace.Tracer;
40+
import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
41+
import io.opentelemetry.context.Context;
3942
import java.util.HashMap;
4043
import java.util.Map;
4144
import java.util.concurrent.CancellationException;
@@ -51,19 +54,28 @@ class OpenTelemetryTracingTracer implements ApiTracer {
5154
private final Tracer tracer;
5255
private final Map<String, Object> attemptAttributes;
5356
private final String attemptSpanName;
57+
private final String operationSpanName;
5458
private final ApiTracerContext apiTracerContext;
5559
// Captures the active trace context from the calling thread at RPC initiation.
56-
// This allows attempt spans—including retries dispatched on background threads—to
57-
// link back to the original parent trace.
58-
private final io.opentelemetry.context.Context parentContext;
60+
// This allows the operation span and attempt spans to link back to the caller's trace.
61+
private final Context parentContext;
62+
// Trace context containing the operationSpan, serving as the parent for attempt spans.
63+
private final Context operationContext;
64+
private @Nullable Span operationSpan;
5965
private @Nullable Span attemptSpan;
6066

6167
@Override
62-
public void injectTraceContext(java.util.Map<String, String> carrier) {
63-
if (attemptSpan != null) {
64-
io.opentelemetry.context.Context context =
65-
io.opentelemetry.context.Context.current().with(attemptSpan);
66-
io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator.getInstance()
68+
public void injectTraceContext(Map<String, String> carrier) {
69+
if (carrier == null) {
70+
return;
71+
}
72+
// Prefer the active attempt span (T4) so outgoing RPC wire context reflects the specific
73+
// attempt;
74+
// fall back to the overall operation span (T3) if no attempt is currently in-flight.
75+
Span currentSpan = attemptSpan != null ? attemptSpan : operationSpan;
76+
if (currentSpan != null) {
77+
Context context = Context.current().with(currentSpan);
78+
W3CTraceContextPropagator.getInstance()
6779
.inject(
6880
context,
6981
carrier,
@@ -75,19 +87,27 @@ public void injectTraceContext(java.util.Map<String, String> carrier) {
7587
}
7688
}
7789

90+
@Override
91+
@SuppressWarnings("MustBeClosedChecker")
92+
public Scope inScope() {
93+
// Attach the active attempt span to the current execution thread context;
94+
// fall back to the overall operation span when between attempts.
95+
Span currentSpan = attemptSpan != null ? attemptSpan : operationSpan;
96+
if (currentSpan == null) {
97+
return () -> {};
98+
}
99+
io.opentelemetry.context.Scope otelScope = currentSpan.makeCurrent();
100+
return otelScope::close;
101+
}
102+
78103
/**
79104
* Creates a new instance of {@code OpenTelemetryTracingTracer}.
80105
*
81106
* @param tracer the {@link Tracer} to use for recording spans
82107
* @param apiTracerContext the {@link ApiTracerContext} to use for recording spans
83108
*/
84109
OpenTelemetryTracingTracer(Tracer tracer, ApiTracerContext apiTracerContext) {
85-
this.tracer = tracer;
86-
this.apiTracerContext = apiTracerContext;
87-
this.attemptSpanName = resolveAttemptSpanName(apiTracerContext);
88-
this.attemptAttributes = new HashMap<>();
89-
this.parentContext = io.opentelemetry.context.Context.current();
90-
buildAttributes();
110+
this(tracer, apiTracerContext, resolveAttemptSpanName(apiTracerContext));
91111
}
92112

93113
/**
@@ -101,14 +121,72 @@ public void injectTraceContext(java.util.Map<String, String> carrier) {
101121
@InternalApi
102122
OpenTelemetryTracingTracer(
103123
Tracer tracer, ApiTracerContext apiTracerContext, String attemptSpanName) {
124+
this(tracer, apiTracerContext, attemptSpanName, resolveOperationSpanName(attemptSpanName));
125+
}
126+
127+
/**
128+
* Creates a new instance of {@code OpenTelemetryTracingTracer} with explicitly provided attempt
129+
* and operation span names.
130+
*
131+
* @param tracer the {@link Tracer} to use for recording spans
132+
* @param apiTracerContext the {@link ApiTracerContext} to use for recording spans
133+
* @param attemptSpanName the name of the individual attempt spans
134+
* @param operationSpanName the name of the overall client request operation span
135+
*/
136+
@InternalApi
137+
OpenTelemetryTracingTracer(
138+
Tracer tracer,
139+
ApiTracerContext apiTracerContext,
140+
String attemptSpanName,
141+
String operationSpanName) {
104142
this.tracer = tracer;
105-
this.attemptSpanName = attemptSpanName;
106143
this.apiTracerContext = apiTracerContext;
144+
this.attemptSpanName = attemptSpanName;
145+
this.operationSpanName = operationSpanName;
107146
this.attemptAttributes = new HashMap<>();
108-
this.parentContext = io.opentelemetry.context.Context.current();
147+
this.parentContext = Context.current();
109148
buildAttributes();
149+
this.operationSpan = startOperationSpan();
150+
this.operationContext = parentContext.with(this.operationSpan);
110151
}
111152

153+
/**
154+
* Starts and initializes the operation-level client request span (T3).
155+
*
156+
* @return the newly started {@link Span} for the overall operation
157+
*/
158+
private Span startOperationSpan() {
159+
SpanBuilder operationSpanBuilder = tracer.spanBuilder(operationSpanName);
160+
operationSpanBuilder.setSpanKind(SpanKind.INTERNAL);
161+
operationSpanBuilder.setParent(parentContext);
162+
operationSpanBuilder.setAllAttributes(
163+
ObservabilityUtils.toOtelAttributes(this.attemptAttributes));
164+
return operationSpanBuilder.startSpan();
165+
}
166+
167+
/**
168+
* Derives the operation-level span name from the attempt span name.
169+
*
170+
* @param attemptSpanName the attempt span name
171+
* @return the operation span name
172+
*/
173+
private static String resolveOperationSpanName(String attemptSpanName) {
174+
if (!Strings.isNullOrEmpty(attemptSpanName)) {
175+
if (attemptSpanName.endsWith("/attempt")) {
176+
String name = attemptSpanName.substring(0, attemptSpanName.length() - "/attempt".length());
177+
return name.isEmpty() ? "operation" : name;
178+
}
179+
return "attempt".equals(attemptSpanName) ? "operation" : attemptSpanName;
180+
}
181+
return "operation";
182+
}
183+
184+
/**
185+
* Resolves the canonical attempt-level span name based on transport and context.
186+
*
187+
* @param apiTracerContext the tracer context containing transport and method metadata
188+
* @return the attempt span name
189+
*/
112190
private static String resolveAttemptSpanName(ApiTracerContext apiTracerContext) {
113191
if (apiTracerContext.transport() == ApiTracerContext.Transport.GRPC) {
114192
// gRPC Uses the full method name as span name.
@@ -124,12 +202,24 @@ private static String resolveAttemptSpanName(ApiTracerContext apiTracerContext)
124202
}
125203
}
126204

205+
/** Copies attempt-level attributes from the tracer context into the local attribute cache. */
127206
private void buildAttributes() {
128207
this.attemptAttributes.putAll(this.apiTracerContext.getAttemptAttributes());
129208
}
130209

131210
@Override
132211
public void attemptStarted(Object request, int attemptNumber) {
212+
// Prevent creating new attempt spans if the overall operation has already concluded.
213+
if (operationSpan == null) {
214+
return;
215+
}
216+
// If a previous attempt was not explicitly closed before a retry started,
217+
// end it cleanly so it does not linger.
218+
if (attemptSpan != null) {
219+
endSpan(attemptSpan, null);
220+
attemptSpan = null;
221+
}
222+
133223
Map<String, Object> currentAttemptAttributes = new HashMap<>(this.attemptAttributes);
134224

135225
if (attemptNumber > 0) {
@@ -148,8 +238,8 @@ public void attemptStarted(Object request, int attemptNumber) {
148238
// Attempt spans are of the CLIENT kind
149239
spanBuilder.setSpanKind(SpanKind.CLIENT);
150240

151-
// Link attempt span to parent context
152-
spanBuilder.setParent(parentContext);
241+
// Link attempt span to operation context (parent T3 span)
242+
spanBuilder.setParent(operationContext);
153243

154244
// Pass the combined attributes to the new SpanBuilder method
155245
spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes));
@@ -160,33 +250,51 @@ public void attemptStarted(Object request, int attemptNumber) {
160250
/**
161251
* Signals that the overall logical operation succeeded.
162252
*
163-
* <p>Closes any remaining in-flight attempt span.
253+
* <p>Closes any remaining in-flight attempt span and ends the operation span.
164254
*/
165255
@Override
166256
public void operationSucceeded() {
167-
recordErrorAndEndAttempt(null);
257+
recordErrorAndEndOperation(null);
168258
}
169259

170260
/**
171261
* Signals that the overall logical operation was cancelled.
172262
*
173-
* <p>Closes any remaining in-flight attempt span with a {@link CancellationException}.
263+
* <p>Closes any remaining in-flight attempt span with a {@link CancellationException} and ends
264+
* the operation span with an ERROR status.
174265
*/
175266
@Override
176267
public void operationCancelled() {
177-
recordErrorAndEndAttempt(new CancellationException());
268+
recordErrorAndEndOperation(new CancellationException());
178269
}
179270

180271
/**
181272
* Signals that the overall logical operation failed permanently.
182273
*
183-
* <p>Closes any remaining in-flight attempt span with the provided error details.
274+
* <p>Closes any remaining in-flight attempt span with the provided error details and ends the
275+
* operation span with an ERROR status.
184276
*
185277
* @param error the cause of the operation failure
186278
*/
187279
@Override
188280
public void operationFailed(Throwable error) {
189-
recordErrorAndEndAttempt(error);
281+
recordErrorAndEndOperation(error);
282+
}
283+
284+
/**
285+
* Records error details and ends both the active attempt span and the operation span.
286+
*
287+
* @param error the exception associated with the operation failure, or {@code null} if successful
288+
*/
289+
private void recordErrorAndEndOperation(@Nullable Throwable error) {
290+
if (attemptSpan != null) {
291+
endSpan(attemptSpan, error);
292+
attemptSpan = null;
293+
}
294+
if (operationSpan != null) {
295+
endSpan(operationSpan, error);
296+
operationSpan = null;
297+
}
190298
}
191299

192300
@Override
@@ -195,7 +303,7 @@ public void attemptSucceeded() {
195303
}
196304

197305
@Override
198-
public void responseHeadersReceived(java.util.Map<String, Object> headers) {
306+
public void responseHeadersReceived(Map<String, Object> headers) {
199307
if (attemptSpan == null) {
200308
return;
201309
}
@@ -215,7 +323,7 @@ public void responseHeadersReceived(java.util.Map<String, Object> headers) {
215323
* @param headers the map of response headers.
216324
* @return the content length in bytes, or -1 if the header is missing or malformed.
217325
*/
218-
private long extractContentLength(java.util.Map<String, Object> headers) {
326+
private long extractContentLength(Map<String, Object> headers) {
219327
try {
220328
if (headers == null || headers.isEmpty()) return -1;
221329
// google-http-client HttpHeaders uses a case-insensitive map but we copy it for safety
@@ -265,20 +373,31 @@ private void recordErrorAndEndAttempt(@Nullable Throwable error) {
265373
if (attemptSpan == null) {
266374
return;
267375
}
376+
endSpan(attemptSpan, error);
377+
attemptSpan = null;
378+
}
268379

380+
/**
381+
* Attaches response status attributes and error messages to the span and ends it.
382+
*
383+
* @param span the span to finish
384+
* @param error the exception that caused the span to end, or {@code null} if successful
385+
*/
386+
private void endSpan(Span span, @Nullable Throwable error) {
269387
Map<String, Object> responseAttributes =
270388
ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport());
271389
if (!responseAttributes.isEmpty()) {
272-
attemptSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes));
390+
span.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes));
273391
}
274392

275-
if (error != null && !Strings.isNullOrEmpty(error.getMessage())) {
276-
attemptSpan.setAttribute(
277-
ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage());
393+
if (error != null) {
394+
span.setStatus(StatusCode.ERROR);
395+
if (!Strings.isNullOrEmpty(error.getMessage())) {
396+
span.setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage());
397+
}
278398
}
279399

280-
attemptSpan.end();
281-
attemptSpan = null;
400+
span.end();
282401
}
283402

284403
@Override

‎sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -229,7 +229,7 @@ void testNewTracer_withContext_grpc_usesFullMethodName() {
229229

230230
tracerInstance.attemptStarted(null, 1);
231231

232-
verify(tracer).spanBuilder("google.cloud.v1.Service/Method");
232+
verify(tracer, atLeastOnce()).spanBuilder("google.cloud.v1.Service/Method");
233233
}
234234

235235
@ParameterizedTest
@@ -256,7 +256,7 @@ void testNewTracer_withContext_http_usesHttpMethodAndPathTemplate(
256256

257257
tracerInstance.attemptStarted(null, 1);
258258

259-
verify(tracer).spanBuilder(expectedSpanName);
259+
verify(tracer, atLeastOnce()).spanBuilder(expectedSpanName);
260260
}
261261

262262
@Test
@@ -274,7 +274,7 @@ void testNewTracer_withContext_http_noHttpMethodOrPathTemplate_usesFullMethodNam
274274

275275
tracerInstance.attemptStarted(null, 1);
276276

277-
verify(tracer).spanBuilder("google.cloud.v1.Service.Method");
277+
verify(tracer, atLeastOnce()).spanBuilder("google.cloud.v1.Service.Method");
278278
}
279279

280280
@Test
@@ -310,7 +310,7 @@ void testNewTracer_mergesFactoryContext() {
310310
tracerInstance.attemptStarted(null, 1);
311311

312312
ArgumentCaptor<Attributes> attributesCaptor = ArgumentCaptor.forClass(Attributes.class);
313-
verify(spanBuilder).setAllAttributes(attributesCaptor.capture());
313+
verify(spanBuilder, atLeastOnce()).setAllAttributes(attributesCaptor.capture());
314314

315315
Attributes attributes = attributesCaptor.getValue();
316316
assertThat(attributes.asMap())

0 commit comments

Comments
 (0)