Skip to content

feat(storage): wire logical operation metrics in Retrying chokepoint #14695

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
nidhiii-27 wants to merge 6 commits into from
Draft

feat(storage): wire logical operation metrics in Retrying chokepoint #14695

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9d14b16
feat(storage): wire logical operation metrics in Retrying chokepoint
nidhiii-27 Oct 20, 2026
0a3696b
refactor(storage): implement Option C scoped metrics context and wire…
nidhiii-27 Oct 20, 2026
26bca57
refactor(storage): narrow logical metrics to Retrying chokepoint and …
nidhiii-27 Oct 20, 2026
e0256ac
chore(storage): merge feat/storage-metrics-context-error-mapping into…
nidhiii-27 Oct 20, 2026
23df75b
feat(storage): populate rpc system and server address in createMetric…
nidhiii-27 Oct 20, 2026
79e4656
feat(storage): address review comments on logical metrics
nidhiii-27 Oct 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions ...orage/google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -551,5 +551,10 @@ public RetrySettings getRetrySettings() {
public ApiClock getClock() {
return HttpStorageOptions.this.getClock();
}

@Override
public @Nullable StorageClientMetrics getStorageClientMetrics() {
return HttpStorageOptions.this.getStorageClientMetrics();
}
}
}
372 changes: 328 additions & 44 deletions java-storage/google-cloud-storage/src/main/java/com/google/cloud/storage/Retrying.java
View file
Open in desktop

Large diffs are not rendered by default.

38 changes: 32 additions & 6 deletions ...age/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageClientMetrics.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@
import io.opentelemetry.api.metrics.LongUpDownCounter;
import io.opentelemetry.api.metrics.Meter;
import io.opentelemetry.api.metrics.MeterProvider;
import java.util.UUID;
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
import org.checkerframework.checker.nullness.qual.Nullable;

/** Package-private instrument registry for OpenTelemetry client metrics. */
final class StorageClientMetrics {
final class StorageClientMetrics implements AutoCloseable {
static final String METER_NAME = "com.google.cloud.storage";

// Standard client attribute keys and values
Expand Down Expand Up @@ -123,18 +123,28 @@ final class StorageClientMetrics {
@Nullable private final DoubleHistogram credentialRefreshDuration;

private final Attributes clientAttributes;
@Nullable private final SdkMeterProvider ownedMeterProvider;

static StorageClientMetrics create(
MeterProvider meterProvider, boolean enableOtelMetrics, boolean enableOtelDebugMetrics) {
return create(
meterProvider, UUID.randomUUID().toString(), enableOtelMetrics, enableOtelDebugMetrics);
meterProvider, (SdkMeterProvider) null, enableOtelMetrics, enableOtelDebugMetrics);
}

static StorageClientMetrics create(
MeterProvider meterProvider,
String instanceId,
boolean enableOtelMetrics,
boolean enableOtelDebugMetrics) {
return create(
meterProvider, (SdkMeterProvider) null, enableOtelMetrics, enableOtelDebugMetrics);
}

static StorageClientMetrics create(
MeterProvider meterProvider,
@Nullable SdkMeterProvider ownedMeterProvider,
boolean enableOtelMetrics,
boolean enableOtelDebugMetrics) {
Meter meter =
meterProvider
.meterBuilder(METER_NAME)
Expand All @@ -145,10 +155,9 @@ static StorageClientMetrics create(
.put(KEY_GCP_CLIENT_SERVICE, SERVICE_STORAGE)
.put(KEY_GCP_CLIENT_VERSION, StorageOptions.version())
.put(KEY_GCP_CLIENT_ARTIFACT, ARTIFACT_STORAGE)
.put(KEY_GCP_CLIENT_INSTANCE_ID, instanceId)
.build();
return new StorageClientMetrics(
meter, clientAttributes, enableOtelMetrics, enableOtelDebugMetrics);
meter, clientAttributes, ownedMeterProvider, enableOtelMetrics, enableOtelDebugMetrics);
}

StorageClientMetrics(Meter meter, boolean enableOtelMetrics, boolean enableOtelDebugMetrics) {
Expand All @@ -158,8 +167,8 @@ static StorageClientMetrics create(
.put(KEY_GCP_CLIENT_SERVICE, SERVICE_STORAGE)
.put(KEY_GCP_CLIENT_VERSION, StorageOptions.version())
.put(KEY_GCP_CLIENT_ARTIFACT, ARTIFACT_STORAGE)
.put(KEY_GCP_CLIENT_INSTANCE_ID, UUID.randomUUID().toString())
.build(),
null,
enableOtelMetrics,
enableOtelDebugMetrics);
}
Expand All @@ -169,7 +178,17 @@ static StorageClientMetrics create(
Attributes clientAttributes,
boolean enableOtelMetrics,
boolean enableOtelDebugMetrics) {
this(meter, clientAttributes, null, enableOtelMetrics, enableOtelDebugMetrics);
}

StorageClientMetrics(
Meter meter,
Attributes clientAttributes,
@Nullable SdkMeterProvider ownedMeterProvider,
boolean enableOtelMetrics,
boolean enableOtelDebugMetrics) {
this.clientAttributes = clientAttributes;
this.ownedMeterProvider = ownedMeterProvider;

if (enableOtelMetrics) {
this.rpcClientCallDuration =
Expand Down Expand Up @@ -409,4 +428,11 @@ Attributes attachClientAttributes(@Nullable Attributes other) {
@Nullable DoubleHistogram getCredentialRefreshDuration() {
return credentialRefreshDuration;
}

@Override
public void close() {
if (ownedMeterProvider != null) {
ownedMeterProvider.close();
}
}
}
133 changes: 129 additions & 4 deletions ...ge/google-cloud-storage/src/main/java/com/google/cloud/storage/StorageMetricsContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,13 @@
import com.google.api.gax.grpc.GrpcCallContext;
import com.google.api.gax.rpc.ApiCallContext;
import io.grpc.CallOptions;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.common.AttributesBuilder;
import io.opentelemetry.context.Context;
import io.opentelemetry.context.Scope;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import org.checkerframework.checker.nullness.qual.Nullable;
Expand All @@ -48,6 +51,8 @@ final class StorageMetricsContext {
private final AtomicLong requestBytes;
private final AtomicLong responseBytes;
private final Attributes baseAttributes;
private final boolean recordOperationOnRetrierExit;
private final AtomicBoolean operationRecorded;

StorageMetricsContext(String operationName, Attributes baseAttributes) {
this(operationName, baseAttributes, System.nanoTime(), Context.current());
Expand All @@ -59,13 +64,37 @@ final class StorageMetricsContext {

StorageMetricsContext(
String operationName, Attributes baseAttributes, long startTimeNanos, Context otelContext) {
this(
operationName,
baseAttributes,
startTimeNanos,
otelContext,
true,
new AtomicBoolean(false),
new AtomicInteger(0),
new AtomicLong(0L),
new AtomicLong(0L));
}

StorageMetricsContext(
String operationName,
Attributes baseAttributes,
long startTimeNanos,
Context otelContext,
boolean recordOperationOnRetrierExit,
AtomicBoolean operationRecorded,
AtomicInteger attemptCount,
AtomicLong requestBytes,
AtomicLong responseBytes) {
this.operationName = requireNonNull(operationName, "operationName cannot be null");
this.baseAttributes = requireNonNull(baseAttributes, "baseAttributes cannot be null");
this.otelContext = requireNonNull(otelContext, "otelContext cannot be null");
this.startTimeNanos = startTimeNanos;
this.attemptCount = new AtomicInteger(0);
this.requestBytes = new AtomicLong(0L);
this.responseBytes = new AtomicLong(0L);
this.recordOperationOnRetrierExit = recordOperationOnRetrierExit;
this.operationRecorded = requireNonNull(operationRecorded, "operationRecorded cannot be null");
this.attemptCount = requireNonNull(attemptCount, "attemptCount cannot be null");
this.requestBytes = requireNonNull(requestBytes, "requestBytes cannot be null");
this.responseBytes = requireNonNull(responseBytes, "responseBytes cannot be null");
}

static StorageMetricsContext create(String operationName) {
Expand All @@ -78,7 +107,103 @@ static StorageMetricsContext create(String operationName, Attributes baseAttribu

static StorageMetricsContext create(
String operationName, Attributes baseAttributes, Context otelContext) {
return new StorageMetricsContext(operationName, baseAttributes, System.nanoTime(), otelContext);
return new StorageMetricsContext(
operationName,
baseAttributes,
System.nanoTime(),
otelContext,
true,
new AtomicBoolean(false),
new AtomicInteger(0),
new AtomicLong(0L),
new AtomicLong(0L));
}

static StorageMetricsContext createForMultiStep(String operationName) {
return createForMultiStep(operationName, Attributes.empty());
}

static StorageMetricsContext createForMultiStep(String operationName, Attributes baseAttributes) {
return createForMultiStep(operationName, baseAttributes, Context.current());
}

static StorageMetricsContext createForMultiStep(
String operationName, Attributes baseAttributes, Context otelContext) {
return new StorageMetricsContext(
operationName,
baseAttributes,
System.nanoTime(),
otelContext,
false,
new AtomicBoolean(false),
new AtomicInteger(0),
new AtomicLong(0L),
new AtomicLong(0L));
}

StorageMetricsContext asSubAttemptContext() {
return new StorageMetricsContext(
operationName,
baseAttributes,
startTimeNanos,
otelContext,
false,
operationRecorded,
attemptCount,
requestBytes,
responseBytes);
}

boolean shouldRecordOperationOnRetrierExit() {
return recordOperationOnRetrierExit;
}

void recordOperation(
@Nullable StorageClientMetrics metrics, @Nullable Throwable terminalThrowable) {
if (metrics == null || !operationRecorded.compareAndSet(false, true)) {
return;
}
try (Scope ignore = otelContext.makeCurrent()) {
String terminalErrorType = StorageErrorTypeMapper.mapThrowable(terminalThrowable);
String clientMethod = baseAttributes.get(StorageClientMetrics.KEY_GCP_CLIENT_METHOD);
if (clientMethod == null || clientMethod.isEmpty()) {
clientMethod = operationName;
}

AttributesBuilder opAttrBuilder = Attributes.builder();
opAttrBuilder.putAll(metrics.getClientAttributes());
baseAttributes.forEach(
(key, val) -> {
if (!key.equals(StorageClientMetrics.KEY_RPC_METHOD)
&& !key.equals(StorageClientMetrics.KEY_NETWORK_PATH)
&& !key.equals(StorageClientMetrics.KEY_RPC_RESPONSE_STATUS_CODE)
&& !key.equals(StorageClientMetrics.KEY_HTTP_RESPONSE_STATUS_CODE)) {
@SuppressWarnings("unchecked")
AttributeKey<Object> objKey = (AttributeKey<Object>) key;
opAttrBuilder.put(objKey, val);
}
});
opAttrBuilder.put(StorageClientMetrics.KEY_GCP_STORAGE_OPERATION, operationName);
opAttrBuilder.put(StorageClientMetrics.KEY_GCP_CLIENT_METHOD, clientMethod);
opAttrBuilder.put(StorageClientMetrics.KEY_ERROR_TYPE, terminalErrorType);
Attributes operationAttributes = opAttrBuilder.build();

if (metrics.getOperations() != null) {
metrics.getOperations().add(1, operationAttributes);
}

if (metrics.getGcpClientRequestDuration() != null) {
String rpcMethod = baseAttributes.get(StorageClientMetrics.KEY_RPC_METHOD);
if (rpcMethod == null || !rpcMethod.contains("/")) {
rpcMethod = "google.storage.v2.Storage/" + operationName;
}
Attributes durationAttributes =
operationAttributes.toBuilder()
.put(StorageClientMetrics.KEY_RPC_METHOD, rpcMethod)
.build();
metrics.getGcpClientRequestDuration().record(getElapsedSeconds(), durationAttributes);
}
}
}

static @Nullable StorageMetricsContext from(@Nullable CallOptions callOptions) {
Expand Down
Loading
Loading