Skip to content

Commit 57366ee

Browse files
shradhakatyalshradhakatyal
authored
feat(storage): integrate OpenTelemetry metrics gating across clients and transfer_manager (#18577)
Integrate the OpenTelemetry metrics gating configuration across Google Cloud Storage client classes and `transfer_manager`. - Add `enable_metrics` and `enable_debug_metrics` parameters to `Client`, `GrpcClient`, and `AsyncGrpcClient`. - Expose `metrics_enabled` and `debug_metrics_enabled` properties across all three client classes. - Preserve `enable_metrics` and `enable_debug_metrics` across multi-process boundaries in `transfer_manager` (`_reduce_client` and `_LazyClient`). - Add unit tests in `test_client.py`, `test_grpc_client.py`, `test_async_grpc_client.py`, and `test_transfer_manager.py`.
1 parent afd4796 commit 57366ee

8 files changed

Lines changed: 496 additions & 4 deletions

File tree

‎packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from google.cloud._storage_v2.services.storage.transports.base import (
2222
DEFAULT_CLIENT_INFO,
2323
)
24-
from google.cloud.storage import __version__
24+
from google.cloud.storage import __version__, _opentelemetry_metrics
2525
from google.cloud.storage.grpc_client import (
2626
_DEFAULT_HOST,
2727
_resolve_direct_path_interconnect,
@@ -73,6 +73,19 @@ class AsyncGrpcClient:
7373
using xDS and standard TLS. Can also be overridden via the
7474
``GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS_OVER_INTERCONNECT`` environment
7575
variable (``"true"`` or ``"false"``). Defaults to ``False``.
76+
77+
:type enable_metrics: bool or None
78+
:param enable_metrics:
79+
(Optional, Experimental) Whether to enable OpenTelemetry metrics. If
80+
``None``, falls back to the ``GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS``
81+
environment variable, or ``False`` if unset.
82+
83+
:type enable_debug_metrics: bool or None
84+
:param enable_debug_metrics:
85+
(Optional, Experimental) Whether to enable debug OpenTelemetry
86+
metrics. If ``None``, falls back to the
87+
``GCP_STORAGE_PYTHON_ENABLE_OTEL_DEBUG_METRICS`` environment variable,
88+
or ``False`` if unset.
7689
"""
7790

7891
def __init__(
@@ -83,7 +96,18 @@ def __init__(
8396
*,
8497
attempt_direct_path=True,
8598
attempt_direct_path_xds_over_interconnect=False,
99+
enable_metrics=None,
100+
enable_debug_metrics=None,
86101
):
102+
self._enable_metrics = enable_metrics
103+
self._enable_debug_metrics = enable_debug_metrics
104+
self._metrics_enabled = _opentelemetry_metrics.is_metrics_enabled(
105+
enable_metrics
106+
)
107+
self._debug_metrics_enabled = _opentelemetry_metrics.is_debug_metrics_enabled(
108+
enable_debug_metrics
109+
)
110+
87111
if isinstance(credentials, auth_credentials.AnonymousCredentials):
88112
if client_options is None or client_options.api_endpoint is None:
89113
raise ValueError(
@@ -115,6 +139,16 @@ def __init__(
115139
attempt_direct_path_xds_over_interconnect=use_dp_interconnect,
116140
)
117141

142+
@property
143+
def metrics_enabled(self) -> bool:
144+
"""(Experimental) Returns ``True`` if OpenTelemetry metrics recording is active for this client."""
145+
return self._metrics_enabled
146+
147+
@property
148+
def debug_metrics_enabled(self) -> bool:
149+
"""(Experimental) Returns ``True`` if debug metrics recording is active for this client."""
150+
return self._debug_metrics_enabled
151+
118152
def _create_anonymous_client(self, client_options, credentials):
119153
channel = grpc.aio.insecure_channel(client_options.api_endpoint)
120154
transport = storage_v2.services.storage.transports.StorageGrpcAsyncIOTransport(

‎packages/google-cloud-storage/google/cloud/storage/client.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
from google.cloud.client import ClientWithProject
3333
from google.cloud.exceptions import NotFound
3434

35+
from google.cloud.storage import _opentelemetry_metrics
3536
from google.cloud.storage._bucket_metadata_cache import BucketMetadataCache
3637
from google.cloud.storage._helpers import (
3738
_DEFAULT_SCHEME,
@@ -126,6 +127,19 @@ class Client(ClientWithProject):
126127
(Optional) An API key. Mutually exclusive with any other credentials.
127128
This parameter is an alias for setting `client_options.api_key` and
128129
will supercede any api key set in the `client_options` parameter.
130+
131+
:type enable_metrics: bool or None
132+
:param enable_metrics:
133+
(Optional, Experimental) Whether to enable OpenTelemetry metrics. If
134+
``None``, falls back to the ``GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS``
135+
environment variable, or ``False`` if unset.
136+
137+
:type enable_debug_metrics: bool or None
138+
:param enable_debug_metrics:
139+
(Optional, Experimental) Whether to enable debug OpenTelemetry
140+
metrics. If ``None``, falls back to the
141+
``GCP_STORAGE_PYTHON_ENABLE_OTEL_DEBUG_METRICS`` environment variable,
142+
or ``False`` if unset.
129143
"""
130144

131145
SCOPE = (
@@ -146,6 +160,8 @@ def __init__(
146160
extra_headers={},
147161
*,
148162
api_key=None,
163+
enable_metrics=None,
164+
enable_debug_metrics=None,
149165
):
150166
self._base_connection = None
151167

@@ -163,6 +179,14 @@ def __init__(
163179
self._initial_client_info = client_info
164180
self._initial_client_options = client_options
165181
self._extra_headers = extra_headers
182+
self._enable_metrics = enable_metrics
183+
self._enable_debug_metrics = enable_debug_metrics
184+
self._metrics_enabled = _opentelemetry_metrics.is_metrics_enabled(
185+
enable_metrics
186+
)
187+
self._debug_metrics_enabled = _opentelemetry_metrics.is_debug_metrics_enabled(
188+
enable_debug_metrics
189+
)
166190

167191
connection_kw_args = {"client_info": client_info}
168192

@@ -294,6 +318,16 @@ def __init__(
294318
self._batch_stack = _LocalStack()
295319
self._bucket_metadata_cache = BucketMetadataCache(self)
296320

321+
@property
322+
def metrics_enabled(self) -> bool:
323+
"""(Experimental) Returns ``True`` if OpenTelemetry metrics recording is active for this client."""
324+
return self._metrics_enabled
325+
326+
@property
327+
def debug_metrics_enabled(self) -> bool:
328+
"""(Experimental) Returns ``True`` if debug metrics recording is active for this client."""
329+
return self._debug_metrics_enabled
330+
297331
def close(self):
298332
"""Close the client and clear any cached metadata or active connections."""
299333
if hasattr(self, "_bucket_metadata_cache") and self._bucket_metadata_cache:

‎packages/google-cloud-storage/google/cloud/storage/grpc_client.py‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
from google.cloud.client import ClientWithProject
2020

2121
from google.cloud import _storage_v2 as storage_v2
22+
from google.cloud.storage import _opentelemetry_metrics
2223

2324
_marker = object()
2425
_DEFAULT_HOST = "storage.googleapis.com"
@@ -103,6 +104,19 @@ class GrpcClient(ClientWithProject):
103104
using xDS and standard TLS. Can also be overridden via the
104105
``GOOGLE_CLOUD_ENABLE_DIRECT_PATH_XDS_OVER_INTERCONNECT`` environment
105106
variable (``"true"`` or ``"false"``). Defaults to ``False``.
107+
108+
:type enable_metrics: bool or None
109+
:param enable_metrics:
110+
(Optional, Experimental) Whether to enable OpenTelemetry metrics. If
111+
``None``, falls back to the ``GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS``
112+
environment variable, or ``False`` if unset.
113+
114+
:type enable_debug_metrics: bool or None
115+
:param enable_debug_metrics:
116+
(Optional, Experimental) Whether to enable debug OpenTelemetry
117+
metrics. If ``None``, falls back to the
118+
``GCP_STORAGE_PYTHON_ENABLE_OTEL_DEBUG_METRICS`` environment variable,
119+
or ``False`` if unset.
106120
"""
107121

108122
def __init__(
@@ -115,9 +129,20 @@ def __init__(
115129
api_key=None,
116130
attempt_direct_path=True,
117131
attempt_direct_path_xds_over_interconnect=False,
132+
enable_metrics=None,
133+
enable_debug_metrics=None,
118134
):
119135
super(GrpcClient, self).__init__(project=project, credentials=credentials)
120136

137+
self._enable_metrics = enable_metrics
138+
self._enable_debug_metrics = enable_debug_metrics
139+
self._metrics_enabled = _opentelemetry_metrics.is_metrics_enabled(
140+
enable_metrics
141+
)
142+
self._debug_metrics_enabled = _opentelemetry_metrics.is_debug_metrics_enabled(
143+
enable_debug_metrics
144+
)
145+
121146
if isinstance(client_options, dict):
122147
if api_key:
123148
client_options["api_key"] = api_key
@@ -138,6 +163,16 @@ def __init__(
138163
attempt_direct_path_xds_over_interconnect=use_dp_interconnect,
139164
)
140165

166+
@property
167+
def metrics_enabled(self) -> bool:
168+
"""(Experimental) Returns ``True`` if OpenTelemetry metrics recording is active for this client."""
169+
return self._metrics_enabled
170+
171+
@property
172+
def debug_metrics_enabled(self) -> bool:
173+
"""(Experimental) Returns ``True`` if debug metrics recording is active for this client."""
174+
return self._debug_metrics_enabled
175+
141176
def _create_gapic_client(
142177
self,
143178
credentials=None,

‎packages/google-cloud-storage/google/cloud/storage/transfer_manager.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1383,6 +1383,8 @@ def _reduce_client(cl):
13831383
client_info = cl._initial_client_info
13841384
client_options = cl._initial_client_options
13851385
extra_headers = getattr(cl, "_extra_headers", {})
1386+
enable_metrics = getattr(cl, "_enable_metrics", None)
1387+
enable_debug_metrics = getattr(cl, "_enable_debug_metrics", None)
13861388

13871389
return _LazyClient, (
13881390
client_object_id,
@@ -1392,6 +1394,8 @@ def _reduce_client(cl):
13921394
client_info,
13931395
client_options,
13941396
extra_headers,
1397+
enable_metrics,
1398+
enable_debug_metrics,
13951399
)
13961400

13971401

@@ -1453,6 +1457,13 @@ def _digest_ordered_checksum_and_size_pairs(checksum_and_size_pairs):
14531457
return crc_digest
14541458

14551459

1460+
# Number of positional Client constructor arguments passed after `id` by
1461+
# `_reduce_client`: (project, credentials, _http, client_info, client_options).
1462+
# Trailing items in `args` correspond to `extra_headers` and keyword-only
1463+
# `enable_metrics` and `enable_debug_metrics` parameters.
1464+
_REDUCE_CLIENT_POS_ARGS = 5
1465+
1466+
14561467
class _LazyClient:
14571468
"""An object that will transform into either a cached or a new Client"""
14581469

@@ -1461,6 +1472,17 @@ def __new__(cls, id, *args, **kwargs):
14611472
if cached_client:
14621473
return cached_client
14631474
else:
1475+
if len(args) > _REDUCE_CLIENT_POS_ARGS:
1476+
kwargs.setdefault("extra_headers", args[_REDUCE_CLIENT_POS_ARGS])
1477+
if len(args) > _REDUCE_CLIENT_POS_ARGS + 1:
1478+
kwargs.setdefault(
1479+
"enable_metrics", args[_REDUCE_CLIENT_POS_ARGS + 1]
1480+
)
1481+
if len(args) > _REDUCE_CLIENT_POS_ARGS + 2:
1482+
kwargs.setdefault(
1483+
"enable_debug_metrics", args[_REDUCE_CLIENT_POS_ARGS + 2]
1484+
)
1485+
args = args[:_REDUCE_CLIENT_POS_ARGS]
14641486
cached_client = Client(*args, **kwargs)
14651487
_cached_clients[id] = cached_client
14661488
return cached_client

‎packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,107 @@ def test_constructor_disables_directpath(self, mock_async_storage_client):
149149
mock_channel = mock_transport_cls.create_channel.return_value
150150
mock_transport_cls.assert_called_once_with(channel=mock_channel)
151151

152+
@mock.patch("google.cloud._storage_v2.StorageAsyncClient")
153+
def test_metrics_properties(self, mock_async_storage_client):
154+
from google.cloud.storage import _opentelemetry_metrics
155+
156+
mock_transport_cls = mock.MagicMock()
157+
mock_async_storage_client.get_transport_class.return_value = mock_transport_cls
158+
mock_creds = _make_credentials()
159+
160+
with mock.patch.multiple(
161+
_opentelemetry_metrics,
162+
HAS_OPENTELEMETRY_METRICS=True,
163+
_ENABLE_METRICS_DEV_GATE=True,
164+
):
165+
client = async_grpc_client.AsyncGrpcClient(
166+
credentials=mock_creds,
167+
enable_metrics=True,
168+
enable_debug_metrics=True,
169+
)
170+
assert client.metrics_enabled is True
171+
assert client.debug_metrics_enabled is True
172+
173+
client_base_false = async_grpc_client.AsyncGrpcClient(
174+
credentials=mock_creds,
175+
enable_metrics=False,
176+
enable_debug_metrics=True,
177+
)
178+
assert client_base_false.metrics_enabled is False
179+
assert client_base_false.debug_metrics_enabled is True
180+
181+
client_both_false = async_grpc_client.AsyncGrpcClient(
182+
credentials=mock_creds,
183+
enable_metrics=False,
184+
enable_debug_metrics=False,
185+
)
186+
assert client_both_false.metrics_enabled is False
187+
assert client_both_false.debug_metrics_enabled is False
188+
189+
with mock.patch.multiple(
190+
_opentelemetry_metrics,
191+
HAS_OPENTELEMETRY_METRICS=True,
192+
_ENABLE_METRICS_DEV_GATE=False,
193+
):
194+
client = async_grpc_client.AsyncGrpcClient(
195+
credentials=mock_creds,
196+
enable_metrics=True,
197+
enable_debug_metrics=True,
198+
)
199+
assert client.metrics_enabled is False
200+
assert client.debug_metrics_enabled is False
201+
202+
@mock.patch("google.cloud._storage_v2.StorageAsyncClient")
203+
def test_ctor_w_invalid_enable_metrics_raises_type_error(
204+
self, mock_async_storage_client
205+
):
206+
mock_transport_cls = mock.MagicMock()
207+
mock_async_storage_client.get_transport_class.return_value = mock_transport_cls
208+
mock_creds = _make_credentials()
209+
210+
with pytest.raises(TypeError):
211+
async_grpc_client.AsyncGrpcClient(
212+
credentials=mock_creds,
213+
enable_metrics="true",
214+
)
215+
with pytest.raises(TypeError):
216+
async_grpc_client.AsyncGrpcClient(
217+
credentials=mock_creds,
218+
enable_debug_metrics="true",
219+
)
220+
221+
@mock.patch("google.cloud._storage_v2.StorageAsyncClient")
222+
def test_ctor_metrics_defaults_and_env_fallback(self, mock_async_storage_client):
223+
from google.cloud.storage import _opentelemetry_metrics
224+
225+
mock_transport_cls = mock.MagicMock()
226+
mock_async_storage_client.get_transport_class.return_value = mock_transport_cls
227+
mock_creds = _make_credentials()
228+
229+
with mock.patch.multiple(
230+
_opentelemetry_metrics,
231+
HAS_OPENTELEMETRY_METRICS=True,
232+
_ENABLE_METRICS_DEV_GATE=True,
233+
):
234+
with mock.patch.dict("os.environ", {}, clear=True):
235+
client = async_grpc_client.AsyncGrpcClient(credentials=mock_creds)
236+
assert client._enable_metrics is None
237+
assert client._enable_debug_metrics is None
238+
assert client.metrics_enabled is False
239+
assert client.debug_metrics_enabled is False
240+
241+
with mock.patch.dict(
242+
"os.environ",
243+
{
244+
"GCP_STORAGE_PYTHON_ENABLE_OTEL_METRICS": "true",
245+
"GCP_STORAGE_PYTHON_ENABLE_OTEL_DEBUG_METRICS": "true",
246+
},
247+
clear=True,
248+
):
249+
client = async_grpc_client.AsyncGrpcClient(credentials=mock_creds)
250+
assert client.metrics_enabled is True
251+
assert client.debug_metrics_enabled is True
252+
152253
@mock.patch("google.cloud._storage_v2.StorageAsyncClient")
153254
def test_grpc_client_property(self, mock_grpc_gapic_client):
154255
# Arrange

0 commit comments

Comments
 (0)