From be87984ee0b945388a44e1c7a42ffd86bb6641b9 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Thu, 15 Oct 2026 15:44:24 +0000 Subject: [PATCH 1/7] feat(storage): add OpenTelemetry standard metrics recorder, views, and helpers Implement core OpenTelemetry standard metrics infrastructure for GCS: - Add _MetricsRecorder with the 9 standard metric instruments - Add custom latency (165 buckets) and byte-size (45 buckets) histogram views - Add classify_error_type for low-cardinality error.type mapping - Add normalize_url_template for bounded HTTP url.template routing - Add start_operation ContextVar for outermost logical operation tracking and nested call suppression [Generated-by: AI] --- .../cloud/storage/_opentelemetry_metrics.py | 959 ++++++++++++++++- .../tests/unit/test__opentelemetry_metrics.py | 981 +++++++++++++++++- 2 files changed, 1936 insertions(+), 4 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py index 8592cc24c65d..c357a968407f 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py +++ b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py @@ -14,9 +14,21 @@ """Manages OpenTelemetry metrics instruments and gating for GCS client.""" +import asyncio +from collections import deque +import concurrent.futures +import contextlib +from contextvars import ContextVar +from dataclasses import dataclass +import http.client import logging import os -from typing import Any, Dict, Optional +import re +import socket +import ssl +import time +from typing import Any, Deque, Dict, Iterator, List, Optional, Set, Tuple +from urllib.parse import urlsplit from google.cloud.storage.version import __version__ @@ -38,8 +50,35 @@ _DEFAULT_ENABLE_METRICS = False _DEFAULT_ENABLE_DEBUG_METRICS = False +DEFAULT_SERVER_ADDRESS = "storage.googleapis.com" + +# --------------------------------------------------------------------------- +# 3. Standard Metric Instrument Names # --------------------------------------------------------------------------- -# 3. Optional OpenTelemetry Dependency Check +METRIC_RPC_CLIENT_CALL_DURATION = "rpc.client.call.duration" +METRIC_HTTP_CLIENT_REQUEST_DURATION = "http.client.request.duration" +METRIC_GCP_CLIENT_REQUEST_DURATION = "gcp.client.request.duration" +METRIC_STORAGE_CLIENT_OPERATIONS = "gcp.storage.client.operations" +METRIC_STORAGE_CLIENT_ATTEMPTS = "gcp.storage.client.attempts" +METRIC_STORAGE_REQUEST_BODY_SIZE = "gcp.storage.client.request.body.size" +METRIC_STORAGE_RESPONSE_BODY_SIZE = "gcp.storage.client.response.body.size" +METRIC_STORAGE_OPERATION_TTFB = "gcp.storage.client.operation.ttfb" +METRIC_STORAGE_CLIENT_ERRORS = "gcp.storage.client.errors" + +_LATENCY_METRIC_NAMES = ( + METRIC_RPC_CLIENT_CALL_DURATION, + METRIC_HTTP_CLIENT_REQUEST_DURATION, + METRIC_GCP_CLIENT_REQUEST_DURATION, + METRIC_STORAGE_OPERATION_TTFB, +) + +_SIZE_METRIC_NAMES = ( + METRIC_STORAGE_REQUEST_BODY_SIZE, + METRIC_STORAGE_RESPONSE_BODY_SIZE, +) + +# --------------------------------------------------------------------------- +# 4. Optional OpenTelemetry Dependency Check # --------------------------------------------------------------------------- try: from opentelemetry import metrics @@ -134,7 +173,7 @@ def is_debug_metrics_enabled( # --------------------------------------------------------------------------- -# 4. Standard Common Attributes & Meter Provider +# 5. Standard Common Attributes & Meter Provider # --------------------------------------------------------------------------- _COMMON_ATTRIBUTES: Dict[str, Any] = { "gcp.client.service": "storage", @@ -178,3 +217,917 @@ def get_meter( __version__, meter_provider=meter_provider, ) + + +# --------------------------------------------------------------------------- +# 6. Custom Histogram Bucket Boundaries & Views +# --------------------------------------------------------------------------- +def _build_latency_buckets() -> Tuple[float, ...]: + """Builds the custom latency histogram bucket boundaries in seconds. + + Uses 2ms resolution for the first 50 buckets (0.000s to 0.098s), then starts + at 0.100s with 10ms resolution and doubles the increment every 10 buckets + while boundary < 300.0s. + """ + boundaries: List[float] = [] + boundary = 0.0 + increment = 0.002 + + for _ in range(50): + boundaries.append(round(boundary, 6)) + boundary += increment + + increment = 0.010 + for i in range(150): + if boundary >= 300.0: + break + if i != 0 and i % 10 == 0: + increment *= 2 + boundaries.append(round(boundary, 6)) + boundary += increment + + return tuple(boundaries) + + +def _build_size_buckets() -> Tuple[int, ...]: + """Builds the custom byte-size histogram bucket boundaries in bytes. + + Uses 128 KiB resolution from 0 up to 4 MiB (33 buckets), then doubles every + bucket up to 16 GiB (12 buckets, 45 total buckets). + """ + kb = 1024 + mb = 1024 * kb + gb = 1024 * mb + + boundaries: List[int] = [] + boundary = 0 + increment = 128 * kb + + while boundary <= 16 * gb: + boundaries.append(boundary) + if boundary < 4 * mb: + boundary += increment + else: + boundary *= 2 + + return tuple(boundaries) + + +LATENCY_HISTOGRAM_BOUNDARIES: Tuple[float, ...] = _build_latency_buckets() +SIZE_HISTOGRAM_BOUNDARIES: Tuple[int, ...] = _build_size_buckets() + + +def get_histogram_views() -> List[Any]: + """Returns OpenTelemetry SDK Views for custom latency and size histograms. + + If ``opentelemetry-sdk`` is not installed, returns an empty list. + """ + try: + from opentelemetry.sdk.metrics.view import ( + ExplicitBucketHistogramAggregation, + View, + ) + except ImportError: + return [] + + views: List[Any] = [] + for name in _LATENCY_METRIC_NAMES: + views.append( + View( + instrument_name=name, + aggregation=ExplicitBucketHistogramAggregation( + boundaries=LATENCY_HISTOGRAM_BOUNDARIES + ), + ) + ) + for name in _SIZE_METRIC_NAMES: + views.append( + View( + instrument_name=name, + aggregation=ExplicitBucketHistogramAggregation( + boundaries=SIZE_HISTOGRAM_BOUNDARIES + ), + ) + ) + return views + + +# --------------------------------------------------------------------------- +# 7. Canonical Error Classification (`error.type`) +# --------------------------------------------------------------------------- +_HTTP_STATUS_TO_ERROR_TYPE: Dict[int, str] = { + 400: "INVALID_ARGUMENT", + 401: "UNAUTHENTICATED", + 403: "PERMISSION_DENIED", + 404: "NOT_FOUND", + 408: "DEADLINE_EXCEEDED", + 409: "ABORTED", + 412: "FAILED_PRECONDITION", + 416: "OUT_OF_RANGE", + 429: "RESOURCE_EXHAUSTED", + 499: "CANCELLED", + 500: "INTERNAL", + 501: "UNIMPLEMENTED", + 503: "UNAVAILABLE", + 504: "DEADLINE_EXCEEDED", +} + + +def _is_valid_status_code(val: Any) -> bool: + """Returns True if ``val`` is a valid HTTP error status code (int in 400..599).""" + return isinstance(val, int) and not isinstance(val, bool) and 400 <= val < 600 + + +def _iter_exception_chain(exc: BaseException) -> Iterator[BaseException]: + """Yields ``exc`` and any nested causes, contexts, reasons, or exception args.""" + visited: Set[int] = {id(exc)} + queue: Deque[BaseException] = deque([exc]) + while queue: + current = queue.popleft() + yield current + + for attr_name in ( + "__cause__", + "__context__", + "cause", + "reason", + "original_exception", + ): + nested = getattr(current, attr_name, None) + if isinstance(nested, BaseException) and id(nested) not in visited: + visited.add(id(nested)) + queue.append(nested) + + for arg in getattr(current, "args", ()): + if isinstance(arg, BaseException) and id(arg) not in visited: + visited.add(id(arg)) + queue.append(arg) + + +def _is_subclass_by_module( + cls: type, module_prefix: str, class_names: Optional[Set[str]] = None +) -> bool: + """Checks whether ``cls`` or any of its base classes match module/name criteria.""" + for base in getattr(cls, "__mro__", ()): + mod = getattr(base, "__module__", "") or "" + if mod.startswith(module_prefix): + if class_names is None or base.__name__ in class_names: + return True + return False + + +def classify_error_type( + exc: Optional[BaseException] = None, + status_code: Optional[int] = None, +) -> str: + """Maps an exception or HTTP status code to a low-cardinality ``error.type``. + + Args: + exc: Optional exception raised during an operation or transport attempt. + status_code: Optional HTTP status code from the response. + + Returns: + str: Canonical low-cardinality error classification string. + """ + valid_status = _is_valid_status_code(status_code) + if exc is None: + if not valid_status or status_code is None: + return "OK" + return _HTTP_STATUS_TO_ERROR_TYPE.get(status_code, str(status_code)) + + chain = list(_iter_exception_chain(exc)) + + # 1. Checksum Mismatch (DataCorruption) + for item in chain: + if _is_subclass_by_module( + item.__class__, "google.cloud.storage", {"DataCorruption"} + ) or _is_subclass_by_module( + item.__class__, "google.resumable_media", {"DataCorruption"} + ): + return "CHECKSUM_MISMATCH" + + # 2. Authentication / Credentials Failure (google.auth.exceptions) + for item in chain: + if _is_subclass_by_module( + item.__class__, + "google.auth.exceptions", + { + "RefreshError", + "TransportError", + "DefaultCredentialsError", + "GoogleAuthError", + }, + ): + return "AUTHENTICATION_ERROR" + + # 3. Cancellation + for item in chain: + if isinstance(item, asyncio.CancelledError): + return "CANCELLED" + + # 4. HTTP response status code on exception (checked before grpc_status_code so + # HTTP 409 Conflict consistently maps to ABORTED across HTTP and Veneer metrics) + for item in chain: + resp = getattr(item, "response", None) + resp_status = getattr(resp, "status_code", None) + if _is_valid_status_code(resp_status): + return _HTTP_STATUS_TO_ERROR_TYPE.get(resp_status, str(resp_status)) + + if item.__class__.__name__ == "Conflict" and _is_subclass_by_module( + item.__class__, "google.api_core.exceptions", {"Conflict"} + ): + return _HTTP_STATUS_TO_ERROR_TYPE[409] + + # 5. gRPC / GoogleAPICallError status code + for item in chain: + grpc_code = getattr(item, "grpc_status_code", None) + if grpc_code is not None: + name = getattr(grpc_code, "name", None) + if name: + return str(name) + return str(grpc_code) + + # Direct grpc.RpcError (has callable .code()) + code_attr = getattr(item, "code", None) + if callable(code_attr): + try: + rpc_status = code_attr() + if rpc_status is not None: + name = getattr(rpc_status, "name", None) + if name: + return str(name) + except Exception: # pragma: no cover + pass + + if _is_valid_status_code(code_attr): + return _HTTP_STATUS_TO_ERROR_TYPE.get(code_attr, str(code_attr)) + + # 6. TLS / SSL Failure + for item in chain: + if ( + isinstance(item, ssl.SSLError) + or _is_subclass_by_module( + item.__class__, + "requests.exceptions", + {"SSLError"}, + ) + or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"SSLError"}, + ) + ): + return "TLS_FAILURE" + + # 7. Specific Socket / Network Failures (DNS, Connection Refused, Connection Reset) + for item in chain: + if isinstance(item, socket.gaierror) or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"NameResolutionError"}, + ): + return "DNS_FAILURE" + if isinstance(item, ConnectionRefusedError): + return "CONNECTION_REFUSED" + if isinstance(item, ConnectionResetError): + return "CONNECTION_RESET" + + # 8. Timeouts (Note: urllib3.exceptions.NewConnectionError subclasses ConnectTimeoutError + # even for non-timeout connection failures, so exclude NewConnectionError here.) + for item in chain: + if item.__class__.__name__ != "NewConnectionError": + if isinstance( + item, + ( + socket.timeout, + TimeoutError, + asyncio.TimeoutError, + concurrent.futures.TimeoutError, + ), + ): + return "TIMEOUT" + if _is_subclass_by_module( + item.__class__, + "requests.exceptions", + {"Timeout", "ConnectTimeout", "ReadTimeout"}, + ) or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"TimeoutError", "ReadTimeoutError", "ConnectTimeoutError"}, + ): + return "TIMEOUT" + + # 9. Generic Connectivity Failure + for item in chain: + if ( + _is_subclass_by_module( + item.__class__, + "requests.exceptions", + {"ConnectionError", "ChunkedEncodingError", "RequestException"}, + ) + or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"HTTPError"}, + ) + or isinstance(item, (OSError, http.client.HTTPException)) + ): + return "CONNECTIVITY" + + # 10. Explicit HTTP status_code fallback + if valid_status and status_code is not None: + return _HTTP_STATUS_TO_ERROR_TYPE.get(status_code, str(status_code)) + + # 11. Client Validation Errors + for item in chain: + if ( + isinstance(item, (ValueError, TypeError)) + or _is_subclass_by_module( + item.__class__, "google.cloud.storage", {"InvalidResponse"} + ) + or _is_subclass_by_module( + item.__class__, "google.resumable_media", {"InvalidResponse"} + ) + ): + return "CLIENT_VALIDATION_ERROR" + + return "UNKNOWN" + + +# --------------------------------------------------------------------------- +# 8. HTTP `url.template` Route Normalization +# --------------------------------------------------------------------------- +_URL_TEMPLATE_PATTERNS: Tuple[Tuple[re.Pattern[str], str], ...] = ( + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/rewriteTo/b/[^/]+/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/copyTo/b/[^/]+/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}/copyTo/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/moveTo/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/compose$"), + "/storage/v1/b/{bucket}/o/{object}/compose", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/restore$"), + "/storage/v1/b/{bucket}/o/{object}/restore", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/acl/[^/]+$"), + "/storage/v1/b/{bucket}/o/{object}/acl/{entity}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/acl$"), + "/storage/v1/b/{bucket}/o/{object}/acl", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/iam(?:/.*)?$"), + "/storage/v1/b/{bucket}/o/{object}/iam", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o$"), + "/storage/v1/b/{bucket}/o", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/acl/[^/]+$"), + "/storage/v1/b/{bucket}/acl/{entity}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/acl$"), + "/storage/v1/b/{bucket}/acl", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/defaultObjectAcl/[^/]+$"), + "/storage/v1/b/{bucket}/defaultObjectAcl/{entity}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/defaultObjectAcl$"), + "/storage/v1/b/{bucket}/defaultObjectAcl", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/iam(?:/.*)?$"), + "/storage/v1/b/{bucket}/iam", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/notificationConfigs/[^/]+$"), + "/storage/v1/b/{bucket}/notificationConfigs/{notification}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/notificationConfigs$"), + "/storage/v1/b/{bucket}/notificationConfigs", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/lockRetentionPolicy$"), + "/storage/v1/b/{bucket}/lockRetentionPolicy", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/restore$"), + "/storage/v1/b/{bucket}/restore", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+$"), + "/storage/v1/b/{bucket}", + ), + ( + re.compile(r"^/storage/v1/b$"), + "/storage/v1/b", + ), + ( + re.compile(r"^/storage/v1/projects/[^/]+/serviceAccount$"), + "/storage/v1/projects/{project}/serviceAccount", + ), + ( + re.compile(r"^/storage/v1/projects/[^/]+/hmacKeys/[^/]+$"), + "/storage/v1/projects/{project}/hmacKeys/{access_id}", + ), + ( + re.compile(r"^/storage/v1/projects/[^/]+/hmacKeys$"), + "/storage/v1/projects/{project}/hmacKeys", + ), + ( + re.compile(r"^/upload/storage/v1/b/[^/]+/o(?:/[^/]+)?$"), + "/upload/storage/v1/b/{bucket}/o", + ), + ( + re.compile(r"^/download/storage/v1/b/[^/]+/o/.+$"), + "/download/storage/v1/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/batch/storage/v1$"), + "/batch/storage/v1", + ), +) + +_XML_MPU_RESERVED_PREFIXES = {"storage", "upload", "download", "batch", "b", "projects"} +_XML_MPU_QUERY_KEYS = {"uploads", "uploadId", "partNumber"} +_XML_MULTI_SEGMENT_RE = re.compile(r"^/([^/]+)/(.+)$") + + +def normalize_url_template(url_or_path: Optional[str]) -> str: + """Normalizes a raw HTTP URL or path into a bounded ``url.template`` string. + + Bucket names, object keys, project IDs, and query strings are stripped to + prevent cardinality explosion. Unrecognized paths fall back to ``"UNKNOWN"``. + + Args: + url_or_path: Full request URL or path string. + + Returns: + str: Low-cardinality ``url.template`` value. + """ + if not url_or_path or not isinstance(url_or_path, str): + return "UNKNOWN" + + parsed = urlsplit(url_or_path) + path = parsed.path or "" + query = parsed.query or "" + + path = "/" + path.strip("/") + if path == "/": + return "UNKNOWN" + + # Normalize relative paths passed to Connection.api_request before /storage/v1 prefixing + if path == "/b" or path.startswith(("/b/", "/projects/")): + path = "/storage/v1" + path + + for pattern, template in _URL_TEMPLATE_PATTERNS: + if pattern.match(path): + return template + + # XML API Multipart Upload routes: /{bucket}/{object} with XML MPU query params + if query: + query_keys = {param.partition("=")[0] for param in query.split("&") if param} + if query_keys & _XML_MPU_QUERY_KEYS: + match_multi = _XML_MULTI_SEGMENT_RE.match(path) + if match_multi and match_multi.group(1) not in _XML_MPU_RESERVED_PREFIXES: + return "/{bucket}/{object}" + + return "UNKNOWN" + + +# --------------------------------------------------------------------------- +# 9. Logical Operation Context & Suppression (`ContextVar`) +# --------------------------------------------------------------------------- +@dataclass(frozen=True) +class _OperationState: + """Represents the active logical GCS Veneer operation in the current context.""" + + client_method: str + rpc_method: str + suppressed: bool = False + + +_OP_STATE: ContextVar[Optional[_OperationState]] = ContextVar( + "_gcs_otel_op_state", default=None +) + + +def get_current_operation() -> Optional[_OperationState]: + """Returns the current logical `_OperationState` in this execution context, if any.""" + return _OP_STATE.get() + + +@contextlib.contextmanager +def start_operation( + recorder: Optional["_MetricsRecorder"], + client_method: str, + rpc_method: str, + suppress: bool = False, +) -> Iterator[Optional[_OperationState]]: + """Context manager tracking a logical user-facing GCS operation. + + If a parent logical operation is already active in the current context, + nested calls become a no-op so only the outermost user-invoked method + emits ``gcp.client.request.duration`` and ``gcp.storage.client.operations``. + + Args: + recorder: Optional `_MetricsRecorder` configured on the client. + client_method: Fully qualified Python method name + (e.g. ``"storage.Blob.download_to_filename"``). + rpc_method: Canonical GCS RPC operation name + (e.g. ``"storage.objects.get"``). + suppress: If True, suppresses logical operation and payload size + emission for this context and any nested calls (used by + ``transfer_manager`` workers and ``BlobReader`` prefetches). + """ + existing = _OP_STATE.get() + if existing is not None: + if suppress and not existing.suppressed: + suppressed_child = _OperationState( + client_method=existing.client_method, + rpc_method=existing.rpc_method, + suppressed=True, + ) + child_token = _OP_STATE.set(suppressed_child) + try: + yield suppressed_child + finally: + _OP_STATE.reset(child_token) + else: + yield existing + return + + recorder_active = recorder is not None and recorder.standard_enabled + if not recorder_active and not suppress: + yield None + return + + state = _OperationState( + client_method=client_method, + rpc_method=rpc_method, + suppressed=suppress, + ) + token = _OP_STATE.set(state) + start_time = time.perf_counter() + error_type = "OK" + try: + yield state + except BaseException as exc: + error_type = classify_error_type(exc) + raise + finally: + duration = max(0.0, time.perf_counter() - start_time) + _OP_STATE.reset(token) + if recorder_active and not suppress and recorder is not None: + recorder.record_logical_operation( + duration=duration, + client_method=client_method, + rpc_method=rpc_method, + error_type=error_type, + ) + + +# --------------------------------------------------------------------------- +# 10. Server Address Parsing & `_MetricsRecorder` +# --------------------------------------------------------------------------- +def _extract_server_address(api_endpoint: Optional[str]) -> str: + """Extracts the low-cardinality hostname from ``api_endpoint``.""" + if not api_endpoint or not isinstance(api_endpoint, str): + return DEFAULT_SERVER_ADDRESS + endpoint = api_endpoint.strip() + if not endpoint: + return DEFAULT_SERVER_ADDRESS + if "://" not in endpoint: + endpoint = "https://" + endpoint.lstrip("/") + parsed = urlsplit(endpoint) + return parsed.hostname or DEFAULT_SERVER_ADDRESS + + +class _MetricsRecorder: + """Holds OpenTelemetry metric instruments and records standard GCS metrics.""" + + def __init__( + self, + enable_metrics: Optional[bool] = None, + enable_debug_metrics: Optional[bool] = None, + meter_provider: Optional[Any] = None, + api_endpoint: Optional[str] = None, + rpc_system: str = "http", + ) -> None: + self.standard_enabled: bool = is_metrics_enabled(enable_metrics) + self.debug_enabled: bool = is_debug_metrics_enabled(enable_debug_metrics) + self.enabled: bool = self.standard_enabled or self.debug_enabled + self._meter_provider: Optional[Any] = meter_provider + self.rpc_system: str = rpc_system + self.server_address: str = _extract_server_address(api_endpoint) + self.meter: Optional[Any] = get_meter( + enable_metrics=enable_metrics, + enable_debug_metrics=enable_debug_metrics, + meter_provider=meter_provider, + ) + + self._base_attributes: Dict[str, Any] = { + **_COMMON_ATTRIBUTES, + "server.address": self.server_address, + "rpc.system.name": self.rpc_system, + } + + if self.standard_enabled and self.meter is not None: + self.rpc_call_duration = self.meter.create_histogram( + METRIC_RPC_CLIENT_CALL_DURATION, + unit="s", + description="Duration of an individual gRPC attempt in seconds (excluding retries).", + ) + self.http_request_duration = self.meter.create_histogram( + METRIC_HTTP_CLIENT_REQUEST_DURATION, + unit="s", + description="Duration of an individual HTTP request attempt in seconds (excluding retries).", + ) + self.client_request_duration = self.meter.create_histogram( + METRIC_GCP_CLIENT_REQUEST_DURATION, + unit="s", + description="End-to-end latency of a logical user-facing Veneer operation in seconds.", + ) + self.operations_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_OPERATIONS, + unit="{operation}", + description="Number of logical GCS client operations.", + ) + self.attempts_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_ATTEMPTS, + unit="{attempt}", + description="Number of physical network transport attempts executed.", + ) + self.request_body_size = self.meter.create_histogram( + METRIC_STORAGE_REQUEST_BODY_SIZE, + unit="By", + description="Application payload bytes submitted by the caller for an upload or write operation.", + ) + self.response_body_size = self.meter.create_histogram( + METRIC_STORAGE_RESPONSE_BODY_SIZE, + unit="By", + description="Application payload bytes delivered to the caller.", + ) + self.operation_ttfb = self.meter.create_histogram( + METRIC_STORAGE_OPERATION_TTFB, + unit="s", + description="Time elapsed from attempt initiation to receipt of the first response byte.", + ) + self.errors_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_ERRORS, + unit="{error}", + description="Number of physical network transport attempts terminating with a non-OK status.", + ) + else: + self.rpc_call_duration = None + self.http_request_duration = None + self.client_request_duration = None + self.operations_counter = None + self.attempts_counter = None + self.request_body_size = None + self.response_body_size = None + self.operation_ttfb = None + self.errors_counter = None + + def get_base_attributes(self) -> Dict[str, Any]: + """Returns a copy of the base attributes for this recorder instance.""" + return self._base_attributes.copy() + + def _build_operation_attributes( + self, + operation: Optional[str] = None, + client_method: Optional[str] = None, + current_op: Optional[_OperationState] = None, + ) -> Dict[str, Any]: + """Builds base attributes enriched with current or explicit operation metadata.""" + attrs = self._base_attributes.copy() + if current_op is None: + current_op = _OP_STATE.get() + resolved_op = operation or (current_op.rpc_method if current_op else None) + resolved_method = client_method or ( + current_op.client_method if current_op else None + ) + if resolved_op is not None: + attrs["gcp.storage.operation"] = resolved_op + if resolved_method is not None: + attrs["gcp.client.method"] = resolved_method + return attrs + + def record_logical_operation( + self, + duration: float, + client_method: str, + rpc_method: str, + error_type: str = "OK", + ) -> None: + """Records end-to-end logical operation duration and count.""" + if not self.standard_enabled or self.meter is None: + return + attrs = { + **self._base_attributes, + "gcp.storage.operation": rpc_method, + "gcp.client.method": client_method, + "error.type": error_type, + } + self.client_request_duration.record(max(0.0, duration), attributes=attrs) + self.operations_counter.add(1, attributes=attrs) + + def record_http_attempt( + self, + duration: float, + http_method: str, + url_template: str, + status_code: Optional[int] = None, + error_type: str = "OK", + operation: Optional[str] = None, + client_method: Optional[str] = None, + network_path: Optional[str] = None, + ) -> None: + """Records a single physical HTTP request attempt.""" + if not self.standard_enabled or self.meter is None: + return + current_op = _OP_STATE.get() + resolved_op = operation or (current_op.rpc_method if current_op else None) + resolved_method = client_method or ( + current_op.client_method if current_op else None + ) + valid_code = ( + isinstance(status_code, int) + and not isinstance(status_code, bool) + and 100 <= status_code < 600 + ) + + # 1. http.client.request.duration attributes + http_attrs: Dict[str, Any] = { + **self._base_attributes, + "http.request.method": http_method.upper() if http_method else "UNKNOWN", + "url.template": url_template, + "error.type": error_type, + } + if valid_code: + http_attrs["http.response.status_code"] = status_code + if network_path is not None: + http_attrs["network.path"] = network_path + + # 2. gcp.storage.client.attempts attributes + attempt_attrs: Dict[str, Any] = { + **self._base_attributes, + "error.type": error_type, + } + if resolved_op is not None: + attempt_attrs["gcp.storage.operation"] = resolved_op + if valid_code: + attempt_attrs["http.response.status_code"] = status_code + if network_path is not None: + attempt_attrs["network.path"] = network_path + + self.http_request_duration.record(max(0.0, duration), attributes=http_attrs) + self.attempts_counter.add(1, attributes=attempt_attrs) + + # 3. gcp.storage.client.errors attributes + if error_type != "OK": + error_attrs: Dict[str, Any] = { + **self._base_attributes, + "error.type": error_type, + } + if resolved_op is not None: + error_attrs["gcp.storage.operation"] = resolved_op + if resolved_method is not None: + error_attrs["gcp.client.method"] = resolved_method + self.errors_counter.add(1, attributes=error_attrs) + + def record_rpc_attempt( + self, + duration: float, + operation: str, + status_code: Optional[str] = None, + error_type: str = "OK", + client_method: Optional[str] = None, + network_path: Optional[str] = None, + ) -> None: + """Records a single physical gRPC call attempt.""" + if not self.standard_enabled or self.meter is None: + return + op_attrs = self._build_operation_attributes( + operation=operation, client_method=client_method + ) + + # 1. rpc.client.call.duration attributes + rpc_attrs: Dict[str, Any] = { + **op_attrs, + "error.type": error_type, + } + if status_code is not None: + rpc_attrs["rpc.response.status_code"] = status_code + if network_path is not None: + rpc_attrs["network.path"] = network_path + + # 2. gcp.storage.client.attempts attributes + attempt_attrs: Dict[str, Any] = { + **self._base_attributes, + "gcp.storage.operation": operation, + "error.type": error_type, + } + if status_code is not None: + attempt_attrs["rpc.response.status_code"] = status_code + if network_path is not None: + attempt_attrs["network.path"] = network_path + + self.rpc_call_duration.record(max(0.0, duration), attributes=rpc_attrs) + self.attempts_counter.add(1, attributes=attempt_attrs) + + # 3. gcp.storage.client.errors attributes + if error_type != "OK": + error_attrs: Dict[str, Any] = { + **op_attrs, + "error.type": error_type, + } + self.errors_counter.add(1, attributes=error_attrs) + + def record_request_body_size( + self, + size: int, + operation: Optional[str] = None, + client_method: Optional[str] = None, + ) -> None: + """Records the payload byte size of an upload or write operation.""" + if not self.standard_enabled or self.meter is None: + return + if not isinstance(size, int) or isinstance(size, bool): + return + current_op = _OP_STATE.get() + if current_op is not None and current_op.suppressed: + return + attrs = self._build_operation_attributes( + operation=operation, client_method=client_method, current_op=current_op + ) + self.request_body_size.record(max(0, size), attributes=attrs) + + def record_response_body_size( + self, + size: int, + operation: Optional[str] = None, + client_method: Optional[str] = None, + ) -> None: + """Records the payload byte size of a download or read operation.""" + if not self.standard_enabled or self.meter is None: + return + if not isinstance(size, int) or isinstance(size, bool): + return + current_op = _OP_STATE.get() + if current_op is not None and current_op.suppressed: + return + attrs = self._build_operation_attributes( + operation=operation, client_method=client_method, current_op=current_op + ) + self.response_body_size.record(max(0, size), attributes=attrs) + + def record_ttfb( + self, + duration: float, + operation: Optional[str] = None, + client_method: Optional[str] = None, + ) -> None: + """Records time to first response byte (TTFB) in seconds.""" + if not self.standard_enabled or self.meter is None: + return + current_op = _OP_STATE.get() + if current_op is not None and current_op.suppressed: + return + attrs = self._build_operation_attributes( + operation=operation, client_method=client_method, current_op=current_op + ) + self.operation_ttfb.record(max(0.0, duration), attributes=attrs) + + def shutdown(self) -> None: + """Flushes pending metrics on a custom MeterProvider if configured.""" + if self._meter_provider is not None and hasattr( + self._meter_provider, "force_flush" + ): + try: + self._meter_provider.force_flush() + except Exception as exc: + logger.debug( + "Failed to force_flush custom MeterProvider during client shutdown: %s", + exc, + ) diff --git a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py index 966cb8a1c78d..cb392b2f9870 100644 --- a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py +++ b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py @@ -12,13 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio +import concurrent.futures +import http.client import importlib +import socket +import ssl import sys +import types from unittest import mock +from google.api_core import exceptions as api_exceptions +from google.auth import exceptions as auth_exceptions import pytest +import requests.exceptions +import urllib3.exceptions from google.cloud.storage import _opentelemetry_metrics +from google.cloud.storage import exceptions as storage_exceptions from google.cloud.storage.version import __version__ @@ -144,7 +155,6 @@ def test_client_setting_overrides_env_var(monkeypatch): def test_non_boolean_client_setting_raises_type_error(): """Passing a non-boolean setting such as 'false' must raise TypeError.""" - # Argument validation executes before the _ENABLE_METRICS_DEV_GATE check. with pytest.raises(TypeError, match="enable_metrics must be a boolean or None"): _opentelemetry_metrics.is_metrics_enabled(enable_metrics="false") # type: ignore[arg-type] @@ -287,3 +297,972 @@ def test_get_meter(monkeypatch): __version__, meter_provider=mock_provider, ) + + +# --------------------------------------------------------------------------- +# Histogram Bucket Boundaries & Views Tests +# --------------------------------------------------------------------------- +def test_histogram_bucket_boundaries(): + """Verify custom latency and byte-size bucket boundaries match specification.""" + latency = _opentelemetry_metrics.LATENCY_HISTOGRAM_BOUNDARIES + assert latency[0] == 0.0 + assert latency[1] == 0.002 + assert latency[49] == 0.098 + assert latency[50] == 0.1 + assert latency[60] == 0.2 + assert latency[-1] < 300.0 + assert list(latency) == sorted(latency) + + sizes = _opentelemetry_metrics.SIZE_HISTOGRAM_BOUNDARIES + assert len(sizes) == 45 + assert sizes[0] == 0 + assert sizes[1] == 128 * 1024 + assert sizes[32] == 4 * 1024 * 1024 + assert sizes[33] == 8 * 1024 * 1024 + assert sizes[-1] == 16 * 1024 * 1024 * 1024 + assert list(sizes) == sorted(sizes) + + +def test_get_histogram_views_with_and_without_sdk(monkeypatch): + """Verify get_histogram_views builds 6 Views when SDK is installed and [] otherwise.""" + fake_view_mod = types.ModuleType("opentelemetry.sdk.metrics.view") + + class FakeAggregation: + def __init__(self, boundaries): + self.boundaries = boundaries + + class FakeView: + def __init__(self, instrument_name, aggregation): + self.instrument_name = instrument_name + self.aggregation = aggregation + + fake_view_mod.ExplicitBucketHistogramAggregation = FakeAggregation + fake_view_mod.View = FakeView + + with monkeypatch.context() as m: + m.setitem(sys.modules, "opentelemetry", types.ModuleType("opentelemetry")) + m.setitem( + sys.modules, "opentelemetry.sdk", types.ModuleType("opentelemetry.sdk") + ) + m.setitem( + sys.modules, + "opentelemetry.sdk.metrics", + types.ModuleType("opentelemetry.sdk.metrics"), + ) + m.setitem(sys.modules, "opentelemetry.sdk.metrics.view", fake_view_mod) + + views = _opentelemetry_metrics.get_histogram_views() + assert len(views) == 6 + latency_views = [ + v + for v in views + if v.instrument_name in _opentelemetry_metrics._LATENCY_METRIC_NAMES + ] + size_views = [ + v + for v in views + if v.instrument_name in _opentelemetry_metrics._SIZE_METRIC_NAMES + ] + assert len(latency_views) == 4 + assert len(size_views) == 2 + assert ( + latency_views[0].aggregation.boundaries + == _opentelemetry_metrics.LATENCY_HISTOGRAM_BOUNDARIES + ) + assert ( + size_views[0].aggregation.boundaries + == _opentelemetry_metrics.SIZE_HISTOGRAM_BOUNDARIES + ) + + # Without opentelemetry-sdk installed + with monkeypatch.context() as m: + m.setitem(sys.modules, "opentelemetry.sdk.metrics.view", None) + assert _opentelemetry_metrics.get_histogram_views() == [] + + +# --------------------------------------------------------------------------- +# Error Classification (`classify_error_type`) Tests +# --------------------------------------------------------------------------- +def test_classify_error_type_none_and_status_codes(): + assert _opentelemetry_metrics.classify_error_type() == "OK" + assert _opentelemetry_metrics.classify_error_type(None, status_code=200) == "OK" + assert _opentelemetry_metrics.classify_error_type(None, status_code=206) == "OK" + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=True) == "OK" # type: ignore[arg-type] + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=400) + == "INVALID_ARGUMENT" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=401) + == "UNAUTHENTICATED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=403) + == "PERMISSION_DENIED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=404) == "NOT_FOUND" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=408) + == "DEADLINE_EXCEEDED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=409) == "ABORTED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=412) + == "FAILED_PRECONDITION" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=416) + == "OUT_OF_RANGE" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=429) + == "RESOURCE_EXHAUSTED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=499) == "CANCELLED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=500) == "INTERNAL" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=501) + == "UNIMPLEMENTED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=503) + == "UNAVAILABLE" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=504) + == "DEADLINE_EXCEEDED" + ) + assert _opentelemetry_metrics.classify_error_type(None, status_code=502) == "502" + assert _opentelemetry_metrics.classify_error_type(None, status_code=99999) == "OK" + + +def test_classify_error_type_exceptions(): + # 1. Checksum Mismatch (including subclasses of DataCorruption) + resp_mock = mock.Mock(status_code=200) + assert ( + _opentelemetry_metrics.classify_error_type( + storage_exceptions.DataCorruption(resp_mock, "crc mismatch") + ) + == "CHECKSUM_MISMATCH" + ) + + class CustomDataCorruption(storage_exceptions.DataCorruption): + pass + + assert ( + _opentelemetry_metrics.classify_error_type( + CustomDataCorruption(resp_mock, "subclass mismatch") + ) + == "CHECKSUM_MISMATCH" + ) + + # 2. Authentication Errors + assert ( + _opentelemetry_metrics.classify_error_type( + auth_exceptions.RefreshError("token expired") + ) + == "AUTHENTICATION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + auth_exceptions.TransportError("auth transport error") + ) + == "AUTHENTICATION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + auth_exceptions.DefaultCredentialsError("missing adc") + ) + == "AUTHENTICATION_ERROR" + ) + + # 3. Cancellation + assert ( + _opentelemetry_metrics.classify_error_type(asyncio.CancelledError()) + == "CANCELLED" + ) + + # 4. HTTP Conflict (409) consistency with status_code=409 + conflict_exc = api_exceptions.Conflict("conflict") + assert _opentelemetry_metrics.classify_error_type(conflict_exc) == "ABORTED" + + # 5. gRPC / GoogleAPICallError status codes + not_found_exc = api_exceptions.NotFound("not found") + assert _opentelemetry_metrics.classify_error_type(not_found_exc) == "NOT_FOUND" + + custom_api_exc = api_exceptions.GoogleAPICallError("custom") + custom_api_exc.grpc_status_code = "RAW_GRPC_STATUS" # type: ignore[assignment] + assert ( + _opentelemetry_metrics.classify_error_type(custom_api_exc) == "RAW_GRPC_STATUS" + ) + + # Direct grpc.RpcError-like exception with .code() method + class FakeStatusCode: + name = "DEADLINE_EXCEEDED" + + class FakeRpcError(Exception): + def code(self): + return FakeStatusCode() + + assert ( + _opentelemetry_metrics.classify_error_type(FakeRpcError()) + == "DEADLINE_EXCEEDED" + ) + + class FakeRpcErrorNoName(Exception): + def code(self): + return object() + + assert _opentelemetry_metrics.classify_error_type(FakeRpcErrorNoName()) == "UNKNOWN" + + # 6. HTTP status code on exception + http_api_exc = api_exceptions.GoogleAPICallError("bad gateway") + http_api_exc.code = 502 + http_api_exc.grpc_status_code = None + assert _opentelemetry_metrics.classify_error_type(http_api_exc) == "502" + + invalid_resp_with_code = storage_exceptions.InvalidResponse( + mock.Mock(status_code=429), "rate limited" + ) + assert ( + _opentelemetry_metrics.classify_error_type(invalid_resp_with_code) + == "RESOURCE_EXHAUSTED" + ) + + req_http_err = requests.exceptions.HTTPError(response=mock.Mock(status_code=503)) + assert _opentelemetry_metrics.classify_error_type(req_http_err) == "UNAVAILABLE" + + # 7. Timeouts + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ConnectTimeout("connect timeout") + ) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ReadTimeout("read timeout") + ) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type(socket.timeout("timed out")) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + urllib3.exceptions.ReadTimeoutError(None, "/", "read timed out") # type: ignore[arg-type] + ) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type(concurrent.futures.TimeoutError()) + == "TIMEOUT" + ) + retry_err = api_exceptions.RetryError( + "deadline exceeded", cause=socket.timeout("timed out") + ) + assert _opentelemetry_metrics.classify_error_type(retry_err) == "TIMEOUT" + + # 8. TLS / SSL Failure + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.SSLError("cert verify failed") + ) + == "TLS_FAILURE" + ) + assert ( + _opentelemetry_metrics.classify_error_type(ssl.SSLError("ssl handshake error")) + == "TLS_FAILURE" + ) + + # 9. DNS, Connection Refused, Connection Reset (direct and wrapped in requests/urllib3) + gai_err = socket.gaierror(-2, "Name or service not known") + new_conn_err = urllib3.exceptions.NewConnectionError(None, "dns failed") # type: ignore[arg-type] + new_conn_err.__cause__ = gai_err + wrapped_dns = requests.exceptions.ConnectionError( + urllib3.exceptions.MaxRetryError( + None, # type: ignore[arg-type] + "/", + reason=new_conn_err, + ) + ) + assert _opentelemetry_metrics.classify_error_type(wrapped_dns) == "DNS_FAILURE" + + refused_err = ConnectionRefusedError(111, "Connection refused") + wrapped_refused = requests.exceptions.ConnectionError(refused_err) + assert ( + _opentelemetry_metrics.classify_error_type(wrapped_refused) + == "CONNECTION_REFUSED" + ) + + reset_err = ConnectionResetError(104, "Connection reset by peer") + wrapped_reset = requests.exceptions.ConnectionError(reset_err) + assert ( + _opentelemetry_metrics.classify_error_type(wrapped_reset) == "CONNECTION_RESET" + ) + + # 10. Generic Connectivity + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ConnectionError("connection aborted") + ) + == "CONNECTIVITY" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ChunkedEncodingError("incomplete read") + ) + == "CONNECTIVITY" + ) + assert ( + _opentelemetry_metrics.classify_error_type(OSError("network unreachable")) + == "CONNECTIVITY" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + http.client.IncompleteRead(b"partial", 10) + ) + == "CONNECTIVITY" + ) + + # 11. Explicit status_code fallback when exception is generic + assert ( + _opentelemetry_metrics.classify_error_type( + RuntimeError("unexpected"), status_code=503 + ) + == "UNAVAILABLE" + ) + + # 12. Client Validation Errors + assert ( + _opentelemetry_metrics.classify_error_type(ValueError("bad arg")) + == "CLIENT_VALIDATION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type(TypeError("bad type")) + == "CLIENT_VALIDATION_ERROR" + ) + invalid_resp_no_code = storage_exceptions.InvalidResponse( + mock.Mock(status_code=None), "missing header" + ) + assert ( + _opentelemetry_metrics.classify_error_type(invalid_resp_no_code) + == "CLIENT_VALIDATION_ERROR" + ) + + # 13. Cyclic exception chain and unknown exception + e1 = RuntimeError("cycle 1") + e2 = RuntimeError("cycle 2", e1) + e1.__cause__ = e2 + assert _opentelemetry_metrics.classify_error_type(e1) == "UNKNOWN" + + +# --------------------------------------------------------------------------- +# Route Normalization (`normalize_url_template`) Tests +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "raw_url,expected", + [ + (None, "UNKNOWN"), + ("", "UNKNOWN"), + (123, "UNKNOWN"), + ("https://storage.googleapis.com", "UNKNOWN"), + ("/", "UNKNOWN"), + ("///", "UNKNOWN"), + ( + "https://storage.googleapis.com//storage/v1/b/my-bucket//", + "/storage/v1/b/{bucket}", + ), + ( + "https://storage.googleapis.com/upload/storage/v1/b/my-bucket/o/my-obj?uploadType=resumable", + "/upload/storage/v1/b/{bucket}/o", + ), + ( + "https://storage.googleapis.com/custom/path?tag=my_uploads", + "UNKNOWN", + ), + ( + "https://storage.googleapis.com/custom/path?my_uploadId=123", + "UNKNOWN", + ), + ( + "https://storage.googleapis.com/storage/v1/b?project=my-proj", + "/storage/v1/b", + ), + ("/b?project=my-proj", "/storage/v1/b"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket?projection=full", + "/storage/v1/b/{bucket}", + ), + ("/b/my-bucket/", "/storage/v1/b/{bucket}"), + ("b/my-bucket", "/storage/v1/b/{bucket}"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o?prefix=foo", + "/storage/v1/b/{bucket}/o", + ), + ("/b/my-bucket/o", "/storage/v1/b/{bucket}/o"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/folder%2Ffile.txt?generation=1", + "/storage/v1/b/{bucket}/o/{object}", + ), + ("/b/my-bucket/o/file.txt", "/storage/v1/b/{bucket}/o/{object}"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/dest.txt/compose", + "/storage/v1/b/{bucket}/o/{object}/compose", + ), + ( + "/b/my-bucket/o/dest.txt/compose", + "/storage/v1/b/{bucket}/o/{object}/compose", + ), + ( + "https://storage.googleapis.com/storage/v1/b/src-b/o/src-o/rewriteTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}", + ), + ( + "/b/src-b/o/src-o/rewriteTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/src-b/o/src-o/copyTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/copyTo/b/{bucket}/o/{object}", + ), + ( + "/b/src-b/o/src-o/copyTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/copyTo/b/{bucket}/o/{object}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/src-o/moveTo/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}", + ), + ( + "/b/my-bucket/o/src-o/moveTo/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/my-o/restore", + "/storage/v1/b/{bucket}/o/{object}/restore", + ), + ( + "/b/my-bucket/o/my-o/restore", + "/storage/v1/b/{bucket}/o/{object}/restore", + ), + ( + "/b/my-bucket/o/my-o/acl", + "/storage/v1/b/{bucket}/o/{object}/acl", + ), + ( + "/b/my-bucket/o/my-o/acl/allUsers", + "/storage/v1/b/{bucket}/o/{object}/acl/{entity}", + ), + ( + "/b/my-bucket/o/my-o/iam", + "/storage/v1/b/{bucket}/o/{object}/iam", + ), + ( + "/b/my-bucket/o/my-o/iam/testPermissions", + "/storage/v1/b/{bucket}/o/{object}/iam", + ), + ( + "/b/my-bucket/acl", + "/storage/v1/b/{bucket}/acl", + ), + ( + "/b/my-bucket/acl/allUsers", + "/storage/v1/b/{bucket}/acl/{entity}", + ), + ( + "/b/my-bucket/defaultObjectAcl", + "/storage/v1/b/{bucket}/defaultObjectAcl", + ), + ( + "/b/my-bucket/defaultObjectAcl/allUsers", + "/storage/v1/b/{bucket}/defaultObjectAcl/{entity}", + ), + ( + "/b/my-bucket/restore", + "/storage/v1/b/{bucket}/restore", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/iam/testPermissions", + "/storage/v1/b/{bucket}/iam", + ), + ("/b/my-bucket/iam", "/storage/v1/b/{bucket}/iam"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/notificationConfigs", + "/storage/v1/b/{bucket}/notificationConfigs", + ), + ( + "/b/my-bucket/notificationConfigs", + "/storage/v1/b/{bucket}/notificationConfigs", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/notificationConfigs/42", + "/storage/v1/b/{bucket}/notificationConfigs/{notification}", + ), + ( + "/b/my-bucket/notificationConfigs/42", + "/storage/v1/b/{bucket}/notificationConfigs/{notification}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/lockRetentionPolicy", + "/storage/v1/b/{bucket}/lockRetentionPolicy", + ), + ( + "/b/my-bucket/lockRetentionPolicy", + "/storage/v1/b/{bucket}/lockRetentionPolicy", + ), + ( + "/storage/v1/projects/my-proj/serviceAccount", + "/storage/v1/projects/{project}/serviceAccount", + ), + ( + "/projects/my-proj/serviceAccount", + "/storage/v1/projects/{project}/serviceAccount", + ), + ( + "/storage/v1/projects/my-proj/hmacKeys", + "/storage/v1/projects/{project}/hmacKeys", + ), + ("/projects/my-proj/hmacKeys", "/storage/v1/projects/{project}/hmacKeys"), + ( + "/storage/v1/projects/my-proj/hmacKeys/GOOG123", + "/storage/v1/projects/{project}/hmacKeys/{access_id}", + ), + ( + "/projects/my-proj/hmacKeys/GOOG123", + "/storage/v1/projects/{project}/hmacKeys/{access_id}", + ), + ( + "https://storage.googleapis.com/upload/storage/v1/b/my-bucket/o?uploadType=resumable", + "/upload/storage/v1/b/{bucket}/o", + ), + ( + "https://storage.googleapis.com/download/storage/v1/b/my-bucket/o/my-obj?alt=media", + "/download/storage/v1/b/{bucket}/o/{object}", + ), + ("https://storage.googleapis.com/batch/storage/v1", "/batch/storage/v1"), + ( + "https://storage.googleapis.com/my-bucket/my-object?uploads", + "/{bucket}/{object}", + ), + ( + "https://storage.googleapis.com/my-bucket/nested/dir/object.bin?uploadId=xyz&partNumber=2", + "/{bucket}/{object}", + ), + ( + "https://storage.googleapis.com/unrecognized/two-segment", + "UNKNOWN", + ), + ("https://storage.googleapis.com/storage/v2", "UNKNOWN"), + ("https://storage.googleapis.com/storage/v2/other?uploadId=xyz", "UNKNOWN"), + ("https://storage.googleapis.com/custom/unrecognized/path/here", "UNKNOWN"), + ], +) +def test_normalize_url_template(raw_url, expected): + assert _opentelemetry_metrics.normalize_url_template(raw_url) == expected + + +# --------------------------------------------------------------------------- +# Logical Operation Context (`start_operation`) Tests +# --------------------------------------------------------------------------- +def test_start_operation_disabled_and_nesting(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + # Disabled recorder -> yields None and records nothing + disabled_recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=False) + with _opentelemetry_metrics.start_operation( + disabled_recorder, "storage.Blob.reload", "storage.objects.get" + ) as op: + assert op is None + assert _opentelemetry_metrics.get_current_operation() is None + + # Enabled recorder -> outermost call records, nested call is suppressed + enabled_recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + with _opentelemetry_metrics.start_operation( + enabled_recorder, + "storage.Blob.download_to_filename", + "storage.objects.get", + ) as outer_op: + assert outer_op is not None + assert outer_op.client_method == "storage.Blob.download_to_filename" + assert outer_op.rpc_method == "storage.objects.get" + assert _opentelemetry_metrics.get_current_operation() == outer_op + + with _opentelemetry_metrics.start_operation( + enabled_recorder, + "storage.Blob.download_to_file", + "storage.objects.get", + ) as inner_op: + assert inner_op is outer_op + + assert _opentelemetry_metrics.get_current_operation() is None + enabled_recorder.client_request_duration.record.assert_called_once() + enabled_recorder.operations_counter.add.assert_called_once() + _, kwargs = enabled_recorder.operations_counter.add.call_args + assert ( + kwargs["attributes"]["gcp.client.method"] == "storage.Blob.download_to_filename" + ) + assert kwargs["attributes"]["gcp.storage.operation"] == "storage.objects.get" + assert kwargs["attributes"]["error.type"] == "OK" + + +def test_start_operation_exception_and_suppression(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + + with pytest.raises(api_exceptions.NotFound): + with _opentelemetry_metrics.start_operation( + recorder, "storage.Bucket.get_blob", "storage.objects.get" + ): + raise api_exceptions.NotFound("missing") + + assert _opentelemetry_metrics.get_current_operation() is None + _, kwargs = recorder.operations_counter.add.call_args + assert kwargs["attributes"]["error.type"] == "NOT_FOUND" + + # Suppressed outer operation (e.g. transfer_manager worker) suppresses both + # logical operation metrics and body size recording + recorder.operations_counter.add.reset_mock() + recorder.client_request_duration.record.reset_mock() + with _opentelemetry_metrics.start_operation( + None, + "storage.transfer_manager.download_many_to_path", + "storage.objects.get", + suppress=True, + ) as suppressed_op: + assert suppressed_op is not None + assert suppressed_op.suppressed is True + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.download_to_filename", + "storage.objects.get", + ): + recorder.record_request_body_size(2048) + recorder.record_response_body_size(4096) + + assert _opentelemetry_metrics.get_current_operation() is None + recorder.operations_counter.add.assert_not_called() + recorder.client_request_duration.record.assert_not_called() + recorder.request_body_size.record.assert_not_called() + recorder.response_body_size.record.assert_not_called() + + # Nested suppress=True inside an active unsuppressed operation (e.g. BlobReader prefetch) + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.open", + "storage.objects.get", + ) as active_op: + assert active_op is not None and active_op.suppressed is False + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.download_as_bytes", + "storage.objects.get", + suppress=True, + ) as child_suppressed: + assert child_suppressed is not None + assert child_suppressed.suppressed is True + assert child_suppressed.client_method == "storage.Blob.open" + # Prefetch body size and TTFB are suppressed inside child_suppressed + recorder.record_response_body_size(40 * 1024 * 1024) + recorder.record_ttfb(0.05) + # Outside the suppressed prefetch block, actual user bytes are recorded + assert _opentelemetry_metrics.get_current_operation() == active_op + recorder.record_response_body_size(1024) + recorder.record_response_body_size(True) # type: ignore[arg-type] + recorder.record_request_body_size(False) # type: ignore[arg-type] + + recorder.response_body_size.record.assert_called_once() + assert recorder.response_body_size.record.call_args[0][0] == 1024 + recorder.operation_ttfb.record.assert_not_called() + recorder.operations_counter.add.assert_called_once() + + +def test_start_operation_asyncio_concurrency_isolation(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + + async def worker(method_name: str, rpc_name: str): + with _opentelemetry_metrics.start_operation(recorder, method_name, rpc_name): + await asyncio.sleep(0.005) + current = _opentelemetry_metrics.get_current_operation() + assert current is not None + assert current.client_method == method_name + assert current.rpc_method == rpc_name + + async def run_all(): + await asyncio.gather( + worker( + "storage.AsyncMultiRangeDownloader.download_ranges", + "google.storage.v2.Storage/BidiReadObject", + ), + worker( + "storage.AsyncAppendableObjectWriter.append", + "google.storage.v2.Storage/BidiWriteObject", + ), + ) + + asyncio.run(run_all()) + assert recorder.operations_counter.add.call_count == 2 + + +# --------------------------------------------------------------------------- +# `_extract_server_address` & `_MetricsRecorder` Tests +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "endpoint,expected", + [ + (None, "storage.googleapis.com"), + ("", "storage.googleapis.com"), + (" ", "storage.googleapis.com"), + ("https://storage.googleapis.com:443", "storage.googleapis.com"), + ("storage.googleapis.com:443", "storage.googleapis.com"), + ("//storage.googleapis.com:443", "storage.googleapis.com"), + ("https://[::1]:8080", "::1"), + ("http://localhost:9023/storage/v1", "localhost"), + ("https://", "storage.googleapis.com"), + ], +) +def test_extract_server_address(endpoint, expected): + assert _opentelemetry_metrics._extract_server_address(endpoint) == expected + + +def test_metrics_recorder_recording_and_shutdown(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + mock_provider = mock.Mock() + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, + meter_provider=mock_provider, + api_endpoint="https://custom.storage.googleapis.com:443", + rpc_system="http", + ) + assert recorder.standard_enabled is True + assert recorder.enabled is True + assert recorder.server_address == "custom.storage.googleapis.com" + + base_attrs = recorder.get_base_attributes() + assert base_attrs["server.address"] == "custom.storage.googleapis.com" + assert base_attrs["rpc.system.name"] == "http" + + # Verify exact instrument names, units, and descriptions + histogram_specs = { + call.args[0]: (call.kwargs["unit"], call.kwargs["description"]) + for call in mock_meter.create_histogram.call_args_list + } + assert histogram_specs == { + "rpc.client.call.duration": ( + "s", + "Duration of an individual gRPC attempt in seconds (excluding retries).", + ), + "http.client.request.duration": ( + "s", + "Duration of an individual HTTP request attempt in seconds (excluding retries).", + ), + "gcp.client.request.duration": ( + "s", + "End-to-end latency of a logical user-facing Veneer operation in seconds.", + ), + "gcp.storage.client.request.body.size": ( + "By", + "Application payload bytes submitted by the caller for an upload or write operation.", + ), + "gcp.storage.client.response.body.size": ( + "By", + "Application payload bytes delivered to the caller.", + ), + "gcp.storage.client.operation.ttfb": ( + "s", + "Time elapsed from attempt initiation to receipt of the first response byte.", + ), + } + counter_specs = { + call.args[0]: (call.kwargs["unit"], call.kwargs["description"]) + for call in mock_meter.create_counter.call_args_list + } + assert counter_specs == { + "gcp.storage.client.operations": ( + "{operation}", + "Number of logical GCS client operations.", + ), + "gcp.storage.client.attempts": ( + "{attempt}", + "Number of physical network transport attempts executed.", + ), + "gcp.storage.client.errors": ( + "{error}", + "Number of physical network transport attempts terminating with a non-OK status.", + ), + } + + # Record within a logical operation context + with _opentelemetry_metrics.start_operation( + recorder, "storage.Blob.upload_from_string", "storage.objects.insert" + ): + recorder.record_http_attempt( + duration=0.05, + http_method="post", + url_template="/upload/storage/v1/b/{bucket}/o", + status_code=503, + error_type="UNAVAILABLE", + network_path="cloudpath", + ) + recorder.record_http_attempt( + duration=0.04, + http_method="POST", + url_template="/upload/storage/v1/b/{bucket}/o", + status_code=200, + error_type="OK", + ) + recorder.record_request_body_size(1024) + recorder.record_response_body_size(512) + recorder.record_ttfb(0.01) + + assert recorder.http_request_duration.record.call_count == 2 + _, http_dur_kwargs = recorder.http_request_duration.record.call_args_list[0] + assert http_dur_kwargs["attributes"]["http.request.method"] == "POST" + assert ( + http_dur_kwargs["attributes"]["url.template"] + == "/upload/storage/v1/b/{bucket}/o" + ) + assert http_dur_kwargs["attributes"]["http.response.status_code"] == 503 + assert http_dur_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert http_dur_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.storage.operation" not in http_dur_kwargs["attributes"] + assert "gcp.client.method" not in http_dur_kwargs["attributes"] + + assert recorder.attempts_counter.add.call_count == 2 + _, attempt_kwargs = recorder.attempts_counter.add.call_args_list[0] + assert ( + attempt_kwargs["attributes"]["gcp.storage.operation"] + == "storage.objects.insert" + ) + assert attempt_kwargs["attributes"]["http.response.status_code"] == 503 + assert attempt_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert attempt_kwargs["attributes"]["network.path"] == "cloudpath" + assert "url.template" not in attempt_kwargs["attributes"] + assert "http.request.method" not in attempt_kwargs["attributes"] + + # Only the 503 attempt records an error + recorder.errors_counter.add.assert_called_once() + _, err_kwargs = recorder.errors_counter.add.call_args + assert err_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert err_kwargs["attributes"]["gcp.storage.operation"] == "storage.objects.insert" + assert ( + err_kwargs["attributes"]["gcp.client.method"] + == "storage.Blob.upload_from_string" + ) + assert "url.template" not in err_kwargs["attributes"] + assert "http.response.status_code" not in err_kwargs["attributes"] + + recorder.request_body_size.record.assert_called_once() + recorder.response_body_size.record.assert_called_once() + recorder.operation_ttfb.record.assert_called_once() + + # gRPC attempt recording (OK and non-OK) + recorder.errors_counter.add.reset_mock() + recorder.attempts_counter.add.reset_mock() + recorder.record_rpc_attempt( + duration=0.02, + operation="google.storage.v2.Storage/BidiReadObject", + status_code="OK", + error_type="OK", + client_method="storage.AsyncMultiRangeDownloader.download_ranges", + network_path="directpath", + ) + recorder.record_rpc_attempt( + duration=0.03, + operation="google.storage.v2.Storage/BidiReadObject", + status_code="UNAVAILABLE", + error_type="UNAVAILABLE", + client_method="storage.AsyncMultiRangeDownloader.download_ranges", + ) + assert recorder.rpc_call_duration.record.call_count == 2 + _, rpc_dur_kwargs = recorder.rpc_call_duration.record.call_args_list[0] + assert ( + rpc_dur_kwargs["attributes"]["gcp.storage.operation"] + == "google.storage.v2.Storage/BidiReadObject" + ) + assert ( + rpc_dur_kwargs["attributes"]["gcp.client.method"] + == "storage.AsyncMultiRangeDownloader.download_ranges" + ) + assert rpc_dur_kwargs["attributes"]["rpc.response.status_code"] == "OK" + assert rpc_dur_kwargs["attributes"]["network.path"] == "directpath" + + assert recorder.attempts_counter.add.call_count == 2 + _, rpc_attempt_kwargs = recorder.attempts_counter.add.call_args_list[0] + assert ( + rpc_attempt_kwargs["attributes"]["gcp.storage.operation"] + == "google.storage.v2.Storage/BidiReadObject" + ) + assert rpc_attempt_kwargs["attributes"]["rpc.response.status_code"] == "OK" + assert rpc_attempt_kwargs["attributes"]["error.type"] == "OK" + assert rpc_attempt_kwargs["attributes"]["network.path"] == "directpath" + assert "gcp.client.method" not in rpc_attempt_kwargs["attributes"] + + recorder.errors_counter.add.assert_called_once() + _, rpc_err_kwargs = recorder.errors_counter.add.call_args + assert rpc_err_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert "rpc.response.status_code" not in rpc_err_kwargs["attributes"] + + # Shutdown flushes custom meter_provider and swallows force_flush errors gracefully + recorder.shutdown() + mock_provider.force_flush.assert_called_once() + + mock_provider.force_flush.side_effect = RuntimeError("flush failed") + recorder.shutdown() # Must not raise + + +def test_metrics_recorder_disabled_no_ops(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", False) + recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + assert recorder.standard_enabled is False + assert recorder.enabled is False + + # All recording methods are safe no-ops when disabled + recorder.record_logical_operation(0.1, "m", "op") + recorder.record_http_attempt(0.1, "GET", "/storage/v1/b") + recorder.record_rpc_attempt(0.1, "google.storage.v2.Storage/ReadObject") + recorder.record_request_body_size(100) + recorder.record_response_body_size(100) + recorder.record_ttfb(0.01) + recorder.shutdown() From 6403d5db4ff641de86f38da91e185155be4fcb4b Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Mon, 19 Oct 2026 15:10:34 +0000 Subject: [PATCH 2/7] feat(storage): add OpenTelemetry standard metrics for Async gRPC client Implement OpenTelemetry standard metrics for AsyncGrpcClient, AsyncMultiRangeDownloader, AsyncAppendableObjectWriter, AsyncReadObjectStream, AsyncWriteObjectStream, and _StreamMultiplexer following the GCS O11y specification. [Generated-by: AI] --- .../cloud/storage/_opentelemetry_metrics.py | 16 +- .../storage/asyncio/_stream_multiplexer.py | 7 + .../asyncio/async_appendable_object_writer.py | 606 +++++++++------- .../storage/asyncio/async_grpc_client.py | 654 ++++++++++++++++-- .../asyncio/async_multi_range_downloader.py | 460 ++++++------ .../asyncio/async_read_object_stream.py | 22 +- .../asyncio/async_write_object_stream.py | 55 +- .../test_async_appendable_object_writer.py | 68 ++ .../unit/asyncio/test_async_grpc_client.py | 349 +++++++++- .../test_async_multi_range_downloader.py | 113 +++ .../asyncio/test_async_write_object_stream.py | 52 ++ .../tests/unit/test__opentelemetry_metrics.py | 40 +- 12 files changed, 1912 insertions(+), 530 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py index c357a968407f..6f8e75269aa6 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py +++ b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py @@ -777,7 +777,9 @@ def start_operation( yield existing return - recorder_active = recorder is not None and recorder.standard_enabled + recorder_active = ( + recorder is not None and getattr(recorder, "standard_enabled", False) is True + ) if not recorder_active and not suppress: yield None return @@ -807,6 +809,14 @@ def start_operation( ) +def get_active_recorder(obj: Any) -> Optional["_MetricsRecorder"]: + """Returns ``obj._metrics_recorder`` if it is an active ``_MetricsRecorder``.""" + recorder = getattr(obj, "_metrics_recorder", None) + if recorder is not None and getattr(recorder, "standard_enabled", False) is True: + return recorder + return None + + # --------------------------------------------------------------------------- # 10. Server Address Parsing & `_MetricsRecorder` # --------------------------------------------------------------------------- @@ -817,7 +827,9 @@ def _extract_server_address(api_endpoint: Optional[str]) -> str: endpoint = api_endpoint.strip() if not endpoint: return DEFAULT_SERVER_ADDRESS - if "://" not in endpoint: + if ":///" in endpoint: + endpoint = "https://" + endpoint.split(":///", 1)[1].lstrip("/") + elif "://" not in endpoint: endpoint = "https://" + endpoint.lstrip("/") parsed = urlsplit(endpoint) return parsed.hostname or DEFAULT_SERVER_ADDRESS diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py index 6251266b7240..7f5bb8f3c88b 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +import inspect import logging from typing import Awaitable, Callable, Dict, Optional, Set @@ -193,6 +194,9 @@ async def reopen_stream( async with self._reopen_lock: if self._stream_generation != broken_generation: return + mark_clean = getattr(self._stream, "_mark_closing_cleanly", None) + if callable(mark_clean) and not inspect.iscoroutinefunction(mark_clean): + mark_clean() self._stop_recv_loop() if self._recv_task: try: @@ -211,6 +215,9 @@ async def reopen_stream( self._ensure_recv_loop() async def close(self) -> None: + mark_clean = getattr(self._stream, "_mark_closing_cleanly", None) + if callable(mark_clean) and not inspect.iscoroutinefunction(mark_clean): + mark_clean() self._stop_recv_loop() if self._recv_task: try: diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py index 1af54d6ffe43..c1cb5d0d27fb 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py @@ -24,7 +24,7 @@ from google.cloud import _storage_v2 from google.cloud._storage_v2.types import BidiWriteObjectRedirectedError from google.cloud._storage_v2.types.storage import BidiWriteObjectRequest -from google.cloud.storage import Blob +from google.cloud.storage import Blob, _opentelemetry_metrics from google.cloud.storage.asyncio.async_grpc_client import ( AsyncGrpcClient, ) @@ -46,6 +46,7 @@ _MAX_CHUNK_SIZE_BYTES = 2 * 1024 * 1024 # 2 MiB _DEFAULT_FLUSH_INTERVAL_BYTES = 16 * 1024 * 1024 # 16 MiB +_BIDI_WRITE_OBJECT_RPC = "google.storage.v2.Storage/BidiWriteObject" _BIDI_WRITE_REDIRECTED_TYPE_URL = ( "type.googleapis.com/google.storage.v2.BidiWriteObjectRedirectedError" ) @@ -219,6 +220,7 @@ def __init__( self.object_resource: Optional[_storage_v2.Object] = None self._flush_count = 0 self.blob: Optional[Blob] = None + self._suppress_append_body_size: bool = False @classmethod def from_blob( @@ -283,17 +285,25 @@ async def state_lookup(self) -> int: :raises ValueError: If the stream is not open (i.e., `open()` has not been called). """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before state_lookup().") + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.state_lookup", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + if not self._is_stream_open: + raise ValueError( + "Stream is not open. Call open() before state_lookup()." + ) - await self.write_obj_stream.send( - _storage_v2.BidiWriteObjectRequest( - state_lookup=True, + await self.write_obj_stream.send( + _storage_v2.BidiWriteObjectRequest( + state_lookup=True, + ) ) - ) - response = await self.write_obj_stream.recv() - self.persisted_size = response.persisted_size - return self.persisted_size + response = await self.write_obj_stream.recv() + self.persisted_size = response.persisted_size + return self.persisted_size def _on_open_error(self, exc): """Extracts routing token and write handle on redirect error during open.""" @@ -340,59 +350,70 @@ async def open( :raises ValueError: If the stream is already open. """ - if self._is_stream_open: - raise ValueError("Underlying bidi-gRPC stream is already open") - - retry_policy = self._merge_retry_policy(retry_policy) - - async def _do_open(): - current_metadata = list(metadata) if metadata else [] + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.open", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + if self._is_stream_open: + raise ValueError("Underlying bidi-gRPC stream is already open") + + retry_policy = self._merge_retry_policy(retry_policy) + + async def _do_open(): + current_metadata = list(metadata) if metadata else [] + + # Cleanup stream from previous failed attempt, if any. + if self.write_obj_stream: + if self.write_obj_stream.is_stream_open: + try: + await self.write_obj_stream.close() + except Exception as e: + logger.warning( + f"Error closing previous write stream during open retry. Got exception: {e}" + ) + self.write_obj_stream = None + self._is_stream_open = False - # Cleanup stream from previous failed attempt, if any. - if self.write_obj_stream: - if self.write_obj_stream.is_stream_open: - try: - await self.write_obj_stream.close() - except Exception as e: - logger.warning( - f"Error closing previous write stream during open retry. Got exception: {e}" + self.write_obj_stream = _AsyncWriteObjectStream( + client=self.client.grpc_client, + bucket_name=self.bucket_name, + object_name=self.object_name, + blob=self.blob, + generation_number=self.generation, + write_handle=self.write_handle, + routing_token=self._routing_token, + storage_class=self.storage_class, + ) + if recorder is not None: + self.write_obj_stream._metrics_recorder = recorder + + if self._routing_token: + current_metadata.append( + ( + "x-goog-request-params", + f"routing_token={self._routing_token}", ) - self.write_obj_stream = None - self._is_stream_open = False - - self.write_obj_stream = _AsyncWriteObjectStream( - client=self.client.grpc_client, - bucket_name=self.bucket_name, - object_name=self.object_name, - blob=self.blob, - generation_number=self.generation, - write_handle=self.write_handle, - routing_token=self._routing_token, - storage_class=self.storage_class, - ) + ) - if self._routing_token: - current_metadata.append( - ("x-goog-request-params", f"routing_token={self._routing_token}") + await self.write_obj_stream.open( + metadata=current_metadata if current_metadata else None ) - await self.write_obj_stream.open( - metadata=current_metadata if current_metadata else None - ) - - if self.write_obj_stream.generation_number: - self.generation = self.write_obj_stream.generation_number - if self.write_obj_stream.write_handle: - self.write_handle = self.write_obj_stream.write_handle - if self.write_obj_stream.persisted_size is not None: - self.persisted_size = self.write_obj_stream.persisted_size - # set offset while opening - self.offset = self.persisted_size + if self.write_obj_stream.generation_number: + self.generation = self.write_obj_stream.generation_number + if self.write_obj_stream.write_handle: + self.write_handle = self.write_obj_stream.write_handle + if self.write_obj_stream.persisted_size is not None: + self.persisted_size = self.write_obj_stream.persisted_size + # set offset while opening + self.offset = self.persisted_size - self._is_stream_open = True - self._routing_token = None + self._is_stream_open = True + self._routing_token = None - await retry_policy(_do_open)() + await retry_policy(_do_open)() async def append( self, @@ -425,106 +446,118 @@ async def append( :raises ValueError: If the stream is not open. """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before append().") - if not data: - logger.debug("No data provided to append; returning without action.") - return - - if retry_policy is None: - retry_policy = AsyncRetry(predicate=_is_write_retryable) - - strategy = _WriteResumptionStrategy() - buffer = io.BytesIO(data) - attempt_count = 0 - - def send_and_recv_generator( - requests: List[BidiWriteObjectRequest], - state: Dict[str, _WriteState], - metadata: Optional[List[Tuple[str, str]]] = None, + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.append", + rpc_method=_BIDI_WRITE_OBJECT_RPC, ): - async def generator(): - nonlocal attempt_count - nonlocal requests - attempt_count += 1 - resp = None - write_state = state["write_state"] - # If this is a retry or redirect, we must re-open the stream - if attempt_count > 1 or write_state.routing_token: - logger.info( - f"Re-opening the stream with attempt_count: {attempt_count}" - ) - - current_metadata = list(metadata) if metadata else [] - if write_state.routing_token: - current_metadata.append( - ( - "x-goog-request-params", - f"routing_token={write_state.routing_token}", - ) - ) - self._routing_token = write_state.routing_token - - self._is_stream_open = False - await self.open(metadata=current_metadata) - - write_state.persisted_size = self.persisted_size - write_state.write_handle = self.write_handle - write_state.routing_token = None - - write_state.user_buffer.seek(write_state.persisted_size) - write_state.bytes_sent = write_state.persisted_size - write_state.bytes_since_last_flush = 0 - self.bytes_appended_since_last_flush = 0 - - requests = strategy.generate_requests(state) - - for chunk_req in requests: - await self.write_obj_stream.send(chunk_req) - if chunk_req.flush: - self._flush_count += 1 - + if not self._is_stream_open: + raise ValueError("Stream is not open. Call open() before append().") + if not data: + logger.debug("No data provided to append; returning without action.") + return + + if retry_policy is None: + retry_policy = AsyncRetry(predicate=_is_write_retryable) + + strategy = _WriteResumptionStrategy() + buffer = io.BytesIO(data) + attempt_count = 0 + + def send_and_recv_generator( + requests: List[BidiWriteObjectRequest], + state: Dict[str, _WriteState], + metadata: Optional[List[Tuple[str, str]]] = None, + ): + async def generator(): + nonlocal attempt_count + nonlocal requests + attempt_count += 1 resp = None - if chunk_req.state_lookup: - # TODO: if there's error, it'll raise error - # and will be handled by `recover_state_on_failure` - resp = await self.write_obj_stream.recv() - - if resp: - if resp.persisted_size is not None: - self.persisted_size = resp.persisted_size - state["write_state"].persisted_size = resp.persisted_size - self.offset = self.persisted_size - if resp.write_handle: - self.write_handle = resp.write_handle - state["write_state"].write_handle = resp.write_handle - - yield resp - - return generator() - - # State initialization - write_state = _WriteState( - _MAX_CHUNK_SIZE_BYTES, - buffer, - self.flush_interval, - enable_checksum=enable_checksum, - ) - write_state.write_handle = self.write_handle - write_state.persisted_size = self.persisted_size - # offset is set during `open()` call. - write_state.bytes_sent = self.offset or 0 - write_state.bytes_since_last_flush = self.bytes_appended_since_last_flush - - retry_manager = _BidiStreamRetryManager( - _WriteResumptionStrategy(), - lambda r, s: send_and_recv_generator(r, s, metadata), - ) - await retry_manager.execute({"write_state": write_state}, retry_policy) + write_state = state["write_state"] + # If this is a retry or redirect, we must re-open the stream + if attempt_count > 1 or write_state.routing_token: + logger.info( + f"Re-opening the stream with attempt_count: {attempt_count}" + ) - # Sync local markers - self.bytes_appended_since_last_flush = write_state.bytes_since_last_flush - self.offset = write_state.bytes_sent + current_metadata = list(metadata) if metadata else [] + if write_state.routing_token: + current_metadata.append( + ( + "x-goog-request-params", + f"routing_token={write_state.routing_token}", + ) + ) + self._routing_token = write_state.routing_token + + self._is_stream_open = False + await self.open(metadata=current_metadata) + + write_state.persisted_size = self.persisted_size + write_state.write_handle = self.write_handle + write_state.routing_token = None + + write_state.user_buffer.seek(write_state.persisted_size) + write_state.bytes_sent = write_state.persisted_size + write_state.bytes_since_last_flush = 0 + self.bytes_appended_since_last_flush = 0 + + requests = strategy.generate_requests(state) + + for chunk_req in requests: + await self.write_obj_stream.send(chunk_req) + if chunk_req.flush: + self._flush_count += 1 + + resp = None + if chunk_req.state_lookup: + # TODO: if there's error, it'll raise error + # and will be handled by `recover_state_on_failure` + resp = await self.write_obj_stream.recv() + + if resp: + if resp.persisted_size is not None: + self.persisted_size = resp.persisted_size + state["write_state"].persisted_size = ( + resp.persisted_size + ) + self.offset = self.persisted_size + if resp.write_handle: + self.write_handle = resp.write_handle + state["write_state"].write_handle = resp.write_handle + + yield resp + + return generator() + + # State initialization + write_state = _WriteState( + _MAX_CHUNK_SIZE_BYTES, + buffer, + self.flush_interval, + enable_checksum=enable_checksum, + ) + write_state.write_handle = self.write_handle + write_state.persisted_size = self.persisted_size + # offset is set during `open()` call. + write_state.bytes_sent = self.offset or 0 + write_state.bytes_since_last_flush = self.bytes_appended_since_last_flush + + retry_manager = _BidiStreamRetryManager( + _WriteResumptionStrategy(), + lambda r, s: send_and_recv_generator(r, s, metadata), + ) + await retry_manager.execute({"write_state": write_state}, retry_policy) + + # Sync local markers + self.bytes_appended_since_last_flush = write_state.bytes_since_last_flush + self.offset = write_state.bytes_sent + if recorder is not None and not self._suppress_append_body_size: + recorder.record_request_body_size( + len(data), operation=_BIDI_WRITE_OBJECT_RPC + ) async def simple_flush(self) -> None: """Flushes the data to the server. @@ -535,15 +568,23 @@ async def simple_flush(self) -> None: :raises ValueError: If the stream is not open (i.e., `open()` has not been called). """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before simple_flush().") + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.simple_flush", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + if not self._is_stream_open: + raise ValueError( + "Stream is not open. Call open() before simple_flush()." + ) - await self.write_obj_stream.send( - _storage_v2.BidiWriteObjectRequest( - flush=True, + await self.write_obj_stream.send( + _storage_v2.BidiWriteObjectRequest( + flush=True, + ) ) - ) - self.bytes_appended_since_last_flush = 0 + self.bytes_appended_since_last_flush = 0 async def flush(self) -> int: """Flushes the data to the server. @@ -554,20 +595,26 @@ async def flush(self) -> int: :raises ValueError: If the stream is not open (i.e., `open()` has not been called). """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before flush().") + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.flush", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + if not self._is_stream_open: + raise ValueError("Stream is not open. Call open() before flush().") - await self.write_obj_stream.send( - _storage_v2.BidiWriteObjectRequest( - flush=True, - state_lookup=True, + await self.write_obj_stream.send( + _storage_v2.BidiWriteObjectRequest( + flush=True, + state_lookup=True, + ) ) - ) - response = await self.write_obj_stream.recv() - self.persisted_size = response.persisted_size - self.offset = self.persisted_size - self.bytes_appended_since_last_flush = 0 - return self.persisted_size + response = await self.write_obj_stream.recv() + self.persisted_size = response.persisted_size + self.offset = self.persisted_size + self.bytes_appended_since_last_flush = 0 + return self.persisted_size async def close( self, @@ -612,51 +659,57 @@ async def close( server. """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before close().") + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.close", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + if not self._is_stream_open: + raise ValueError("Stream is not open. Call open() before close().") - if full_object_checksum is not None and not finalize_on_close: - raise ValueError( - "full_object_checksum can only be provided when finalize_on_close is True." - ) + if full_object_checksum is not None and not finalize_on_close: + raise ValueError( + "full_object_checksum can only be provided when finalize_on_close is True." + ) - if finalize_on_close: - return await self.finalize( - full_object_checksum=full_object_checksum, - retry_policy=retry_policy, - ) + if finalize_on_close: + return await self.finalize( + full_object_checksum=full_object_checksum, + retry_policy=retry_policy, + ) - retry_policy = self._merge_retry_policy(retry_policy) + retry_policy = self._merge_retry_policy(retry_policy) - attempt_count = 0 - expected_offset = self.offset + attempt_count = 0 + expected_offset = self.offset - async def _do_close(): - nonlocal attempt_count - attempt_count += 1 + async def _do_close(): + nonlocal attempt_count + attempt_count += 1 - if attempt_count > 1: - logger.info( - f"Re-opening the stream for close retry attempt: {attempt_count}" - ) - self._is_stream_open = False - await self.open() - if ( - self.offset is not None - and expected_offset is not None - and self.offset != expected_offset - ): - raise exceptions.InternalServerError( - f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" + if attempt_count > 1: + logger.info( + f"Re-opening the stream for close retry attempt: {attempt_count}" ) + self._is_stream_open = False + await self.open() + if ( + self.offset is not None + and expected_offset is not None + and self.offset != expected_offset + ): + raise exceptions.InternalServerError( + f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" + ) - await self.write_obj_stream.close() - return self.persisted_size + await self.write_obj_stream.close() + return self.persisted_size - try: - return await retry_policy(_do_close)() - finally: - self._is_stream_open = False + try: + return await retry_policy(_do_close)() + finally: + self._is_stream_open = False async def finalize( self, @@ -699,67 +752,75 @@ async def finalize( full_object_checksum does not match the checksum computed by the server. """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before finalize().") - - if full_object_checksum is None: - finalize_req = _storage_v2.BidiWriteObjectRequest(finish_write=True) - elif isinstance(full_object_checksum, bool) or not isinstance( - full_object_checksum, int + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.finalize", + rpc_method=_BIDI_WRITE_OBJECT_RPC, ): - raise TypeError("full_object_checksum must be an integer.") - elif not (0 <= full_object_checksum <= 0xFFFFFFFF): - raise ValueError("full_object_checksum must be a 32-bit unsigned integer.") - else: - finalize_req = _storage_v2.BidiWriteObjectRequest( - finish_write=True, - object_checksums=_storage_v2.ObjectChecksums( - crc32c=full_object_checksum - ), - ) + if not self._is_stream_open: + raise ValueError("Stream is not open. Call open() before finalize().") + + if full_object_checksum is None: + finalize_req = _storage_v2.BidiWriteObjectRequest(finish_write=True) + elif isinstance(full_object_checksum, bool) or not isinstance( + full_object_checksum, int + ): + raise TypeError("full_object_checksum must be an integer.") + elif not (0 <= full_object_checksum <= 0xFFFFFFFF): + raise ValueError( + "full_object_checksum must be a 32-bit unsigned integer." + ) + else: + finalize_req = _storage_v2.BidiWriteObjectRequest( + finish_write=True, + object_checksums=_storage_v2.ObjectChecksums( + crc32c=full_object_checksum + ), + ) - retry_policy = self._merge_retry_policy(retry_policy) + retry_policy = self._merge_retry_policy(retry_policy) - attempt_count = 0 - expected_offset = self.offset + attempt_count = 0 + expected_offset = self.offset - async def _do_finalize(): - nonlocal attempt_count - attempt_count += 1 + async def _do_finalize(): + nonlocal attempt_count + attempt_count += 1 - if attempt_count > 1: - logger.info( - f"Re-opening the stream for finalize retry attempt: {attempt_count}" - ) - self._is_stream_open = False - await self.open() - if ( - self.offset is not None - and expected_offset is not None - and self.offset != expected_offset - ): - raise exceptions.InternalServerError( - f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" + if attempt_count > 1: + logger.info( + f"Re-opening the stream for finalize retry attempt: {attempt_count}" ) + self._is_stream_open = False + await self.open() + if ( + self.offset is not None + and expected_offset is not None + and self.offset != expected_offset + ): + raise exceptions.InternalServerError( + f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" + ) - await self.write_obj_stream.send(finalize_req) - response = await self.write_obj_stream.recv() - self.object_resource = response.resource - self.persisted_size = self.object_resource.size - return self.object_resource - - try: - return await retry_policy(_do_finalize)() - finally: - if self.write_obj_stream: - try: - await self.write_obj_stream.close() - except Exception as e: - logger.debug( - f"Stream close during finalize cleanup resulted in: {e}" - ) - self._is_stream_open = False - self.offset = None + await self.write_obj_stream.send(finalize_req) + response = await self.write_obj_stream.recv() + self.object_resource = response.resource + self.persisted_size = self.object_resource.size + return self.object_resource + + try: + return await retry_policy(_do_finalize)() + finally: + if self.write_obj_stream: + try: + await self.write_obj_stream.close() + except Exception as e: + logger.debug( + f"Stream close during finalize cleanup resulted in: {e}" + ) + self._is_stream_open = False + self.offset = None @property def is_stream_open(self) -> bool: @@ -792,5 +853,22 @@ async def append_from_file( :param file_obj: A file handle opened in binary mode for reading. """ - while block := file_obj.read(block_size): - await self.append(block) + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.append_from_file", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + total_bytes = 0 + prev_suppress = self._suppress_append_body_size + self._suppress_append_body_size = True + try: + while block := file_obj.read(block_size): + await self.append(block) + total_bytes += len(block) + finally: + self._suppress_append_body_size = prev_suppress + if recorder is not None and total_bytes > 0: + recorder.record_request_body_size( + total_bytes, operation=_BIDI_WRITE_OBJECT_RPC + ) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py index 21278cd624bf..cb0239770e9e 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py @@ -14,6 +14,12 @@ """An async client for interacting with Google Cloud Storage using the gRPC API.""" +import asyncio +from collections import deque +import inspect +import time +from typing import Any, Optional, Tuple + import grpc from google.auth import credentials as auth_credentials @@ -28,6 +34,507 @@ _rewrite_host_for_interconnect, ) +_GRPC_STATUS_NAMES = {sc.name for sc in grpc.StatusCode} + + +def _extract_rpc_operation(method_name: Any) -> str: + """Normalizes a gRPC method path into ``package.Service/Method``.""" + if isinstance(method_name, bytes): + method_name = method_name.decode("utf-8", errors="replace") + if not isinstance(method_name, str): + return "UNKNOWN" + normalized = method_name.strip().strip("/") + return normalized if normalized else "UNKNOWN" + + +def _resolve_status_and_error(exc: BaseException) -> Tuple[Optional[str], str]: + """Resolves ``(rpc.response.status_code, error.type)`` from an exception.""" + if isinstance(exc, asyncio.CancelledError): + return "CANCELLED", "CANCELLED" + + status_code: Optional[str] = None + for item in _opentelemetry_metrics._iter_exception_chain(exc): + grpc_code = getattr(item, "grpc_status_code", None) + if grpc_code is not None: + name = getattr(grpc_code, "name", None) + status_code = str(name) if name else str(grpc_code) + break + + code_attr = getattr(item, "code", None) + if callable(code_attr): + try: + rpc_status = code_attr() + if rpc_status is not None: + name = getattr(rpc_status, "name", None) + status_code = str(name) if name else str(rpc_status) + break + except Exception: # pragma: no cover + pass + + error_type = _opentelemetry_metrics.classify_error_type(exc) + if status_code is None and error_type in _GRPC_STATUS_NAMES: + status_code = error_type + return status_code, error_type + + +def _resolve_network_path( + call: Any, default_network_path: Optional[str] = None +) -> Optional[str]: + """Resolves ``"directpath"`` or ``"cloudpath"`` from ``call.peer()`` or channel config. + + Note: Python's ``grpc.aio.Call`` does not currently expose ``peer()`` on + C-core async calls; when ``peer()`` is unavailable, falls back to the + channel's configured ``default_network_path``. + """ + peer_fn = getattr(call, "peer", None) + if callable(peer_fn): + try: + peer_val = peer_fn() + if isinstance(peer_val, bytes): + peer_val = peer_val.decode("utf-8", errors="replace") + if isinstance(peer_val, str) and peer_val: + if "2001:4860:8040:" in peer_val or peer_val.startswith( + "ipv4:34.126." + ): + return "directpath" + if peer_val.startswith(("ipv4:", "ipv6:")): + return "cloudpath" + except Exception: # pragma: no cover + pass + return default_network_path + + +def _mark_call_closing_cleanly(call: Any) -> None: + """Marks a wrapped gRPC call as closing cleanly before normal cancellation.""" + seen = set() + queue = deque([call]) + steps = 0 + while queue and steps < 16: + steps += 1 + current = queue.popleft() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + inst_dict = getattr(current, "__dict__", None) + if isinstance(current, _TrackedCallBase) or ( + isinstance(inst_dict, dict) and "_closing_cleanly" in inst_dict + ): + current._closing_cleanly = True + if not isinstance(inst_dict, dict): + continue + for attr in ( + "_call", + "call", + "_wrapped", + "_last_returned_call_from_interceptors", + ): + inner = inst_dict.get(attr) + if inner is not None: + queue.append(inner) + task = inst_dict.get("_interceptors_task") + if task is not None and hasattr(task, "done") and task.done(): + try: + if not task.cancelled() and task.exception() is None: + queue.append(task.result()) + except Exception: # pragma: no cover + pass + + +class _TrackedCallBase: + """Base proxy for ``grpc.aio.Call`` objects that records attempt metrics once.""" + + def __init__( + self, + call: Any, + recorder: _opentelemetry_metrics._MetricsRecorder, + operation: str, + start_time: float, + op_state: Optional[_opentelemetry_metrics._OperationState], + network_path: Optional[str] = None, + ) -> None: + self._call = call + self._recorder = recorder + self._operation = operation + self._start_time = start_time + self._client_method = ( + op_state.client_method if op_state is not None else None + ) + self._network_path = network_path + self._recorded = False + self._closing_cleanly = False + + def _record_once( + self, status_code: Optional[str], error_type: str + ) -> None: + if not self._recorded: + self._recorded = True + duration = max(0.0, time.perf_counter() - self._start_time) + resolved_path = _resolve_network_path( + self._call, self._network_path + ) + self._recorder.record_rpc_attempt( + duration=duration, + operation=self._operation, + status_code=status_code, + error_type=error_type, + client_method=self._client_method, + network_path=resolved_path, + ) + + def _record_from_exception(self, exc: BaseException) -> None: + if isinstance(exc, asyncio.CancelledError) and self._closing_cleanly: + self._record_once(status_code="OK", error_type="OK") + else: + status_code, error_type = _resolve_status_and_error(exc) + self._record_once(status_code=status_code, error_type=error_type) + + async def wait_for_connection(self) -> None: + try: + await self._call.wait_for_connection() + except BaseException as exc: + self._record_from_exception(exc) + raise + + def cancel(self) -> bool: + result = self._call.cancel() + if self._closing_cleanly: + self._record_once(status_code="OK", error_type="OK") + else: + self._record_once(status_code="CANCELLED", error_type="CANCELLED") + return result + + def cancelled(self) -> bool: + return self._call.cancelled() + + def done(self) -> bool: + return self._call.done() + + def add_done_callback(self, callback: Any) -> None: + self._call.add_done_callback(callback) + + def time_remaining(self) -> Optional[float]: + return self._call.time_remaining() + + async def initial_metadata(self) -> Any: + return await self._call.initial_metadata() + + async def trailing_metadata(self) -> Any: + return await self._call.trailing_metadata() + + async def code(self) -> Any: + return await self._call.code() + + async def details(self) -> str: + return await self._call.details() + + async def debug_error_string(self) -> str: + return await self._call.debug_error_string() + + +class _TrackedUnaryResponseMixin: + """Mixin for unary-response gRPC calls.""" + + async def _await_impl(self) -> Any: + try: + response = await self._call + except BaseException as exc: + self._record_from_exception(exc) + raise + else: + self._record_once(status_code="OK", error_type="OK") + return response + + def __await__(self): + return self._await_impl().__await__() + + +class _TrackedStreamResponseMixin: + """Mixin for streaming-response gRPC calls.""" + + async def _aiter_impl(self): + completed_normally = False + try: + async for response in self._call: + yield response + completed_normally = True + except GeneratorExit: + if self._closing_cleanly: + self._record_once(status_code="OK", error_type="OK") + else: + self._record_once( + status_code="CANCELLED", error_type="CANCELLED" + ) + raise + except BaseException as exc: + self._record_from_exception(exc) + raise + finally: + if completed_normally: + self._record_once(status_code="OK", error_type="OK") + + def __aiter__(self): + return self._aiter_impl() + + async def read(self) -> Any: + try: + response = await self._call.read() + except BaseException as exc: + self._record_from_exception(exc) + raise + else: + if response is grpc.aio.EOF or response == grpc.aio.EOF: + self._record_once(status_code="OK", error_type="OK") + return response + + +class _TrackedStreamRequestMixin: + """Mixin for streaming-request gRPC calls.""" + + async def write(self, request: Any) -> None: + try: + await self._call.write(request) + except BaseException as exc: + self._record_from_exception(exc) + raise + + async def done_writing(self) -> None: + try: + await self._call.done_writing() + except BaseException as exc: + self._record_from_exception(exc) + raise + + @property + def _done_writing_flag(self) -> bool: + return getattr(self._call, "_done_writing_flag", False) + + @_done_writing_flag.setter + def _done_writing_flag(self, value: bool) -> None: + setattr(self._call, "_done_writing_flag", value) + + +class _TrackedUnaryUnaryCall( + _TrackedUnaryResponseMixin, _TrackedCallBase, grpc.aio.UnaryUnaryCall +): + """Tracked ``grpc.aio.UnaryUnaryCall`` proxy.""" + + +class _TrackedUnaryStreamCall( + _TrackedStreamResponseMixin, _TrackedCallBase, grpc.aio.UnaryStreamCall +): + """Tracked ``grpc.aio.UnaryStreamCall`` proxy.""" + + +class _TrackedStreamUnaryCall( + _TrackedUnaryResponseMixin, + _TrackedStreamRequestMixin, + _TrackedCallBase, + grpc.aio.StreamUnaryCall, +): + """Tracked ``grpc.aio.StreamUnaryCall`` proxy.""" + + +class _TrackedStreamStreamCall( + _TrackedStreamResponseMixin, + _TrackedStreamRequestMixin, + _TrackedCallBase, + grpc.aio.StreamStreamCall, +): + """Tracked ``grpc.aio.StreamStreamCall`` proxy.""" + + +class _MetricsUnaryUnaryInterceptor(grpc.aio.UnaryUnaryClientInterceptor): + """Client interceptor recording metrics for unary-unary gRPC attempts.""" + + def __init__( + self, + recorder: _opentelemetry_metrics._MetricsRecorder, + network_path: Optional[str] = None, + ) -> None: + self._recorder = recorder + self._network_path = network_path + + async def intercept_unary_unary( + self, continuation, client_call_details, request + ): + operation = _extract_rpc_operation( + getattr(client_call_details, "method", None) + ) + op_state = _opentelemetry_metrics.get_current_operation() + start_time = time.perf_counter() + try: + call = await continuation(client_call_details, request) + except BaseException as exc: + duration = max(0.0, time.perf_counter() - start_time) + status_code, error_type = _resolve_status_and_error(exc) + self._recorder.record_rpc_attempt( + duration=duration, + operation=operation, + status_code=status_code, + error_type=error_type, + client_method=( + op_state.client_method if op_state is not None else None + ), + network_path=self._network_path, + ) + raise + return _TrackedUnaryUnaryCall( + call, + self._recorder, + operation, + start_time, + op_state, + network_path=self._network_path, + ) + + +class _MetricsUnaryStreamInterceptor(grpc.aio.UnaryStreamClientInterceptor): + """Client interceptor recording metrics for unary-stream gRPC attempts.""" + + def __init__( + self, + recorder: _opentelemetry_metrics._MetricsRecorder, + network_path: Optional[str] = None, + ) -> None: + self._recorder = recorder + self._network_path = network_path + + async def intercept_unary_stream( + self, continuation, client_call_details, request + ): + operation = _extract_rpc_operation( + getattr(client_call_details, "method", None) + ) + op_state = _opentelemetry_metrics.get_current_operation() + start_time = time.perf_counter() + try: + call = await continuation(client_call_details, request) + except BaseException as exc: + duration = max(0.0, time.perf_counter() - start_time) + status_code, error_type = _resolve_status_and_error(exc) + self._recorder.record_rpc_attempt( + duration=duration, + operation=operation, + status_code=status_code, + error_type=error_type, + client_method=( + op_state.client_method if op_state is not None else None + ), + network_path=self._network_path, + ) + raise + return _TrackedUnaryStreamCall( + call, + self._recorder, + operation, + start_time, + op_state, + network_path=self._network_path, + ) + + +class _MetricsStreamUnaryInterceptor(grpc.aio.StreamUnaryClientInterceptor): + """Client interceptor recording metrics for stream-unary gRPC attempts.""" + + def __init__( + self, + recorder: _opentelemetry_metrics._MetricsRecorder, + network_path: Optional[str] = None, + ) -> None: + self._recorder = recorder + self._network_path = network_path + + async def intercept_stream_unary( + self, continuation, client_call_details, request_iterator + ): + operation = _extract_rpc_operation( + getattr(client_call_details, "method", None) + ) + op_state = _opentelemetry_metrics.get_current_operation() + start_time = time.perf_counter() + try: + call = await continuation(client_call_details, request_iterator) + except BaseException as exc: + duration = max(0.0, time.perf_counter() - start_time) + status_code, error_type = _resolve_status_and_error(exc) + self._recorder.record_rpc_attempt( + duration=duration, + operation=operation, + status_code=status_code, + error_type=error_type, + client_method=( + op_state.client_method if op_state is not None else None + ), + network_path=self._network_path, + ) + raise + return _TrackedStreamUnaryCall( + call, + self._recorder, + operation, + start_time, + op_state, + network_path=self._network_path, + ) + + +class _MetricsStreamStreamInterceptor(grpc.aio.StreamStreamClientInterceptor): + """Client interceptor recording metrics for stream-stream gRPC attempts.""" + + def __init__( + self, + recorder: _opentelemetry_metrics._MetricsRecorder, + network_path: Optional[str] = None, + ) -> None: + self._recorder = recorder + self._network_path = network_path + + async def intercept_stream_stream( + self, continuation, client_call_details, request_iterator + ): + operation = _extract_rpc_operation( + getattr(client_call_details, "method", None) + ) + op_state = _opentelemetry_metrics.get_current_operation() + start_time = time.perf_counter() + try: + call = await continuation(client_call_details, request_iterator) + except BaseException as exc: + duration = max(0.0, time.perf_counter() - start_time) + status_code, error_type = _resolve_status_and_error(exc) + self._recorder.record_rpc_attempt( + duration=duration, + operation=operation, + status_code=status_code, + error_type=error_type, + client_method=( + op_state.client_method if op_state is not None else None + ), + network_path=self._network_path, + ) + raise + return _TrackedStreamStreamCall( + call, + self._recorder, + operation, + start_time, + op_state, + network_path=self._network_path, + ) + + +def _create_metrics_interceptors( + recorder: _opentelemetry_metrics._MetricsRecorder, + network_path: Optional[str] = None, +) -> Tuple[grpc.aio.ClientInterceptor, ...]: + """Creates the 4 ``grpc.aio`` client interceptors for ``recorder``.""" + return ( + _MetricsUnaryUnaryInterceptor(recorder, network_path=network_path), + _MetricsUnaryStreamInterceptor(recorder, network_path=network_path), + _MetricsStreamUnaryInterceptor(recorder, network_path=network_path), + _MetricsStreamStreamInterceptor(recorder, network_path=network_path), + ) + def _validate_metadata(metadata): """Validates that metadata is a sequence of (key, value) pairs.""" @@ -108,6 +615,19 @@ def __init__( enable_debug_metrics ) + api_endpoint = None + if isinstance(client_options, dict): + api_endpoint = client_options.get("api_endpoint") + elif client_options is not None: + api_endpoint = getattr(client_options, "api_endpoint", None) + + self._metrics_recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=enable_metrics, + enable_debug_metrics=enable_debug_metrics, + api_endpoint=api_endpoint, + rpc_system="grpc", + ) + if isinstance(credentials, auth_credentials.AnonymousCredentials): if client_options is None or client_options.api_endpoint is None: raise ValueError( @@ -116,6 +636,7 @@ def __init__( self._grpc_client = self._create_anonymous_client( client_options, credentials ) + self._grpc_client._metrics_recorder = self._metrics_recorder return if client_info is None: @@ -138,6 +659,7 @@ def __init__( attempt_direct_path=attempt_direct_path, attempt_direct_path_xds_over_interconnect=use_dp_interconnect, ) + self._grpc_client._metrics_recorder = self._metrics_recorder @property def metrics_enabled(self) -> bool: @@ -150,7 +672,15 @@ def debug_metrics_enabled(self) -> bool: return self._debug_metrics_enabled def _create_anonymous_client(self, client_options, credentials): - channel = grpc.aio.insecure_channel(client_options.api_endpoint) + if self._metrics_recorder.standard_enabled: + channel = grpc.aio.insecure_channel( + client_options.api_endpoint, + interceptors=_create_metrics_interceptors( + self._metrics_recorder, network_path="cloudpath" + ), + ) + else: + channel = grpc.aio.insecure_channel(client_options.api_endpoint) transport = storage_v2.services.storage.transports.StorageGrpcAsyncIOTransport( channel=channel, credentials=credentials ) @@ -187,6 +717,17 @@ def _create_async_grpc_client( host = getattr(client_options, "api_endpoint", None) or _DEFAULT_HOST quota_project_id = getattr(client_options, "quota_project_id", None) + extra_channel_kwargs = {} + if self._metrics_recorder.standard_enabled: + network_path = ( + "directpath" + if (attempt_direct_path or attempt_direct_path_xds_over_interconnect) + else "cloudpath" + ) + extra_channel_kwargs["interceptors"] = _create_metrics_interceptors( + self._metrics_recorder, network_path=network_path + ) + if attempt_direct_path_xds_over_interconnect: host = _rewrite_host_for_interconnect(host) channel = transport_cls.create_channel( @@ -196,6 +737,7 @@ def _create_async_grpc_client( attempt_direct_path_xds_over_interconnect=True, credentials=credentials, options=(("grpc.primary_user_agent", primary_user_agent),), + **extra_channel_kwargs, ) else: channel = transport_cls.create_channel( @@ -204,6 +746,7 @@ def _create_async_grpc_client( attempt_direct_path=attempt_direct_path, credentials=credentials, options=(("grpc.primary_user_agent", primary_user_agent),), + **extra_channel_kwargs, ) transport = transport_cls(channel=channel) @@ -213,6 +756,23 @@ def _create_async_grpc_client( client_options=client_options, ) + async def close(self) -> None: + """Closes the underlying gRPC transport and flushes pending metrics.""" + try: + transport = getattr(self._grpc_client, "transport", None) + if transport is not None and hasattr(transport, "close"): + res = transport.close() + if inspect.isawaitable(res): + await res + finally: + self._metrics_recorder.shutdown() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + @property def grpc_client(self): """The underlying gRPC client. @@ -276,25 +836,31 @@ async def delete_object( :type retry: :class:`~google.api_core.retry_async.AsyncRetry` or None :param retry: (Optional) Designation of what errors, if any, should be retried. """ - _validate_metadata(metadata) - # The gRPC API requires the bucket name to be in the format "projects/_/buckets/bucket_name" - bucket_path = f"projects/_/buckets/{bucket_name}" - request = storage_v2.DeleteObjectRequest( - bucket=bucket_path, - object=object_name, - generation=generation, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - **kwargs, - ) - await self._grpc_client.delete_object( - request=request, - metadata=metadata, - timeout=timeout, - retry=retry, - ) + recorder = _opentelemetry_metrics.get_active_recorder(self) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncGrpcClient.delete_object", + rpc_method="google.storage.v2.Storage/DeleteObject", + ): + _validate_metadata(metadata) + # The gRPC API requires the bucket name to be in the format "projects/_/buckets/bucket_name" + bucket_path = f"projects/_/buckets/{bucket_name}" + request = storage_v2.DeleteObjectRequest( + bucket=bucket_path, + object=object_name, + generation=generation, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + **kwargs, + ) + await self._grpc_client.delete_object( + request=request, + metadata=metadata, + timeout=timeout, + retry=retry, + ) async def get_object( self, @@ -356,25 +922,31 @@ async def get_object( :rtype: :class:`google.cloud._storage_v2.types.Object` :returns: The object metadata resource. """ - _validate_metadata(metadata) - bucket_path = f"projects/_/buckets/{bucket_name}" - - request = storage_v2.GetObjectRequest( - bucket=bucket_path, - object=object_name, - generation=generation, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - soft_deleted=soft_deleted or False, - **kwargs, - ) + recorder = _opentelemetry_metrics.get_active_recorder(self) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncGrpcClient.get_object", + rpc_method="google.storage.v2.Storage/GetObject", + ): + _validate_metadata(metadata) + bucket_path = f"projects/_/buckets/{bucket_name}" + + request = storage_v2.GetObjectRequest( + bucket=bucket_path, + object=object_name, + generation=generation, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + soft_deleted=soft_deleted or False, + **kwargs, + ) - # Calls the underlying GAPIC StorageAsyncClient.get_object method - return await self._grpc_client.get_object( - request=request, - metadata=metadata, - timeout=timeout, - retry=retry, - ) + # Calls the underlying GAPIC StorageAsyncClient.get_object method + return await self._grpc_client.get_object( + request=request, + metadata=metadata, + timeout=timeout, + retry=retry, + ) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py index 32757bf754b7..284b66565dab 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py @@ -16,6 +16,7 @@ import asyncio import logging +import time from io import BytesIO from typing import Any, Dict, List, Optional, Tuple @@ -23,6 +24,7 @@ from google.api_core.retry_async import AsyncRetry from google.cloud import _storage_v2 +from google.cloud.storage import _opentelemetry_metrics from google.cloud.storage._helpers import generate_random_56_bit_integer from google.cloud.storage.asyncio._stream_multiplexer import ( _StreamEnd, @@ -48,6 +50,7 @@ from ._utils import raise_if_no_fast_crc32c _MAX_READ_RANGES_PER_BIDI_READ_REQUEST = 100 +_BIDI_READ_OBJECT_RPC = "google.storage.v2.Storage/BidiReadObject" _COMMON_READ_RETRYABLE_EXCEPTIONS = ( exceptions.InternalServerError, exceptions.ServiceUnavailable, @@ -151,16 +154,22 @@ async def create_mrd( :rtype: :class:`~google.cloud.storage.asyncio.async_multi_range_downloader.AsyncMultiRangeDownloader` :returns: An initialized AsyncMultiRangeDownloader instance for reading. """ - mrd = cls( - client, - bucket_name, - object_name, - generation=generation, - read_handle=read_handle, - **kwargs, - ) - await mrd.open(retry_policy=retry_policy, metadata=metadata) - return mrd + recorder = _opentelemetry_metrics.get_active_recorder(client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.create_mrd", + rpc_method=_BIDI_READ_OBJECT_RPC, + ): + mrd = cls( + client, + bucket_name, + object_name, + generation=generation, + read_handle=read_handle, + **kwargs, + ) + await mrd.open(retry_policy=retry_policy, metadata=metadata) + return mrd def __init__( self, @@ -241,82 +250,88 @@ async def open( metadata: Optional[List[Tuple[str, str]]] = None, ) -> None: """Opens the bidi-gRPC connection to read from the object.""" - if self._is_stream_open: - raise ValueError("Underlying bidi-gRPC stream is already open") - - if retry_policy is None: + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.open", + rpc_method=_BIDI_READ_OBJECT_RPC, + ): + if self._is_stream_open: + raise ValueError("Underlying bidi-gRPC stream is already open") + + if retry_policy is None: + + def on_error_wrapper(exc): + self._open_retries += 1 + self._on_open_error(exc) + + retry_policy = AsyncRetry( + predicate=_is_open_retryable, on_error=on_error_wrapper + ) + else: + original_on_error = retry_policy._on_error + + def combined_on_error(exc): + self._open_retries += 1 + self._on_open_error(exc) + if original_on_error: + original_on_error(exc) + + retry_policy = AsyncRetry( + predicate=_is_open_retryable, + initial=retry_policy._initial, + maximum=retry_policy._maximum, + multiplier=retry_policy._multiplier, + deadline=retry_policy._deadline, + on_error=combined_on_error, + ) - def on_error_wrapper(exc): - self._open_retries += 1 - self._on_open_error(exc) + async def _do_open(): + current_metadata = list(metadata) if metadata else [] - retry_policy = AsyncRetry( - predicate=_is_open_retryable, on_error=on_error_wrapper - ) - else: - original_on_error = retry_policy._on_error - - def combined_on_error(exc): - self._open_retries += 1 - self._on_open_error(exc) - if original_on_error: - original_on_error(exc) - - retry_policy = AsyncRetry( - predicate=_is_open_retryable, - initial=retry_policy._initial, - maximum=retry_policy._maximum, - multiplier=retry_policy._multiplier, - deadline=retry_policy._deadline, - on_error=combined_on_error, - ) - - async def _do_open(): - current_metadata = list(metadata) if metadata else [] + # Cleanup stream from previous failed attempt, if any. + if self.read_obj_str: + if self.read_obj_str.is_stream_open: + try: + await self.read_obj_str.close() + except exceptions.GoogleAPICallError as e: + logger.warning( + f"Failed to close existing stream during resumption: {e}" + ) + self.read_obj_str = None + self._is_stream_open = False + + self.read_obj_str = _AsyncReadObjectStream( + client=self.client.grpc_client, + bucket_name=self.bucket_name, + object_name=self.object_name, + generation_number=self.generation, + read_handle=self.read_handle, + ) - # Cleanup stream from previous failed attempt, if any. - if self.read_obj_str: - if self.read_obj_str.is_stream_open: - try: - await self.read_obj_str.close() - except exceptions.GoogleAPICallError as e: - logger.warning( - f"Failed to close existing stream during resumption: {e}" - ) - self.read_obj_str = None - self._is_stream_open = False - - self.read_obj_str = _AsyncReadObjectStream( - client=self.client.grpc_client, - bucket_name=self.bucket_name, - object_name=self.object_name, - generation_number=self.generation, - read_handle=self.read_handle, - ) + if self._routing_token: + current_metadata.append( + ("x-goog-request-params", f"routing_token={self._routing_token}") + ) + self._routing_token = None - if self._routing_token: - current_metadata.append( - ("x-goog-request-params", f"routing_token={self._routing_token}") + await self.read_obj_str.open( + metadata=current_metadata if current_metadata else None ) - self._routing_token = None - await self.read_obj_str.open( - metadata=current_metadata if current_metadata else None - ) + if self.read_obj_str.generation_number: + self.generation = self.read_obj_str.generation_number + if self.read_obj_str.read_handle: + self.read_handle = self.read_obj_str.read_handle + if self.read_obj_str.persisted_size is not None: + self.persisted_size = self.read_obj_str.persisted_size + self.is_finalized = self.read_obj_str.is_finalized + self.full_obj_server_crc32c = self.read_obj_str.full_obj_server_crc32c - if self.read_obj_str.generation_number: - self.generation = self.read_obj_str.generation_number - if self.read_obj_str.read_handle: - self.read_handle = self.read_obj_str.read_handle - if self.read_obj_str.persisted_size is not None: - self.persisted_size = self.read_obj_str.persisted_size - self.is_finalized = self.read_obj_str.is_finalized - self.full_obj_server_crc32c = self.read_obj_str.full_obj_server_crc32c - - self._is_stream_open = True + self._is_stream_open = True - await retry_policy(_do_open)() - self._multiplexer = _StreamMultiplexer(self.read_obj_str) + await retry_policy(_do_open)() + self._multiplexer = _StreamMultiplexer(self.read_obj_str) def _create_stream_factory(self, state, metadata): """Create a factory that opens a new stream with current routing state.""" @@ -401,147 +416,178 @@ async def download_ranges( """ - if len(read_ranges) > 1000: - raise ValueError( - "Invalid input - length of read_ranges cannot be more than 1000" - ) + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + rpc_method=_BIDI_READ_OBJECT_RPC, + ): + if len(read_ranges) > 1000: + raise ValueError( + "Invalid input - length of read_ranges cannot be more than 1000" + ) - if enable_checksum: - raise_if_no_fast_crc32c() + if enable_checksum: + raise_if_no_fast_crc32c() + + if not self._is_stream_open: + raise ValueError("Underlying bidi-gRPC stream is not open") + + if retry_policy is None: + retry_policy = AsyncRetry(predicate=_is_read_retryable) + + # Initialize Global State for Retry Strategy + download_states = {} + for read_range in read_ranges: + read_id = generate_random_56_bit_integer() + # Unpack tuple into self-documenting variable names to improve readability. + offset, length, user_buffer = read_range + + # Heuristic to detect full object reads: + # - Implicit full object read: start offset is 0 and length is 0 (read all). + # - Explicit full object read: start offset is 0 and length matches the exact persisted size. + is_full_object_read = (offset == 0 and length == 0) or ( + self.persisted_size is not None + and offset == 0 + and length == self.persisted_size + ) + download_states[read_id] = _DownloadState( + initial_offset=offset, + initial_length=length, + user_buffer=user_buffer, + is_full_object_read=is_full_object_read, + ) - if not self._is_stream_open: - raise ValueError("Underlying bidi-gRPC stream is not open") + initial_state = { + "download_states": download_states, + "read_handle": self.read_handle, + "routing_token": None, + "enable_checksum": enable_checksum, + "full_obj_server_crc32c": self.full_obj_server_crc32c, + } - if retry_policy is None: - retry_policy = AsyncRetry(predicate=_is_read_retryable) - - # Initialize Global State for Retry Strategy - download_states = {} - for read_range in read_ranges: - read_id = generate_random_56_bit_integer() - # Unpack tuple into self-documenting variable names to improve readability. - offset, length, user_buffer = read_range - - # Heuristic to detect full object reads: - # - Implicit full object read: start offset is 0 and length is 0 (read all). - # - Explicit full object read: start offset is 0 and length matches the exact persisted size. - is_full_object_read = (offset == 0 and length == 0) or ( - self.persisted_size is not None - and offset == 0 - and length == self.persisted_size - ) - download_states[read_id] = _DownloadState( - initial_offset=offset, - initial_length=length, - user_buffer=user_buffer, - is_full_object_read=is_full_object_read, - ) + read_ids = set(download_states.keys()) + queue = self._multiplexer.register(read_ids) - initial_state = { - "download_states": download_states, - "read_handle": self.read_handle, - "routing_token": None, - "enable_checksum": enable_checksum, - "full_obj_server_crc32c": self.full_obj_server_crc32c, - } - - read_ids = set(download_states.keys()) - queue = self._multiplexer.register(read_ids) - - try: - attempt_count = 0 - last_broken_generation = None - - def send_and_recv_via_multiplexer( - requests: List[_storage_v2.ReadRange], - state: Dict[str, Any], - ): - async def generator(): - nonlocal attempt_count, last_broken_generation - attempt_count += 1 - - if attempt_count > 1: - logger.info( - f"Resuming download (attempt {attempt_count}) for {len(requests)} ranges." - ) - - # Reopen stream if needed - should_reopen = ( - attempt_count > 1 and last_broken_generation is not None - ) or (attempt_count == 1 and metadata is not None) - if should_reopen: - broken_gen = ( - last_broken_generation - if attempt_count > 1 - else self._multiplexer.stream_generation - ) - stream_factory = self._create_stream_factory(state, metadata) - await self._multiplexer.reopen_stream( - broken_gen, stream_factory - ) - - stream_generation = self._multiplexer.stream_generation - - # Send Requests - pending_read_ids = {r.read_id for r in requests} - for i in range( - 0, len(requests), _MAX_READ_RANGES_PER_BIDI_READ_REQUEST - ): - batch = requests[i : i + _MAX_READ_RANGES_PER_BIDI_READ_REQUEST] - try: - await self._multiplexer.send( - _storage_v2.BidiReadObjectRequest(read_ranges=batch) + try: + attempt_count = 0 + last_broken_generation = None + + def send_and_recv_via_multiplexer( + requests: List[_storage_v2.ReadRange], + state: Dict[str, Any], + ): + async def generator(): + nonlocal attempt_count, last_broken_generation + attempt_count += 1 + attempt_start = time.perf_counter() + ttfb_recorded = False + + if attempt_count > 1: + logger.info( + f"Resuming download (attempt {attempt_count}) for {len(requests)} ranges." ) - except Exception: - last_broken_generation = stream_generation - raise - # Receive Responses - while pending_read_ids: - item = await queue.get() + # Reopen stream if needed + should_reopen = ( + attempt_count > 1 and last_broken_generation is not None + ) or (attempt_count == 1 and metadata is not None) + if should_reopen: + if ( + attempt_count == 1 + and metadata is not None + and self.read_obj_str is not None + and hasattr(self.read_obj_str, "_mark_closing_cleanly") + ): + self.read_obj_str._mark_closing_cleanly() + broken_gen = ( + last_broken_generation + if attempt_count > 1 + else self._multiplexer.stream_generation + ) + stream_factory = self._create_stream_factory(state, metadata) + await self._multiplexer.reopen_stream( + broken_gen, stream_factory + ) - if isinstance(item, _StreamEnd): - if pending_read_ids: - last_broken_generation = stream_generation - raise exceptions.ServiceUnavailable( - "Stream ended with pending read_ids" + stream_generation = self._multiplexer.stream_generation + + # Send Requests + pending_read_ids = {r.read_id for r in requests} + for i in range( + 0, len(requests), _MAX_READ_RANGES_PER_BIDI_READ_REQUEST + ): + batch = requests[ + i : i + _MAX_READ_RANGES_PER_BIDI_READ_REQUEST + ] + try: + await self._multiplexer.send( + _storage_v2.BidiReadObjectRequest(read_ranges=batch) ) - break - - if isinstance(item, _StreamError): - if item.generation < stream_generation: - continue # stale error, skip - last_broken_generation = item.generation - raise item.exception - - # Track completion - if item.object_data_ranges: - for data_range in item.object_data_ranges: - if data_range.range_end: - pending_read_ids.discard( - data_range.read_range.read_id + except Exception: + last_broken_generation = stream_generation + raise + + # Receive Responses + while pending_read_ids: + item = await queue.get() + + if isinstance(item, _StreamEnd): + if pending_read_ids: + last_broken_generation = stream_generation + raise exceptions.ServiceUnavailable( + "Stream ended with pending read_ids" + ) + break + + if isinstance(item, _StreamError): + if item.generation < stream_generation: + continue # stale error, skip + last_broken_generation = item.generation + raise item.exception + + # Track completion + if item.object_data_ranges: + if not ttfb_recorded and recorder is not None: + ttfb_recorded = True + recorder.record_ttfb( + time.perf_counter() - attempt_start, + operation=_BIDI_READ_OBJECT_RPC, ) - yield item + for data_range in item.object_data_ranges: + if data_range.range_end: + pending_read_ids.discard( + data_range.read_range.read_id + ) + yield item + + return generator() + + strategy = _ReadResumptionStrategy() + retry_manager = _BidiStreamRetryManager( + strategy, send_and_recv_via_multiplexer + ) - return generator() + try: + await retry_manager.execute(initial_state, retry_policy) + except DataCorruption: + if self.is_stream_open: + await self.close() + raise - strategy = _ReadResumptionStrategy() - retry_manager = _BidiStreamRetryManager( - strategy, send_and_recv_via_multiplexer - ) + if recorder is not None: + total_bytes = sum( + ds.bytes_written for ds in download_states.values() + ) + recorder.record_response_body_size( + total_bytes, operation=_BIDI_READ_OBJECT_RPC + ) - try: - await retry_manager.execute(initial_state, retry_policy) - except DataCorruption: - if self.is_stream_open: - await self.close() - raise - - if initial_state.get("read_handle"): - self.read_handle = initial_state["read_handle"] - finally: - if self._multiplexer is not None: - self._multiplexer.unregister(read_ids) + if initial_state.get("read_handle"): + self.read_handle = initial_state["read_handle"] + finally: + if self._multiplexer is not None: + self._multiplexer.unregister(read_ids) async def close(self): """ diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py index d3c0780673c7..314a502240f5 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py @@ -14,15 +14,19 @@ import asyncio import logging -from typing import List, Optional, Tuple +from typing import Any, List, Optional, Tuple from google.api_core.bidi_async import AsyncBidiRpc from google.cloud import _storage_v2 +from google.cloud.storage import _opentelemetry_metrics from google.cloud.storage.asyncio.async_abstract_object_stream import ( _AsyncAbstractObjectStream, ) -from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_grpc_client import ( + AsyncGrpcClient, + _mark_call_closing_cleanly, +) logger = logging.getLogger(__name__) @@ -58,6 +62,7 @@ def __init__( object_name: str, generation_number: Optional[int] = None, read_handle: Optional[_storage_v2.BidiReadHandle] = None, + metrics_recorder: Optional[Any] = None, ) -> None: if client is None: raise ValueError("client must be provided") @@ -73,6 +78,11 @@ def __init__( ) self.client: AsyncGrpcClient.grpc_client = client self.read_handle: Optional[_storage_v2.BidiReadHandle] = read_handle + self._metrics_recorder = ( + metrics_recorder + if metrics_recorder is not None + else _opentelemetry_metrics.get_active_recorder(client) + ) self._full_bucket_name = f"projects/_/buckets/{self.bucket_name}" @@ -161,6 +171,13 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: await self._close_socket_like_rpc() raise + def _mark_closing_cleanly(self) -> None: + """Marks the underlying gRPC call as closing cleanly for metrics.""" + if self.socket_like_rpc is not None: + call = getattr(self.socket_like_rpc, "call", None) + if call is not None: + _mark_call_closing_cleanly(call) + async def _close_socket_like_rpc(self) -> None: try: await self.socket_like_rpc.close() @@ -174,6 +191,7 @@ async def close(self) -> None: try: if self.socket_like_rpc.is_active: await self.requests_done() + self._mark_closing_cleanly() finally: self._is_stream_open = False await self._close_socket_like_rpc() diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py index 2d78a8c20fd1..b6649cb192c9 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py @@ -12,18 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Tuple +import asyncio +import time +from typing import Any, List, Optional, Tuple import grpc from google.api_core.bidi_async import AsyncBidiRpc from google.cloud import _storage_v2 -from google.cloud.storage import Blob, _grpc_conversions +from google.cloud.storage import Blob, _grpc_conversions, _opentelemetry_metrics from google.cloud.storage.asyncio import _utils from google.cloud.storage.asyncio.async_abstract_object_stream import ( _AsyncAbstractObjectStream, ) -from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_grpc_client import ( + AsyncGrpcClient, + _mark_call_closing_cleanly, +) + +_BIDI_WRITE_OBJECT_RPC = "google.storage.v2.Storage/BidiWriteObject" class _AsyncWriteObjectStream(_AsyncAbstractObjectStream): @@ -73,6 +80,7 @@ def __init__( routing_token: Optional[str] = None, blob: Optional[Blob] = None, storage_class: Optional[str] = None, + metrics_recorder: Optional[Any] = None, ) -> None: if client is None: raise ValueError("client must be provided") @@ -91,6 +99,11 @@ def __init__( self.routing_token: Optional[str] = routing_token self.blob: Optional[Blob] = blob self.storage_class: Optional[str] = storage_class + self._metrics_recorder = ( + metrics_recorder + if metrics_recorder is not None + else _opentelemetry_metrics.get_active_recorder(client) + ) self._full_bucket_name = f"projects/_/buckets/{self.bucket_name}" self.rpc = self.client._client._transport._wrapped_methods[ @@ -168,8 +181,18 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: metadata=final_metadata, ) - await self.socket_like_rpc.open() # this is actually 1 send - response = await self.socket_like_rpc.recv() + attempt_start = time.perf_counter() + try: + await self.socket_like_rpc.open() # this is actually 1 send + response = await self.socket_like_rpc.recv() + except (asyncio.CancelledError, Exception): + await self._close_socket_like_rpc() + raise + if self._metrics_recorder is not None: + self._metrics_recorder.record_ttfb( + time.perf_counter() - attempt_start, + operation=_BIDI_WRITE_OBJECT_RPC, + ) self._is_stream_open = True if response.persisted_size: @@ -187,13 +210,29 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: if response.write_handle: self.write_handle = response.write_handle + async def _close_socket_like_rpc(self) -> None: + try: + await self.socket_like_rpc.close() + except Exception: + pass + + def _mark_closing_cleanly(self) -> None: + """Marks the underlying gRPC call as closing cleanly for metrics.""" + if self.socket_like_rpc is not None: + call = getattr(self.socket_like_rpc, "call", None) + if call is not None: + _mark_call_closing_cleanly(call) + async def close(self) -> None: """Closes the bidi-gRPC connection.""" if not self._is_stream_open: raise ValueError("Stream is not open") - await self.requests_done() - await self.socket_like_rpc.close() - self._is_stream_open = False + try: + await self.requests_done() + self._mark_closing_cleanly() + finally: + self._is_stream_open = False + await self.socket_like_rpc.close() async def requests_done(self): """Signals that all requests have been sent.""" diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py index e6fcd90f0b84..245c920e2784 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -870,3 +870,71 @@ async def test_close_retry_on_redirect_error(self, mock_appendable_writer): assert writer.write_handle.handle == b"h2" writer.open.assert_awaited_once() assert not writer._is_stream_open + + @pytest.mark.asyncio + async def test_writer_metrics_recording(self, mock_appendable_writer): + mock_recorder = mock.MagicMock() + mock_recorder.standard_enabled = True + mock_client = mock_appendable_writer["mock_client"] + mock_client._metrics_recorder = mock_recorder + mock_stream = mock_appendable_writer["mock_stream"] + + writer = self._make_one(mock_client) + await writer.open() + assert mock_stream._metrics_recorder is mock_recorder + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncAppendableObjectWriter.open" + ) + + # Direct append records request_body_size once + mock_recorder.record_logical_operation.reset_mock() + await writer.append(b"hello") + mock_recorder.record_request_body_size.assert_called_once_with( + 5, operation="google.storage.v2.Storage/BidiWriteObject" + ) + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncAppendableObjectWriter.append" + ) + + # append_from_file suppresses per-chunk append metrics and records total size once + mock_recorder.record_logical_operation.reset_mock() + mock_recorder.record_request_body_size.reset_mock() + await writer.append_from_file(io.BytesIO(b"abcdefghij"), block_size=4) + mock_recorder.record_request_body_size.assert_called_once_with( + 10, operation="google.storage.v2.Storage/BidiWriteObject" + ) + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncAppendableObjectWriter.append_from_file" + ) + + # state_lookup, simple_flush, flush, close(finalize_on_close=True) + for method_name, call_coro in ( + ("state_lookup", writer.state_lookup()), + ("simple_flush", writer.simple_flush()), + ("flush", writer.flush()), + ): + mock_recorder.record_logical_operation.reset_mock() + await call_coro + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == f"storage.asyncio.AsyncAppendableObjectWriter.{method_name}" + ) + + mock_recorder.record_logical_operation.reset_mock() + mock_stream.recv.return_value = storage_type.BidiWriteObjectResponse( + resource=storage_type.Object(size=15) + ) + await writer.close(finalize_on_close=True) + # close(finalize_on_close=True) records outer close without duplicating finalize + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncAppendableObjectWriter.close" + ) diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py index 0b96a92eb055..913efae4016d 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py @@ -12,15 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio from unittest import mock +import grpc import pytest from google.api_core import client_info as client_info_lib from google.api_core import client_options from google.auth import credentials as auth_credentials from google.auth.credentials import AnonymousCredentials -from google.cloud.storage import __version__ +from google.cloud.storage import __version__, _opentelemetry_metrics from google.cloud.storage.asyncio import async_grpc_client @@ -648,3 +650,348 @@ def test_async_grpc_client_interconnect_env_override( ): with pytest.raises(ValueError): async_grpc_client.AsyncGrpcClient(credentials=mock_creds) + + @mock.patch("google.cloud._storage_v2.StorageAsyncClient") + @pytest.mark.asyncio + async def test_metrics_interceptors_and_client_lifecycle( + self, mock_async_storage_client, monkeypatch + ): + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + + mock_transport_cls = mock.MagicMock() + mock_async_storage_client.get_transport_class.return_value = mock_transport_cls + mock_gapic_client = mock.AsyncMock() + mock_gapic_client.transport = mock.MagicMock() + mock_gapic_client.transport.close = mock.AsyncMock() + mock_async_storage_client.return_value = mock_gapic_client + + client = async_grpc_client.AsyncGrpcClient( + credentials=_make_credentials(), + client_options={"api_endpoint": "custom.storage.googleapis.com:443"}, + enable_metrics=True, + ) + assert client.metrics_enabled is True + assert client._metrics_recorder.server_address == "custom.storage.googleapis.com" + assert client._grpc_client._metrics_recorder is client._metrics_recorder + + create_channel_kwargs = mock_transport_cls.create_channel.call_args.kwargs + assert "interceptors" in create_channel_kwargs + assert len(create_channel_kwargs["interceptors"]) == 4 + + mock_recorder = mock.MagicMock() + mock_recorder.standard_enabled = True + client._metrics_recorder = mock_recorder + + await client.delete_object("b", "o") + mock_recorder.record_logical_operation.assert_called_once() + _, del_kwargs = mock_recorder.record_logical_operation.call_args + assert ( + del_kwargs["client_method"] + == "storage.asyncio.AsyncGrpcClient.delete_object" + ) + assert del_kwargs["rpc_method"] == "google.storage.v2.Storage/DeleteObject" + assert del_kwargs["error_type"] == "OK" + + mock_recorder.record_logical_operation.reset_mock() + await client.get_object("b", "o") + mock_recorder.record_logical_operation.assert_called_once() + _, get_kwargs = mock_recorder.record_logical_operation.call_args + assert ( + get_kwargs["client_method"] == "storage.asyncio.AsyncGrpcClient.get_object" + ) + assert get_kwargs["rpc_method"] == "google.storage.v2.Storage/GetObject" + assert get_kwargs["error_type"] == "OK" + + async with client as entered: + assert entered is client + mock_gapic_client.transport.close.assert_awaited_once() + mock_recorder.shutdown.assert_called_once() + + @mock.patch( + "google.cloud._storage_v2.services.storage.transports.StorageGrpcAsyncIOTransport" + ) + @mock.patch("google.cloud._storage_v2.StorageAsyncClient") + @mock.patch("grpc.aio.insecure_channel") + def test_anonymous_client_with_metrics_enabled( + self, + mock_insecure_channel, + mock_async_storage_client, + mock_transport, + monkeypatch, + ): + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + opts = client_options.ClientOptions(api_endpoint="localhost:9000") + client = async_grpc_client.AsyncGrpcClient( + credentials=auth_credentials.AnonymousCredentials(), + client_options=opts, + enable_metrics=True, + ) + assert client.metrics_enabled is True + _, kwargs = mock_insecure_channel.call_args + assert "interceptors" in kwargs + assert len(kwargs["interceptors"]) == 4 + + @pytest.mark.asyncio + async def test_tracked_calls_and_interceptors(self): + assert ( + async_grpc_client._extract_rpc_operation( + b"/google.storage.v2.Storage/GetObject" + ) + == "google.storage.v2.Storage/GetObject" + ) + assert async_grpc_client._extract_rpc_operation(None) == "UNKNOWN" + assert async_grpc_client._extract_rpc_operation("/") == "UNKNOWN" + + status_code, err_type = async_grpc_client._resolve_status_and_error( + asyncio.CancelledError() + ) + assert (status_code, err_type) == ("CANCELLED", "CANCELLED") + + class _FakeRpcError(Exception): + def code(self): + return grpc.StatusCode.UNAVAILABLE + + assert async_grpc_client._resolve_status_and_error(_FakeRpcError()) == ( + "UNAVAILABLE", + "UNAVAILABLE", + ) + + from google.api_core import exceptions as api_exceptions + + assert async_grpc_client._resolve_status_and_error( + api_exceptions.NotFound("missing") + ) == ("NOT_FOUND", "NOT_FOUND") + assert async_grpc_client._resolve_status_and_error( + api_exceptions.Conflict("conflict") + ) == ("ABORTED", "ABORTED") + + recorder = mock.MagicMock() + recorder.standard_enabled = True + u_u, u_s, s_u, s_s = async_grpc_client._create_metrics_interceptors(recorder) + + details = mock.MagicMock(method="/google.storage.v2.Storage/GetObject") + + # 1. Unary-Unary success and methods delegation + class _AwaitableCall: + def __init__(self, result=None, exc=None): + self._result = result + self._exc = exc + self._done_writing_flag = False + self.wait_for_connection = mock.AsyncMock() + self.cancel = mock.MagicMock(return_value=True) + self.cancelled = mock.MagicMock(return_value=False) + self.done = mock.MagicMock(return_value=True) + self.add_done_callback = mock.MagicMock() + self.time_remaining = mock.MagicMock(return_value=5.0) + self.initial_metadata = mock.AsyncMock(return_value=(("k", "v"),)) + self.trailing_metadata = mock.AsyncMock(return_value=(("t", "v"),)) + self.code = mock.AsyncMock(return_value=grpc.StatusCode.OK) + self.details = mock.AsyncMock(return_value="ok") + self.debug_error_string = mock.AsyncMock(return_value="dbg") + self.write = mock.AsyncMock() + self.done_writing = mock.AsyncMock() + + async def _coro(self): + if self._exc: + raise self._exc + return self._result + + def __await__(self): + return self._coro().__await__() + + raw_uu = _AwaitableCall(result="resp") + tracked_uu = await u_u.intercept_unary_unary( + mock.AsyncMock(return_value=raw_uu), details, "req" + ) + await tracked_uu.wait_for_connection() + assert tracked_uu.cancelled() is False + assert tracked_uu.done() is True + tracked_uu.add_done_callback(lambda c: None) + assert tracked_uu.time_remaining() == 5.0 + assert await tracked_uu.initial_metadata() == (("k", "v"),) + assert await tracked_uu.trailing_metadata() == (("t", "v"),) + assert await tracked_uu.code() == grpc.StatusCode.OK + assert await tracked_uu.details() == "ok" + assert await tracked_uu.debug_error_string() == "dbg" + assert await tracked_uu == "resp" + recorder.record_rpc_attempt.assert_called_once() + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + + # 2. Stream-Unary error and continuation error + recorder.record_rpc_attempt.reset_mock() + raw_su = _AwaitableCall(exc=_FakeRpcError()) + tracked_su = await s_u.intercept_stream_unary( + mock.AsyncMock(return_value=raw_su), details, iter([]) + ) + tracked_su._done_writing_flag = True + assert tracked_su._done_writing_flag is True + await tracked_su.write("chunk") + await tracked_su.done_writing() + with pytest.raises(_FakeRpcError): + await tracked_su + assert ( + recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "UNAVAILABLE" + ) + + for interceptor, method_name in ( + (u_u, "intercept_unary_unary"), + (u_s, "intercept_unary_stream"), + (s_u, "intercept_stream_unary"), + (s_s, "intercept_stream_stream"), + ): + recorder.record_rpc_attempt.reset_mock() + with pytest.raises(_FakeRpcError): + await getattr(interceptor, method_name)( + mock.AsyncMock(side_effect=_FakeRpcError()), details, "req" + ) + recorder.record_rpc_attempt.assert_called_once() + + # 3. Unary-Stream __aiter__ and Stream-Stream read() + clean close + class _StreamCall(_AwaitableCall): + def __init__(self, items): + super().__init__() + self._items = list(items) + + def __aiter__(self): + async def _gen(): + for item in self._items: + if isinstance(item, BaseException): + raise item + yield item + + return _gen() + + async def read(self): + if not self._items: + return grpc.aio.EOF + item = self._items.pop(0) + if isinstance(item, BaseException): + raise item + return item + + recorder.record_rpc_attempt.reset_mock() + tracked_us = await u_s.intercept_unary_stream( + mock.AsyncMock(return_value=_StreamCall(["a", "b"])), details, "req" + ) + collected = [x async for x in tracked_us] + assert collected == ["a", "b"] + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + + recorder.record_rpc_attempt.reset_mock() + tracked_ss = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=_StreamCall(["x"])), details, iter([]) + ) + assert await tracked_ss.read() == "x" + assert await tracked_ss.read() == grpc.aio.EOF + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + + # Clean close vs abnormal cancel (including _interceptors_task, cycle, and non-dict object) + recorder.record_rpc_attempt.reset_mock() + tracked_ss_clean = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=_StreamCall([])), details, iter([]) + ) + done_fut = asyncio.get_running_loop().create_future() + done_fut.set_result(tracked_ss_clean) + wrapper = mock.MagicMock(_call=42, _interceptors_task=done_fut) + wrapper._wrapped = wrapper # cycle + async_grpc_client._mark_call_closing_cleanly(None) + async_grpc_client._mark_call_closing_cleanly(wrapper) + assert tracked_ss_clean._closing_cleanly is True + tracked_ss_clean.cancel() + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + + # Error paths in wait_for_connection, read, write, done_writing + err_call = _StreamCall([_FakeRpcError()]) + err_call.wait_for_connection = mock.AsyncMock(side_effect=_FakeRpcError()) + err_call.write = mock.AsyncMock(side_effect=_FakeRpcError()) + err_call.done_writing = mock.AsyncMock(side_effect=_FakeRpcError()) + + for op_fn in ( + lambda c: c.wait_for_connection(), + lambda c: c.read(), + lambda c: c.write("x"), + lambda c: c.done_writing(), + ): + recorder.record_rpc_attempt.reset_mock() + c = await s_s.intercept_stream_stream( + mock.AsyncMock( + return_value=_StreamCall([_FakeRpcError()]) + if op_fn + else err_call + ), + details, + iter([]), + ) + c._call = err_call + err_call._items = [_FakeRpcError()] + with pytest.raises(_FakeRpcError): + await op_fn(c) + recorder.record_rpc_attempt.assert_called_once() + + # CancelledError when _closing_cleanly is True records OK + recorder.record_rpc_attempt.reset_mock() + clean_cancel_call = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=_StreamCall([asyncio.CancelledError()])), + details, + iter([]), + ) + clean_cancel_call._closing_cleanly = True + with pytest.raises(asyncio.CancelledError): + await clean_cancel_call.read() + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + + recorder.record_rpc_attempt.reset_mock() + tracked_ss_cancel = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=_StreamCall([])), details, iter([]) + ) + tracked_ss_cancel.cancel() + assert ( + recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "CANCELLED" + ) + + # Early break from __aiter__ (clean vs cancelled) and peer() network_path detection + recorder.record_rpc_attempt.reset_mock() + stream_with_peer = _StreamCall(["a", "b", "c"]) + stream_with_peer.peer = lambda: b"ipv6:[2001:4860:8040::1]:443" + tracked_early = await u_s.intercept_unary_stream( + mock.AsyncMock(return_value=stream_with_peer), details, "req" + ) + ait = tracked_early.__aiter__() + assert await ait.__anext__() == "a" + await ait.aclose() + assert ( + recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "CANCELLED" + ) + assert ( + recorder.record_rpc_attempt.call_args.kwargs["network_path"] == "directpath" + ) + + recorder.record_rpc_attempt.reset_mock() + stream_cloud = _StreamCall(["a", "b"]) + stream_cloud.peer = lambda: "ipv4:142.250.1.1:443" + tracked_early_clean = await u_s.intercept_unary_stream( + mock.AsyncMock(return_value=stream_cloud), details, "req" + ) + tracked_early_clean._closing_cleanly = True + ait_clean = tracked_early_clean.__aiter__() + assert await ait_clean.__anext__() == "a" + await ait_clean.aclose() + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + assert ( + recorder.record_rpc_attempt.call_args.kwargs["network_path"] == "cloudpath" + ) + + # Mid-iteration exception in __aiter__ + recorder.record_rpc_attempt.reset_mock() + tracked_err_iter = await u_s.intercept_unary_stream( + mock.AsyncMock(return_value=_StreamCall(["a", _FakeRpcError()])), + details, + "req", + ) + with pytest.raises(_FakeRpcError): + _ = [x async for x in tracked_err_iter] + assert ( + recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "UNAVAILABLE" + ) diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py index afe1dfd221d9..5f6f3992f383 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py @@ -865,3 +865,116 @@ def test_is_read_retryable_predicate(self): assert _is_read_retryable(exceptions.NotFound("not found")) is False assert _is_read_retryable(exceptions.PermissionDenied("denied")) is False assert _is_read_retryable(exceptions.InvalidArgument("invalid")) is False + + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader.generate_random_56_bit_integer" + ) + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader._AsyncReadObjectStream" + ) + @pytest.mark.asyncio + async def test_mrd_metrics_recording( + self, mock_cls_async_read_object_stream, mock_random_int + ): + mock_recorder = mock.MagicMock() + mock_recorder.standard_enabled = True + mock_client = mock.MagicMock() + mock_client._metrics_recorder = mock_recorder + mock_client.grpc_client = mock.AsyncMock() + + stream = mock.MagicMock() + stream.open = AsyncMock() + stream.close = AsyncMock() + stream.send = AsyncMock() + stream._mark_closing_cleanly = mock.MagicMock() + stream.generation_number = _TEST_GENERATION_NUMBER + stream.persisted_size = _TEST_OBJECT_SIZE + stream.read_handle = _TEST_READ_HANDLE + stream.is_finalized = True + stream.full_obj_server_crc32c = None + stream.is_stream_open = True + mock_cls_async_read_object_stream.return_value = stream + + response = _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=b"data", crc32c=google_crc32c.value(b"data") + ), + range_end=True, + read_range=_storage_v2.ReadRange( + read_offset=0, read_length=4, read_id=123 + ), + ) + ] + ) + stream.recv = AsyncMock(side_effect=[response]) + mock_random_int.return_value = 123 + + mrd = await AsyncMultiRangeDownloader.create_mrd( + mock_client, + _TEST_BUCKET_NAME, + _TEST_OBJECT_NAME, + ) + # create_mrd suppresses nested open() so only 1 logical operation is recorded + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncMultiRangeDownloader.create_mrd" + ) + + mock_recorder.record_logical_operation.reset_mock() + buf = BytesIO() + await mrd.download_ranges([(0, 4, buf)]) + mock_recorder.record_ttfb.assert_called_once() + mock_recorder.record_response_body_size.assert_called_once_with( + 4, operation="google.storage.v2.Storage/BidiReadObject" + ) + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncMultiRangeDownloader.download_ranges" + ) + + # Verify download_ranges with custom metadata marks stream clean before reopen + # and records TTFB only once even across multiple response chunks + chunk1 = _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=b"ab", crc32c=google_crc32c.value(b"ab") + ), + range_end=False, + read_range=_storage_v2.ReadRange( + read_offset=0, read_length=2, read_id=123 + ), + ) + ] + ) + chunk2 = _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=b"cd", crc32c=google_crc32c.value(b"cd") + ), + range_end=True, + read_range=_storage_v2.ReadRange( + read_offset=2, read_length=2, read_id=123 + ), + ) + ] + ) + stream.recv = AsyncMock(side_effect=[chunk1, chunk2]) + mock_recorder.record_ttfb.reset_mock() + mock_recorder.record_response_body_size.reset_mock() + stream._mark_closing_cleanly.reset_mock() + buf2 = BytesIO() + await mrd.download_ranges([(0, 4, buf2)], metadata=[("x-custom", "val")]) + stream._mark_closing_cleanly.assert_called() + mock_recorder.record_ttfb.assert_called_once() + mock_recorder.record_response_body_size.assert_called_once_with( + 4, operation="google.storage.v2.Storage/BidiReadObject" + ) + + await mrd.close() + stream._mark_closing_cleanly.assert_called() diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py index 049c9f6608a9..b98170146131 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py @@ -393,3 +393,55 @@ async def test_methods_require_open_raises(self, mock_client): await stream.recv() with pytest.raises(ValueError, match="Stream is not open"): await stream.close() + + @mock.patch("google.cloud.storage.asyncio.async_write_object_stream.AsyncBidiRpc") + @pytest.mark.asyncio + async def test_open_records_write_ttfb_and_close_marks_clean( + self, mock_rpc_cls, mock_client + ): + mock_recorder = MagicMock() + mock_recorder.standard_enabled = True + mock_rpc = mock_rpc_cls.return_value + mock_rpc.open = AsyncMock() + mock_rpc.send = AsyncMock() + mock_rpc.recv = AsyncMock( + side_effect=[ + MagicMock( + persisted_size=0, resource=None, write_handle=b"h1" + ), + grpc.aio.EOF, + ] + ) + mock_rpc.close = AsyncMock() + mock_call = MagicMock(_closing_cleanly=False) + mock_rpc.call = mock_call + + stream = _AsyncWriteObjectStream( + mock_client, BUCKET, OBJECT, metrics_recorder=mock_recorder + ) + await stream.open() + mock_recorder.record_ttfb.assert_called_once() + assert ( + mock_recorder.record_ttfb.call_args.kwargs["operation"] + == "google.storage.v2.Storage/BidiWriteObject" + ) + + await stream.close() + assert mock_call._closing_cleanly is True + + @mock.patch("google.cloud.storage.asyncio.async_write_object_stream.AsyncBidiRpc") + @pytest.mark.asyncio + async def test_open_closes_rpc_when_initial_recv_fails( + self, mock_rpc_cls, mock_client + ): + mock_rpc = mock_rpc_cls.return_value + mock_rpc.open = AsyncMock() + mock_rpc.recv = AsyncMock(side_effect=RuntimeError("initial recv failed")) + mock_rpc.close = AsyncMock() + + stream = _AsyncWriteObjectStream(mock_client, BUCKET, OBJECT) + with pytest.raises(RuntimeError, match="initial recv failed"): + await stream.open() + + mock_rpc.close.assert_awaited_once() + assert not stream.is_stream_open diff --git a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py index cb392b2f9870..b23907b97b84 100644 --- a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py +++ b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py @@ -1025,11 +1025,11 @@ async def worker(method_name: str, rpc_name: str): async def run_all(): await asyncio.gather( worker( - "storage.AsyncMultiRangeDownloader.download_ranges", + "storage.asyncio.AsyncMultiRangeDownloader.download_ranges", "google.storage.v2.Storage/BidiReadObject", ), worker( - "storage.AsyncAppendableObjectWriter.append", + "storage.asyncio.AsyncAppendableObjectWriter.append", "google.storage.v2.Storage/BidiWriteObject", ), ) @@ -1205,7 +1205,7 @@ def test_metrics_recorder_recording_and_shutdown(monkeypatch): operation="google.storage.v2.Storage/BidiReadObject", status_code="OK", error_type="OK", - client_method="storage.AsyncMultiRangeDownloader.download_ranges", + client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", network_path="directpath", ) recorder.record_rpc_attempt( @@ -1213,7 +1213,7 @@ def test_metrics_recorder_recording_and_shutdown(monkeypatch): operation="google.storage.v2.Storage/BidiReadObject", status_code="UNAVAILABLE", error_type="UNAVAILABLE", - client_method="storage.AsyncMultiRangeDownloader.download_ranges", + client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", ) assert recorder.rpc_call_duration.record.call_count == 2 _, rpc_dur_kwargs = recorder.rpc_call_duration.record.call_args_list[0] @@ -1223,7 +1223,7 @@ def test_metrics_recorder_recording_and_shutdown(monkeypatch): ) assert ( rpc_dur_kwargs["attributes"]["gcp.client.method"] - == "storage.AsyncMultiRangeDownloader.download_ranges" + == "storage.asyncio.AsyncMultiRangeDownloader.download_ranges" ) assert rpc_dur_kwargs["attributes"]["rpc.response.status_code"] == "OK" assert rpc_dur_kwargs["attributes"]["network.path"] == "directpath" @@ -1252,6 +1252,36 @@ def test_metrics_recorder_recording_and_shutdown(monkeypatch): recorder.shutdown() # Must not raise +def test_get_active_recorder_and_grpc_uri_schemes(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + + assert ( + _opentelemetry_metrics._extract_server_address("dns:///custom.endpoint.com:443") + == "custom.endpoint.com" + ) + assert ( + _opentelemetry_metrics._extract_server_address( + "google-c2p-experimental:///storage.googleapis.com" + ) + == "storage.googleapis.com" + ) + + # Unconfigured MagicMock should not be treated as an active recorder + unconfigured_mock = mock.MagicMock() + assert _opentelemetry_metrics.get_active_recorder(unconfigured_mock) is None + assert _opentelemetry_metrics.get_active_recorder(None) is None + + active_rec = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + holder = mock.MagicMock() + holder._metrics_recorder = active_rec + assert _opentelemetry_metrics.get_active_recorder(holder) is active_rec + + disabled_rec = _opentelemetry_metrics._MetricsRecorder(enable_metrics=False) + holder._metrics_recorder = disabled_rec + assert _opentelemetry_metrics.get_active_recorder(holder) is None + + def test_metrics_recorder_disabled_no_ops(monkeypatch): monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", False) recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) From 73b91c45d00d742e213037e49a11a2bfa0f4aff4 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Mon, 19 Oct 2026 19:00:34 +0000 Subject: [PATCH 3/7] fix(storage): fix ruff lint, py310 lower-bound metrics fallback, and branch coverage [Generated-by: AI] --- .../cloud/storage/_opentelemetry_metrics.py | 31 ++++---- .../asyncio/async_appendable_object_writer.py | 6 +- .../storage/asyncio/async_grpc_client.py | 70 ++++++------------- .../asyncio/async_multi_range_downloader.py | 9 ++- .../asyncio/async_read_object_stream.py | 5 +- .../asyncio/async_write_object_stream.py | 5 +- .../unit/asyncio/test_async_grpc_client.py | 30 ++++++-- .../asyncio/test_async_write_object_stream.py | 4 +- .../tests/unit/test__opentelemetry_metrics.py | 34 ++++++++- 9 files changed, 109 insertions(+), 85 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py index 6f8e75269aa6..f7394c9b2740 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py +++ b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py @@ -15,11 +15,8 @@ """Manages OpenTelemetry metrics instruments and gating for GCS client.""" import asyncio -from collections import deque import concurrent.futures import contextlib -from contextvars import ContextVar -from dataclasses import dataclass import http.client import logging import os @@ -27,6 +24,9 @@ import socket import ssl import time +from collections import deque +from contextvars import ContextVar +from dataclasses import dataclass from typing import Any, Deque, Dict, Iterator, List, Optional, Set, Tuple from urllib.parse import urlsplit @@ -85,6 +85,7 @@ HAS_OPENTELEMETRY_METRICS = True except ImportError: + metrics = None # type: ignore[assignment] HAS_OPENTELEMETRY_METRICS = False logger.debug( "OpenTelemetry metrics package (opentelemetry-api >= 1.12.0) is not " @@ -207,9 +208,12 @@ def get_meter( Optional[Any]: The OpenTelemetry Meter instance if standard or debug metrics are enabled, or None otherwise. """ - if not ( - is_metrics_enabled(enable_metrics) - or is_debug_metrics_enabled(enable_debug_metrics) + if ( + not ( + is_metrics_enabled(enable_metrics) + or is_debug_metrics_enabled(enable_debug_metrics) + ) + or metrics is None ): return None return metrics.get_meter( @@ -238,13 +242,13 @@ def _build_latency_buckets() -> Tuple[float, ...]: boundary += increment increment = 0.010 - for i in range(150): - if boundary >= 300.0: - break + i = 0 + while boundary < 300.0: if i != 0 and i % 10 == 0: increment *= 2 boundaries.append(round(boundary, 6)) boundary += increment + i += 1 return tuple(boundaries) @@ -452,11 +456,10 @@ def classify_error_type( if callable(code_attr): try: rpc_status = code_attr() - if rpc_status is not None: - name = getattr(rpc_status, "name", None) - if name: - return str(name) - except Exception: # pragma: no cover + name = getattr(rpc_status, "name", None) + if name: + return str(name) + except Exception: # pragma: NO COVER pass if _is_valid_status_code(code_attr): diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py index c1cb5d0d27fb..c0a4d70e138b 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py @@ -520,9 +520,9 @@ async def generator(): if resp: if resp.persisted_size is not None: self.persisted_size = resp.persisted_size - state["write_state"].persisted_size = ( - resp.persisted_size - ) + state[ + "write_state" + ].persisted_size = resp.persisted_size self.offset = self.persisted_size if resp.write_handle: self.write_handle = resp.write_handle diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py index cb0239770e9e..6b0145d9d0b9 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py @@ -15,9 +15,8 @@ """An async client for interacting with Google Cloud Storage using the gRPC API.""" import asyncio -from collections import deque -import inspect import time +from collections import deque from typing import Any, Optional, Tuple import grpc @@ -64,11 +63,11 @@ def _resolve_status_and_error(exc: BaseException) -> Tuple[Optional[str], str]: if callable(code_attr): try: rpc_status = code_attr() - if rpc_status is not None: - name = getattr(rpc_status, "name", None) - status_code = str(name) if name else str(rpc_status) + name = getattr(rpc_status, "name", None) + if name: + status_code = str(name) break - except Exception: # pragma: no cover + except Exception: # pragma: NO COVER pass error_type = _opentelemetry_metrics.classify_error_type(exc) @@ -92,14 +91,12 @@ def _resolve_network_path( peer_val = peer_fn() if isinstance(peer_val, bytes): peer_val = peer_val.decode("utf-8", errors="replace") - if isinstance(peer_val, str) and peer_val: - if "2001:4860:8040:" in peer_val or peer_val.startswith( - "ipv4:34.126." - ): + if isinstance(peer_val, str): + if "2001:4860:8040:" in peer_val or peer_val.startswith("ipv4:34.126."): return "directpath" if peer_val.startswith(("ipv4:", "ipv6:")): return "cloudpath" - except Exception: # pragma: no cover + except Exception: # pragma: NO COVER pass return default_network_path @@ -134,9 +131,8 @@ def _mark_call_closing_cleanly(call: Any) -> None: task = inst_dict.get("_interceptors_task") if task is not None and hasattr(task, "done") and task.done(): try: - if not task.cancelled() and task.exception() is None: - queue.append(task.result()) - except Exception: # pragma: no cover + queue.append(task.result()) + except Exception: # pragma: NO COVER pass @@ -156,22 +152,16 @@ def __init__( self._recorder = recorder self._operation = operation self._start_time = start_time - self._client_method = ( - op_state.client_method if op_state is not None else None - ) + self._client_method = op_state.client_method if op_state is not None else None self._network_path = network_path self._recorded = False self._closing_cleanly = False - def _record_once( - self, status_code: Optional[str], error_type: str - ) -> None: + def _record_once(self, status_code: Optional[str], error_type: str) -> None: if not self._recorded: self._recorded = True duration = max(0.0, time.perf_counter() - self._start_time) - resolved_path = _resolve_network_path( - self._call, self._network_path - ) + resolved_path = _resolve_network_path(self._call, self._network_path) self._recorder.record_rpc_attempt( duration=duration, operation=self._operation, @@ -261,9 +251,7 @@ async def _aiter_impl(self): if self._closing_cleanly: self._record_once(status_code="OK", error_type="OK") else: - self._record_once( - status_code="CANCELLED", error_type="CANCELLED" - ) + self._record_once(status_code="CANCELLED", error_type="CANCELLED") raise except BaseException as exc: self._record_from_exception(exc) @@ -354,12 +342,8 @@ def __init__( self._recorder = recorder self._network_path = network_path - async def intercept_unary_unary( - self, continuation, client_call_details, request - ): - operation = _extract_rpc_operation( - getattr(client_call_details, "method", None) - ) + async def intercept_unary_unary(self, continuation, client_call_details, request): + operation = _extract_rpc_operation(getattr(client_call_details, "method", None)) op_state = _opentelemetry_metrics.get_current_operation() start_time = time.perf_counter() try: @@ -399,12 +383,8 @@ def __init__( self._recorder = recorder self._network_path = network_path - async def intercept_unary_stream( - self, continuation, client_call_details, request - ): - operation = _extract_rpc_operation( - getattr(client_call_details, "method", None) - ) + async def intercept_unary_stream(self, continuation, client_call_details, request): + operation = _extract_rpc_operation(getattr(client_call_details, "method", None)) op_state = _opentelemetry_metrics.get_current_operation() start_time = time.perf_counter() try: @@ -447,9 +427,7 @@ def __init__( async def intercept_stream_unary( self, continuation, client_call_details, request_iterator ): - operation = _extract_rpc_operation( - getattr(client_call_details, "method", None) - ) + operation = _extract_rpc_operation(getattr(client_call_details, "method", None)) op_state = _opentelemetry_metrics.get_current_operation() start_time = time.perf_counter() try: @@ -492,9 +470,7 @@ def __init__( async def intercept_stream_stream( self, continuation, client_call_details, request_iterator ): - operation = _extract_rpc_operation( - getattr(client_call_details, "method", None) - ) + operation = _extract_rpc_operation(getattr(client_call_details, "method", None)) op_state = _opentelemetry_metrics.get_current_operation() start_time = time.perf_counter() try: @@ -759,11 +735,7 @@ def _create_async_grpc_client( async def close(self) -> None: """Closes the underlying gRPC transport and flushes pending metrics.""" try: - transport = getattr(self._grpc_client, "transport", None) - if transport is not None and hasattr(transport, "close"): - res = transport.close() - if inspect.isawaitable(res): - await res + await self._grpc_client.transport.close() finally: self._metrics_recorder.shutdown() diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py index 284b66565dab..95a2ffe798f1 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py @@ -311,7 +311,10 @@ async def _do_open(): if self._routing_token: current_metadata.append( - ("x-goog-request-params", f"routing_token={self._routing_token}") + ( + "x-goog-request-params", + f"routing_token={self._routing_token}", + ) ) self._routing_token = None @@ -505,7 +508,9 @@ async def generator(): if attempt_count > 1 else self._multiplexer.stream_generation ) - stream_factory = self._create_stream_factory(state, metadata) + stream_factory = self._create_stream_factory( + state, metadata + ) await self._multiplexer.reopen_stream( broken_gen, stream_factory ) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py index 314a502240f5..8322c3fa3d99 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py @@ -173,10 +173,7 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: def _mark_closing_cleanly(self) -> None: """Marks the underlying gRPC call as closing cleanly for metrics.""" - if self.socket_like_rpc is not None: - call = getattr(self.socket_like_rpc, "call", None) - if call is not None: - _mark_call_closing_cleanly(call) + _mark_call_closing_cleanly(getattr(self.socket_like_rpc, "call", None)) async def _close_socket_like_rpc(self) -> None: try: diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py index b6649cb192c9..deff956c7efb 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py @@ -218,10 +218,7 @@ async def _close_socket_like_rpc(self) -> None: def _mark_closing_cleanly(self) -> None: """Marks the underlying gRPC call as closing cleanly for metrics.""" - if self.socket_like_rpc is not None: - call = getattr(self.socket_like_rpc, "call", None) - if call is not None: - _mark_call_closing_cleanly(call) + _mark_call_closing_cleanly(getattr(self.socket_like_rpc, "call", None)) async def close(self) -> None: """Closes the bidi-gRPC connection.""" diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py index 913efae4016d..2a227d4484dc 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py @@ -672,7 +672,9 @@ async def test_metrics_interceptors_and_client_lifecycle( enable_metrics=True, ) assert client.metrics_enabled is True - assert client._metrics_recorder.server_address == "custom.storage.googleapis.com" + assert ( + client._metrics_recorder.server_address == "custom.storage.googleapis.com" + ) assert client._grpc_client._metrics_recorder is client._metrics_recorder create_channel_kwargs = mock_transport_cls.create_channel.call_args.kwargs @@ -917,9 +919,7 @@ async def read(self): recorder.record_rpc_attempt.reset_mock() c = await s_s.intercept_stream_stream( mock.AsyncMock( - return_value=_StreamCall([_FakeRpcError()]) - if op_fn - else err_call + return_value=_StreamCall([_FakeRpcError()]) if op_fn else err_call ), details, iter([]), @@ -947,9 +947,31 @@ async def read(self): mock.AsyncMock(return_value=_StreamCall([])), details, iter([]) ) tracked_ss_cancel.cancel() + tracked_ss_cancel.cancel() # second call is a no-op assert ( recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "CANCELLED" ) + assert ( + async_grpc_client._resolve_network_path( + mock.MagicMock(peer=lambda: 123), "cloudpath" + ) + == "cloudpath" + ) + assert ( + async_grpc_client._resolve_network_path( + mock.MagicMock(peer=lambda: "unix:/tmp/sock"), "cloudpath" + ) + == "cloudpath" + ) + + class _NoneCodeRpcErr(Exception): + def code(self): + return None + + assert async_grpc_client._resolve_status_and_error(_NoneCodeRpcErr()) == ( + "UNKNOWN", + "UNKNOWN", + ) # Early break from __aiter__ (clean vs cancelled) and peer() network_path detection recorder.record_rpc_attempt.reset_mock() diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py index b98170146131..056bff5c4eb4 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py @@ -406,9 +406,7 @@ async def test_open_records_write_ttfb_and_close_marks_clean( mock_rpc.send = AsyncMock() mock_rpc.recv = AsyncMock( side_effect=[ - MagicMock( - persisted_size=0, resource=None, write_handle=b"h1" - ), + MagicMock(persisted_size=0, resource=None, write_handle=b"h1"), grpc.aio.EOF, ] ) diff --git a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py index b23907b97b84..275baa8f7c5c 100644 --- a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py +++ b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py @@ -22,11 +22,11 @@ import types from unittest import mock -from google.api_core import exceptions as api_exceptions -from google.auth import exceptions as auth_exceptions import pytest import requests.exceptions import urllib3.exceptions +from google.api_core import exceptions as api_exceptions +from google.auth import exceptions as auth_exceptions from google.cloud.storage import _opentelemetry_metrics from google.cloud.storage import exceptions as storage_exceptions @@ -57,6 +57,8 @@ def test_opentelemetry_import_error_on_load(monkeypatch): is False ) assert _opentelemetry_metrics.get_meter(enable_metrics=True) is None + m.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + assert _opentelemetry_metrics.get_meter(enable_metrics=True) is None finally: importlib.reload(_opentelemetry_metrics) @@ -1244,6 +1246,34 @@ def test_metrics_recorder_recording_and_shutdown(monkeypatch): assert rpc_err_kwargs["attributes"]["error.type"] == "UNAVAILABLE" assert "rpc.response.status_code" not in rpc_err_kwargs["attributes"] + # Record outside of start_operation with status_code=None and callable .code() returning None + recorder.record_http_attempt( + duration=0.01, + http_method="GET", + url_template="/storage/v1/b", + status_code=None, + error_type="UNKNOWN", + ) + recorder.record_rpc_attempt( + duration=0.01, + operation="google.storage.v2.Storage/GetObject", + status_code=None, + error_type="UNKNOWN", + ) + recorder.record_request_body_size(64) + + class _NoneCodeErr(Exception): + def code(self): + return None + + assert _opentelemetry_metrics.classify_error_type(_NoneCodeErr()) == "UNKNOWN" + assert ( + _opentelemetry_metrics.classify_error_type( + urllib3.exceptions.NewConnectionError(None, "conn err") + ) + == "CONNECTIVITY" + ) + # Shutdown flushes custom meter_provider and swallows force_flush errors gracefully recorder.shutdown() mock_provider.force_flush.assert_called_once() From fd2daac0d55ecebb202013d8b9d17c7cef1d914a Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Mon, 19 Oct 2026 19:30:31 +0000 Subject: [PATCH 4/7] fix(storage): delegate unhandled attributes in _TrackedCallBase.__getattr__ [Generated-by: AI] --- .../google/cloud/storage/asyncio/async_grpc_client.py | 3 +++ .../tests/unit/asyncio/test_async_grpc_client.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py index 6b0145d9d0b9..62931c67bcc6 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py @@ -157,6 +157,9 @@ def __init__( self._recorded = False self._closing_cleanly = False + def __getattr__(self, name: str) -> Any: + return getattr(self._call, name) + def _record_once(self, status_code: Optional[str], error_type: str) -> None: if not self._recorded: self._recorded = True diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py index 2a227d4484dc..2166a26498be 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py @@ -804,9 +804,11 @@ def __await__(self): return self._coro().__await__() raw_uu = _AwaitableCall(result="resp") + raw_uu.custom_attr = "delegated_val" tracked_uu = await u_u.intercept_unary_unary( mock.AsyncMock(return_value=raw_uu), details, "req" ) + assert tracked_uu.custom_attr == "delegated_val" await tracked_uu.wait_for_connection() assert tracked_uu.cancelled() is False assert tracked_uu.done() is True From 2285c1a01e85d20ad49e600d4b48bcb40c8d4635 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Mon, 19 Oct 2026 19:35:54 +0000 Subject: [PATCH 5/7] fix(storage): harden error classification against mocks and future cancellations [Generated-by: AI] --- .../cloud/storage/_opentelemetry_metrics.py | 32 +++++++++++-------- .../tests/unit/test__opentelemetry_metrics.py | 14 ++++++++ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py index f7394c9b2740..27675bc298a0 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py +++ b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py @@ -362,21 +362,25 @@ def _iter_exception_chain(exc: BaseException) -> Iterator[BaseException]: visited.add(id(nested)) queue.append(nested) - for arg in getattr(current, "args", ()): - if isinstance(arg, BaseException) and id(arg) not in visited: - visited.add(id(arg)) - queue.append(arg) + args = getattr(current, "args", ()) + if isinstance(args, (tuple, list)): + for arg in args: + if isinstance(arg, BaseException) and id(arg) not in visited: + visited.add(id(arg)) + queue.append(arg) def _is_subclass_by_module( - cls: type, module_prefix: str, class_names: Optional[Set[str]] = None + cls: type, module_prefix: str, class_names: Set[str] ) -> bool: """Checks whether ``cls`` or any of its base classes match module/name criteria.""" - for base in getattr(cls, "__mro__", ()): - mod = getattr(base, "__module__", "") or "" - if mod.startswith(module_prefix): - if class_names is None or base.__name__ in class_names: - return True + mro = getattr(cls, "__mro__", None) + if isinstance(mro, tuple): + for base in mro: + mod = getattr(base, "__module__", "") or "" + if mod == module_prefix or mod.startswith(module_prefix + "."): + if getattr(base, "__name__", None) in class_names: + return True return False @@ -426,7 +430,9 @@ def classify_error_type( # 3. Cancellation for item in chain: - if isinstance(item, asyncio.CancelledError): + if isinstance( + item, (asyncio.CancelledError, concurrent.futures.CancelledError) + ): return "CANCELLED" # 4. HTTP response status code on exception (checked before grpc_status_code so @@ -457,8 +463,8 @@ def classify_error_type( try: rpc_status = code_attr() name = getattr(rpc_status, "name", None) - if name: - return str(name) + if isinstance(name, str) and name: + return name except Exception: # pragma: NO COVER pass diff --git a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py index 275baa8f7c5c..79a2e76eaee5 100644 --- a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py +++ b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py @@ -1266,7 +1266,21 @@ class _NoneCodeErr(Exception): def code(self): return None + class _NonTupleArgsErr(Exception): + args = 123 # type: ignore[assignment] + assert _opentelemetry_metrics.classify_error_type(_NoneCodeErr()) == "UNKNOWN" + assert _opentelemetry_metrics.classify_error_type(_NonTupleArgsErr()) == "UNKNOWN" + assert ( + _opentelemetry_metrics.classify_error_type(concurrent.futures.CancelledError()) + == "CANCELLED" + ) + assert ( + _opentelemetry_metrics._is_subclass_by_module( + mock.MagicMock(__mro__=123), "google.cloud.storage", {"DataCorruption"} + ) + is False + ) assert ( _opentelemetry_metrics.classify_error_type( urllib3.exceptions.NewConnectionError(None, "conn err") From ab81d9b0e8953ab307d7ebac5befad6ed4164147 Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Sun, 25 Oct 2026 16:55:49 +0000 Subject: [PATCH 6/7] fix(storage): align Async gRPC OTel metrics with Monarch schema and resolve PR feedback - Align all 10 metric instruments, attributes, and 48 size buckets with the Monarch schema - Resolve per-call network.path dynamically via GFE metadata headers and DirectPath CIDRs - Separate short gcp.storage.operation names from full rpc.method paths - Add meter_provider parameter to AsyncGrpcClient.__init__ - Track active-operation attempt durations and gcp.storage.client.retries across Bidi stream opens and resumptions [Generated-by: AI] --- .../cloud/storage/_opentelemetry_metrics.py | 352 +++++++++++--- .../asyncio/async_appendable_object_writer.py | 184 +++++-- .../storage/asyncio/async_grpc_client.py | 457 ++++++++++++++---- .../asyncio/async_multi_range_downloader.py | 81 +++- .../asyncio/async_read_object_stream.py | 15 + .../asyncio/async_write_object_stream.py | 19 +- .../retry/bidi_stream_retry_manager.py | 35 +- .../retry/test_bidi_stream_retry_manager.py | 78 +++ .../test_async_appendable_object_writer.py | 126 ++++- .../unit/asyncio/test_async_grpc_client.py | 383 ++++++++++++++- .../test_async_multi_range_downloader.py | 143 +++++- .../asyncio/test_async_read_object_stream.py | 59 +++ .../asyncio/test_async_write_object_stream.py | 6 +- .../tests/unit/test__opentelemetry_metrics.py | 341 +++++++++++-- 14 files changed, 2013 insertions(+), 266 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py index 27675bc298a0..f9583d5baba4 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py +++ b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py @@ -26,8 +26,18 @@ import time from collections import deque from contextvars import ContextVar -from dataclasses import dataclass -from typing import Any, Deque, Dict, Iterator, List, Optional, Set, Tuple +from dataclasses import dataclass, field +from typing import ( + Any, + Deque, + Dict, + FrozenSet, + Iterator, + List, + Optional, + Set, + Tuple, +) from urllib.parse import urlsplit from google.cloud.storage.version import __version__ @@ -60,16 +70,17 @@ METRIC_GCP_CLIENT_REQUEST_DURATION = "gcp.client.request.duration" METRIC_STORAGE_CLIENT_OPERATIONS = "gcp.storage.client.operations" METRIC_STORAGE_CLIENT_ATTEMPTS = "gcp.storage.client.attempts" +METRIC_STORAGE_CLIENT_RETRIES = "gcp.storage.client.retries" METRIC_STORAGE_REQUEST_BODY_SIZE = "gcp.storage.client.request.body.size" METRIC_STORAGE_RESPONSE_BODY_SIZE = "gcp.storage.client.response.body.size" -METRIC_STORAGE_OPERATION_TTFB = "gcp.storage.client.operation.ttfb" +METRIC_STORAGE_CLIENT_TTFB = "gcp.storage.client.ttfb" METRIC_STORAGE_CLIENT_ERRORS = "gcp.storage.client.errors" _LATENCY_METRIC_NAMES = ( METRIC_RPC_CLIENT_CALL_DURATION, METRIC_HTTP_CLIENT_REQUEST_DURATION, METRIC_GCP_CLIENT_REQUEST_DURATION, - METRIC_STORAGE_OPERATION_TTFB, + METRIC_STORAGE_CLIENT_TTFB, ) _SIZE_METRIC_NAMES = ( @@ -179,7 +190,6 @@ def is_debug_metrics_enabled( _COMMON_ATTRIBUTES: Dict[str, Any] = { "gcp.client.service": "storage", "gcp.client.version": __version__, - "gcp.client.repo": "googleapis/google-cloud-python", "gcp.client.artifact": "google-cloud-storage", } @@ -256,8 +266,9 @@ def _build_latency_buckets() -> Tuple[float, ...]: def _build_size_buckets() -> Tuple[int, ...]: """Builds the custom byte-size histogram bucket boundaries in bytes. - Uses 128 KiB resolution from 0 up to 4 MiB (33 buckets), then doubles every - bucket up to 16 GiB (12 buckets, 45 total buckets). + Uses 128 KiB increments up to 4 MiB, then doubles the increment on each + subsequent bucket while ``boundary <= 16 GiB`` (48 total buckets, matching + Go and Java SDKs). """ kb = 1024 mb = 1024 * kb @@ -269,10 +280,9 @@ def _build_size_buckets() -> Tuple[int, ...]: while boundary <= 16 * gb: boundaries.append(boundary) - if boundary < 4 * mb: - boundary += increment - else: - boundary *= 2 + boundary += increment + if boundary >= 4 * mb: + increment *= 2 return tuple(boundaries) @@ -362,12 +372,13 @@ def _iter_exception_chain(exc: BaseException) -> Iterator[BaseException]: visited.add(id(nested)) queue.append(nested) - args = getattr(current, "args", ()) - if isinstance(args, (tuple, list)): - for arg in args: - if isinstance(arg, BaseException) and id(arg) not in visited: - visited.add(id(arg)) - queue.append(arg) + for seq_attr in ("args", "errors"): + seq = getattr(current, seq_attr, ()) + if isinstance(seq, (tuple, list)): + for item in seq: + if isinstance(item, BaseException) and id(item) not in visited: + visited.add(id(item)) + queue.append(item) def _is_subclass_by_module( @@ -725,15 +736,96 @@ def normalize_url_template(url_or_path: Optional[str]) -> str: # --------------------------------------------------------------------------- -# 9. Logical Operation Context & Suppression (`ContextVar`) +# 9. Logical Operation Normalization, Context & Suppression (`ContextVar`) # --------------------------------------------------------------------------- -@dataclass(frozen=True) +_GRPC_LOGICAL_OPERATION_OVERRIDES: Dict[str, str] = { + "BidiReadObject": "ReadObject", + "BidiWriteObject": "WriteObject", +} + +_RPC_V2_METHODS: FrozenSet[str] = frozenset( + { + "BidiReadObject", + "BidiWriteObject", + "CancelResumableWrite", + "ComposeObject", + "CreateBucket", + "DeleteBucket", + "DeleteObject", + "GetBucket", + "GetIamPolicy", + "GetObject", + "ListBuckets", + "ListObjects", + "LockBucketRetentionPolicy", + "MoveObject", + "QueryWriteStatus", + "ReadObject", + "RestoreObject", + "RewriteObject", + "SetIamPolicy", + "StartResumableWrite", + "TestIamPermissions", + "UpdateBucket", + "UpdateObject", + "WriteObject", + } +) + + +def normalize_logical_operation(operation: Optional[str]) -> str: + """Normalizes an RPC path or method name into a short logical ``gcp.storage.operation``.""" + if not operation or not isinstance(operation, str): + return "UNKNOWN" + cleaned = operation.strip().strip("/") + if not cleaned: + return "UNKNOWN" + short_name = cleaned.rsplit("/", 1)[-1] if "/" in cleaned else cleaned + return _GRPC_LOGICAL_OPERATION_OVERRIDES.get(short_name, short_name) + + +def resolve_rpc_method( + operation: Optional[str], rpc_method: Optional[str] = None +) -> str: + """Resolves the full RPC path (e.g. ``google.storage.v2.Storage/ReadObject``) for ``rpc.method``.""" + raw = rpc_method if (rpc_method and isinstance(rpc_method, str)) else operation + if not raw or not isinstance(raw, str): + return "UNKNOWN" + cleaned = raw.strip().strip("/") + if not cleaned or cleaned == "UNKNOWN": + return "UNKNOWN" + if "/" in cleaned: + return cleaned + if cleaned in _RPC_V2_METHODS: + return f"google.storage.v2.Storage/{cleaned}" + if "." in cleaned: + return cleaned + return f"google.storage.v1.Storage/{cleaned}" + + +@dataclass class _OperationState: """Represents the active logical GCS Veneer operation in the current context.""" client_method: str + operation: str rpc_method: str suppressed: bool = False + start_time: float = field(default_factory=time.perf_counter) + finished: bool = False + last_failure_error_type: Optional[str] = None + recorded_attempts: int = 0 + + def mark_attempt_failed(self, error_type: str) -> None: + """Records the error type of a failed attempt for a subsequent retry.""" + if error_type and error_type != "OK": + self.last_failure_error_type = error_type + + def consume_retry_error(self) -> Optional[str]: + """Consumes and returns the prior failed attempt's error type, if any.""" + prior = self.last_failure_error_type + self.last_failure_error_type = None + return prior _OP_STATE: ContextVar[Optional[_OperationState]] = ContextVar( @@ -743,15 +835,19 @@ class _OperationState: def get_current_operation() -> Optional[_OperationState]: """Returns the current logical `_OperationState` in this execution context, if any.""" - return _OP_STATE.get() + op = _OP_STATE.get() + if op is not None and op.finished: + return None + return op @contextlib.contextmanager def start_operation( recorder: Optional["_MetricsRecorder"], client_method: str, - rpc_method: str, + operation: str, suppress: bool = False, + rpc_method: Optional[str] = None, ) -> Iterator[Optional[_OperationState]]: """Context manager tracking a logical user-facing GCS operation. @@ -763,24 +859,30 @@ def start_operation( recorder: Optional `_MetricsRecorder` configured on the client. client_method: Fully qualified Python method name (e.g. ``"storage.Blob.download_to_filename"``). - rpc_method: Canonical GCS RPC operation name - (e.g. ``"storage.objects.get"``). + operation: Short logical GCS operation name + (e.g. ``"ReadObject"``, ``"GetObject"``). suppress: If True, suppresses logical operation and payload size emission for this context and any nested calls (used by ``transfer_manager`` workers and ``BlobReader`` prefetches). + rpc_method: Optional full RPC path + (e.g. ``"google.storage.v2.Storage/BidiReadObject"``). If omitted, + inferred from ``operation``. """ - existing = _OP_STATE.get() + existing = get_current_operation() if existing is not None: if suppress and not existing.suppressed: suppressed_child = _OperationState( client_method=existing.client_method, + operation=existing.operation, rpc_method=existing.rpc_method, suppressed=True, + start_time=existing.start_time, ) child_token = _OP_STATE.set(suppressed_child) try: yield suppressed_child finally: + suppressed_child.finished = True _OP_STATE.reset(child_token) else: yield existing @@ -793,13 +895,19 @@ def start_operation( yield None return + resolved_operation = normalize_logical_operation(operation) + resolved_rpc_method = resolve_rpc_method( + resolved_operation, rpc_method=rpc_method or operation + ) + start_time = time.perf_counter() state = _OperationState( client_method=client_method, - rpc_method=rpc_method, + operation=resolved_operation, + rpc_method=resolved_rpc_method, suppressed=suppress, + start_time=start_time, ) token = _OP_STATE.set(state) - start_time = time.perf_counter() error_type = "OK" try: yield state @@ -807,17 +915,66 @@ def start_operation( error_type = classify_error_type(exc) raise finally: + state.finished = True duration = max(0.0, time.perf_counter() - start_time) _OP_STATE.reset(token) if recorder_active and not suppress and recorder is not None: recorder.record_logical_operation( duration=duration, client_method=client_method, - rpc_method=rpc_method, + operation=resolved_operation, error_type=error_type, + rpc_method=resolved_rpc_method, ) +def record_pending_retry( + recorder: Optional["_MetricsRecorder"], + operation: Optional[str] = None, + network_path: Optional[str] = None, +) -> None: + """Emits a pending ``gcp.storage.client.retries`` increment if a prior attempt failed.""" + if recorder is None or not getattr(recorder, "standard_enabled", False): + return + current_op = get_current_operation() + if current_op is None: + return + prior_error = current_op.consume_retry_error() + if prior_error is not None: + recorder.record_retry( + operation=operation or current_op.operation, + error_type=prior_error, + network_path=network_path, + ) + + +def handle_unrecorded_retryable_failure( + recorder: Optional["_MetricsRecorder"], + exc: BaseException, + attempts_before: int, + will_retry: bool, + network_path: Optional[str] = None, +) -> None: + """Tracks retry state when a failed attempt bypassed transport-level attempt recording.""" + if recorder is None or not getattr(recorder, "standard_enabled", False): + return + current_op = get_current_operation() + if current_op is None: + return + if current_op.recorded_attempts > attempts_before: + return + current_op.recorded_attempts += 1 + prior_error = current_op.consume_retry_error() + if prior_error is not None: + recorder.record_retry( + operation=current_op.operation, + error_type=prior_error, + network_path=network_path, + ) + if will_retry: + current_op.mark_attempt_failed(classify_error_type(exc)) + + def get_active_recorder(obj: Any) -> Optional["_MetricsRecorder"]: """Returns ``obj._metrics_recorder`` if it is an active ``_MetricsRecorder``.""" recorder = getattr(obj, "_metrics_recorder", None) @@ -899,6 +1056,11 @@ def __init__( unit="{attempt}", description="Number of physical network transport attempts executed.", ) + self.retries_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_RETRIES, + unit="{retry}", + description="Number of transport retries executed after a failed attempt.", + ) self.request_body_size = self.meter.create_histogram( METRIC_STORAGE_REQUEST_BODY_SIZE, unit="By", @@ -909,11 +1071,12 @@ def __init__( unit="By", description="Application payload bytes delivered to the caller.", ) - self.operation_ttfb = self.meter.create_histogram( - METRIC_STORAGE_OPERATION_TTFB, + self.ttfb = self.meter.create_histogram( + METRIC_STORAGE_CLIENT_TTFB, unit="s", description="Time elapsed from attempt initiation to receipt of the first response byte.", ) + self.operation_ttfb = self.ttfb self.errors_counter = self.meter.create_counter( METRIC_STORAGE_CLIENT_ERRORS, unit="{error}", @@ -925,8 +1088,10 @@ def __init__( self.client_request_duration = None self.operations_counter = None self.attempts_counter = None + self.retries_counter = None self.request_body_size = None self.response_body_size = None + self.ttfb = None self.operation_ttfb = None self.errors_counter = None @@ -943,13 +1108,13 @@ def _build_operation_attributes( """Builds base attributes enriched with current or explicit operation metadata.""" attrs = self._base_attributes.copy() if current_op is None: - current_op = _OP_STATE.get() - resolved_op = operation or (current_op.rpc_method if current_op else None) + current_op = get_current_operation() + raw_op = operation or (current_op.operation if current_op else None) resolved_method = client_method or ( current_op.client_method if current_op else None ) - if resolved_op is not None: - attrs["gcp.storage.operation"] = resolved_op + if raw_op is not None: + attrs["gcp.storage.operation"] = normalize_logical_operation(raw_op) if resolved_method is not None: attrs["gcp.client.method"] = resolved_method return attrs @@ -958,20 +1123,52 @@ def record_logical_operation( self, duration: float, client_method: str, - rpc_method: str, + operation: str, error_type: str = "OK", + rpc_method: Optional[str] = None, ) -> None: """Records end-to-end logical operation duration and count.""" if not self.standard_enabled or self.meter is None: return - attrs = { + logical_op = normalize_logical_operation(operation) + full_rpc_method = resolve_rpc_method( + logical_op, rpc_method=rpc_method or operation + ) + op_attrs = { **self._base_attributes, - "gcp.storage.operation": rpc_method, + "gcp.storage.operation": logical_op, "gcp.client.method": client_method, "error.type": error_type, } - self.client_request_duration.record(max(0.0, duration), attributes=attrs) - self.operations_counter.add(1, attributes=attrs) + duration_attrs = { + **op_attrs, + "rpc.method": full_rpc_method, + } + self.client_request_duration.record( + max(0.0, duration), attributes=duration_attrs + ) + self.operations_counter.add(1, attributes=op_attrs) + + def record_retry( + self, + operation: Optional[str] = None, + error_type: str = "UNKNOWN", + network_path: Optional[str] = None, + ) -> None: + """Records a transport retry attempt with the prior failed attempt's error type.""" + if not self.standard_enabled or self.meter is None: + return + current_op = get_current_operation() + raw_op = operation or (current_op.operation if current_op else None) + retry_attrs: Dict[str, Any] = { + **self._base_attributes, + "error.type": error_type, + } + if raw_op is not None: + retry_attrs["gcp.storage.operation"] = normalize_logical_operation(raw_op) + if network_path is not None: + retry_attrs["network.path"] = network_path + self.retries_counter.add(1, attributes=retry_attrs) def record_http_attempt( self, @@ -987,11 +1184,24 @@ def record_http_attempt( """Records a single physical HTTP request attempt.""" if not self.standard_enabled or self.meter is None: return - current_op = _OP_STATE.get() - resolved_op = operation or (current_op.rpc_method if current_op else None) - resolved_method = client_method or ( - current_op.client_method if current_op else None + del client_method + current_op = get_current_operation() + raw_op = operation or (current_op.operation if current_op else None) + resolved_op = ( + normalize_logical_operation(raw_op) if raw_op is not None else None ) + if current_op is not None: + current_op.recorded_attempts += 1 + prior_error = current_op.consume_retry_error() + if prior_error is not None: + self.record_retry( + operation=resolved_op, + error_type=prior_error, + network_path=network_path, + ) + if error_type != "OK": + current_op.mark_attempt_failed(error_type) + valid_code = ( isinstance(status_code, int) and not isinstance(status_code, bool) @@ -1033,29 +1243,47 @@ def record_http_attempt( } if resolved_op is not None: error_attrs["gcp.storage.operation"] = resolved_op - if resolved_method is not None: - error_attrs["gcp.client.method"] = resolved_method + if network_path is not None: + error_attrs["network.path"] = network_path self.errors_counter.add(1, attributes=error_attrs) def record_rpc_attempt( self, duration: float, - operation: str, + operation: Optional[str] = None, status_code: Optional[str] = None, error_type: str = "OK", - client_method: Optional[str] = None, + rpc_method: Optional[str] = None, network_path: Optional[str] = None, + op_state: Optional[_OperationState] = None, ) -> None: """Records a single physical gRPC call attempt.""" if not self.standard_enabled or self.meter is None: return - op_attrs = self._build_operation_attributes( - operation=operation, client_method=client_method + active_op = op_state if op_state is not None else get_current_operation() + raw_rpc = ( + rpc_method or operation or (active_op.rpc_method if active_op else None) ) + resolved_rpc_method = resolve_rpc_method(operation, rpc_method=raw_rpc) + raw_op = operation or rpc_method or (active_op.operation if active_op else None) + resolved_op = normalize_logical_operation(raw_op) + + if active_op is not None: + active_op.recorded_attempts += 1 + prior_error = active_op.consume_retry_error() + if prior_error is not None: + self.record_retry( + operation=resolved_op, + error_type=prior_error, + network_path=network_path, + ) + if error_type != "OK": + active_op.mark_attempt_failed(error_type) # 1. rpc.client.call.duration attributes rpc_attrs: Dict[str, Any] = { - **op_attrs, + **self._base_attributes, + "rpc.method": resolved_rpc_method, "error.type": error_type, } if status_code is not None: @@ -1066,7 +1294,7 @@ def record_rpc_attempt( # 2. gcp.storage.client.attempts attributes attempt_attrs: Dict[str, Any] = { **self._base_attributes, - "gcp.storage.operation": operation, + "gcp.storage.operation": resolved_op, "error.type": error_type, } if status_code is not None: @@ -1080,9 +1308,12 @@ def record_rpc_attempt( # 3. gcp.storage.client.errors attributes if error_type != "OK": error_attrs: Dict[str, Any] = { - **op_attrs, + **self._base_attributes, + "gcp.storage.operation": resolved_op, "error.type": error_type, } + if network_path is not None: + error_attrs["network.path"] = network_path self.errors_counter.add(1, attributes=error_attrs) def record_request_body_size( @@ -1096,7 +1327,7 @@ def record_request_body_size( return if not isinstance(size, int) or isinstance(size, bool): return - current_op = _OP_STATE.get() + current_op = get_current_operation() if current_op is not None and current_op.suppressed: return attrs = self._build_operation_attributes( @@ -1115,7 +1346,7 @@ def record_response_body_size( return if not isinstance(size, int) or isinstance(size, bool): return - current_op = _OP_STATE.get() + current_op = get_current_operation() if current_op is not None and current_op.suppressed: return attrs = self._build_operation_attributes( @@ -1127,18 +1358,21 @@ def record_ttfb( self, duration: float, operation: Optional[str] = None, - client_method: Optional[str] = None, + network_path: Optional[str] = None, ) -> None: """Records time to first response byte (TTFB) in seconds.""" if not self.standard_enabled or self.meter is None: return - current_op = _OP_STATE.get() + current_op = get_current_operation() if current_op is not None and current_op.suppressed: return - attrs = self._build_operation_attributes( - operation=operation, client_method=client_method, current_op=current_op - ) - self.operation_ttfb.record(max(0.0, duration), attributes=attrs) + raw_op = operation or (current_op.operation if current_op else None) + attrs: Dict[str, Any] = self._base_attributes.copy() + if raw_op is not None: + attrs["gcp.storage.operation"] = normalize_logical_operation(raw_op) + if network_path is not None: + attrs["network.path"] = network_path + self.ttfb.record(max(0.0, duration), attributes=attrs) def shutdown(self) -> None: """Flushes pending metrics on a custom MeterProvider if configured.""" diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py index c0a4d70e138b..974dfa194b63 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py @@ -27,6 +27,7 @@ from google.cloud.storage import Blob, _opentelemetry_metrics from google.cloud.storage.asyncio.async_grpc_client import ( AsyncGrpcClient, + _resolve_network_path, ) from google.cloud.storage.asyncio.async_write_object_stream import ( _AsyncWriteObjectStream, @@ -276,6 +277,21 @@ def from_blob( instance.blob = blob return instance + def _current_network_path( + self, + exc: Optional[BaseException] = None, + received_server_response: bool = True, + ) -> str: + """Resolves ``network.path`` from the underlying Bidi write stream call.""" + call = getattr( + getattr(self.write_obj_stream, "socket_like_rpc", None), "call", None + ) + return _resolve_network_path( + call=call, + exc=exc, + received_server_response=received_server_response, + ) + async def state_lookup(self) -> int: """Returns the persisted_size @@ -289,6 +305,7 @@ async def state_lookup(self) -> int: with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncAppendableObjectWriter.state_lookup", + operation="WriteObject", rpc_method=_BIDI_WRITE_OBJECT_RPC, ): if not self._is_stream_open: @@ -354,6 +371,7 @@ async def open( with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncAppendableObjectWriter.open", + operation="WriteObject", rpc_method=_BIDI_WRITE_OBJECT_RPC, ): if self._is_stream_open: @@ -362,6 +380,10 @@ async def open( retry_policy = self._merge_retry_policy(retry_policy) async def _do_open(): + current_op = _opentelemetry_metrics.get_current_operation() + attempts_before = ( + current_op.recorded_attempts if current_op is not None else 0 + ) current_metadata = list(metadata) if metadata else [] # Cleanup stream from previous failed attempt, if any. @@ -397,9 +419,31 @@ async def _do_open(): ) ) - await self.write_obj_stream.open( - metadata=current_metadata if current_metadata else None - ) + try: + await self.write_obj_stream.open( + metadata=current_metadata if current_metadata else None + ) + except Exception as exc: + if recorder is not None: + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + exc, + attempts_before=attempts_before, + will_retry=bool(retry_policy._predicate(exc)), + network_path=self._current_network_path( + exc=exc, received_server_response=False + ), + ) + raise + + if recorder is not None: + _opentelemetry_metrics.record_pending_retry( + recorder, + operation="WriteObject", + network_path=self._current_network_path( + received_server_response=True + ), + ) if self.write_obj_stream.generation_number: self.generation = self.write_obj_stream.generation_number @@ -450,6 +494,7 @@ async def append( with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncAppendableObjectWriter.append", + operation="WriteObject", rpc_method=_BIDI_WRITE_OBJECT_RPC, ): if not self._is_stream_open: @@ -494,6 +539,14 @@ async def generator(): self._is_stream_open = False await self.open(metadata=current_metadata) + if recorder is not None: + _opentelemetry_metrics.record_pending_retry( + recorder, + operation="WriteObject", + network_path=self._current_network_path( + received_server_response=True + ), + ) write_state.persisted_size = self.persisted_size write_state.write_handle = self.write_handle @@ -548,6 +601,10 @@ async def generator(): retry_manager = _BidiStreamRetryManager( _WriteResumptionStrategy(), lambda r, s: send_and_recv_generator(r, s, metadata), + recorder=recorder, + network_path_getter=lambda: self._current_network_path( + received_server_response=True + ), ) await retry_manager.execute({"write_state": write_state}, retry_policy) @@ -555,9 +612,7 @@ async def generator(): self.bytes_appended_since_last_flush = write_state.bytes_since_last_flush self.offset = write_state.bytes_sent if recorder is not None and not self._suppress_append_body_size: - recorder.record_request_body_size( - len(data), operation=_BIDI_WRITE_OBJECT_RPC - ) + recorder.record_request_body_size(len(data), operation="WriteObject") async def simple_flush(self) -> None: """Flushes the data to the server. @@ -572,6 +627,7 @@ async def simple_flush(self) -> None: with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncAppendableObjectWriter.simple_flush", + operation="WriteObject", rpc_method=_BIDI_WRITE_OBJECT_RPC, ): if not self._is_stream_open: @@ -599,6 +655,7 @@ async def flush(self) -> int: with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncAppendableObjectWriter.flush", + operation="WriteObject", rpc_method=_BIDI_WRITE_OBJECT_RPC, ): if not self._is_stream_open: @@ -663,6 +720,7 @@ async def close( with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncAppendableObjectWriter.close", + operation="WriteObject", rpc_method=_BIDI_WRITE_OBJECT_RPC, ): if not self._is_stream_open: @@ -687,23 +745,48 @@ async def close( async def _do_close(): nonlocal attempt_count attempt_count += 1 + current_op = _opentelemetry_metrics.get_current_operation() + attempts_before = ( + current_op.recorded_attempts if current_op is not None else 0 + ) - if attempt_count > 1: - logger.info( - f"Re-opening the stream for close retry attempt: {attempt_count}" - ) - self._is_stream_open = False - await self.open() - if ( - self.offset is not None - and expected_offset is not None - and self.offset != expected_offset - ): - raise exceptions.InternalServerError( - f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" + try: + if attempt_count > 1: + logger.info( + f"Re-opening the stream for close retry attempt: {attempt_count}" ) + self._is_stream_open = False + await self.open() + if recorder is not None: + _opentelemetry_metrics.record_pending_retry( + recorder, + operation="WriteObject", + network_path=self._current_network_path( + received_server_response=True + ), + ) + if ( + self.offset is not None + and expected_offset is not None + and self.offset != expected_offset + ): + raise exceptions.InternalServerError( + f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" + ) - await self.write_obj_stream.close() + await self.write_obj_stream.close() + except Exception as exc: + if recorder is not None: + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + exc, + attempts_before=attempts_before, + will_retry=bool(retry_policy._predicate(exc)), + network_path=self._current_network_path( + exc=exc, received_server_response=False + ), + ) + raise return self.persisted_size try: @@ -756,6 +839,7 @@ async def finalize( with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncAppendableObjectWriter.finalize", + operation="WriteObject", rpc_method=_BIDI_WRITE_OBJECT_RPC, ): if not self._is_stream_open: @@ -787,24 +871,49 @@ async def finalize( async def _do_finalize(): nonlocal attempt_count attempt_count += 1 + current_op = _opentelemetry_metrics.get_current_operation() + attempts_before = ( + current_op.recorded_attempts if current_op is not None else 0 + ) - if attempt_count > 1: - logger.info( - f"Re-opening the stream for finalize retry attempt: {attempt_count}" - ) - self._is_stream_open = False - await self.open() - if ( - self.offset is not None - and expected_offset is not None - and self.offset != expected_offset - ): - raise exceptions.InternalServerError( - f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" + try: + if attempt_count > 1: + logger.info( + f"Re-opening the stream for finalize retry attempt: {attempt_count}" ) + self._is_stream_open = False + await self.open() + if recorder is not None: + _opentelemetry_metrics.record_pending_retry( + recorder, + operation="WriteObject", + network_path=self._current_network_path( + received_server_response=True + ), + ) + if ( + self.offset is not None + and expected_offset is not None + and self.offset != expected_offset + ): + raise exceptions.InternalServerError( + f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" + ) - await self.write_obj_stream.send(finalize_req) - response = await self.write_obj_stream.recv() + await self.write_obj_stream.send(finalize_req) + response = await self.write_obj_stream.recv() + except Exception as exc: + if recorder is not None: + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + exc, + attempts_before=attempts_before, + will_retry=bool(retry_policy._predicate(exc)), + network_path=self._current_network_path( + exc=exc, received_server_response=False + ), + ) + raise self.object_resource = response.resource self.persisted_size = self.object_resource.size return self.object_resource @@ -857,6 +966,7 @@ async def append_from_file( with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncAppendableObjectWriter.append_from_file", + operation="WriteObject", rpc_method=_BIDI_WRITE_OBJECT_RPC, ): total_bytes = 0 @@ -869,6 +979,4 @@ async def append_from_file( finally: self._suppress_append_body_size = prev_suppress if recorder is not None and total_bytes > 0: - recorder.record_request_body_size( - total_bytes, operation=_BIDI_WRITE_OBJECT_RPC - ) + recorder.record_request_body_size(total_bytes, operation="WriteObject") diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py index 62931c67bcc6..a9bb0135f40a 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py @@ -15,9 +15,11 @@ """An async client for interacting with Google Cloud Storage using the gRPC API.""" import asyncio +import ipaddress +import re import time from collections import deque -from typing import Any, Optional, Tuple +from typing import Any, Iterator, Optional, Tuple import grpc from google.auth import credentials as auth_credentials @@ -34,9 +36,15 @@ ) _GRPC_STATUS_NAMES = {sc.name for sc in grpc.StatusCode} +_DIRECTPATH_IPV4_NET = ipaddress.ip_network("34.126.0.0/18") +_DIRECTPATH_IPV6_NET = ipaddress.ip_network("2001:4860:8040::/42") +_GFE_METADATA_KEYS = frozenset({"x-goog-gfe-service-time", "x-guploader-uploadid"}) +_PEER_IP_RE = re.compile( + r"ipv4:([0-9.]+)|ipv6:(?:(?:\[|%5B|%5b)([0-9a-fA-F:.]+)(?:\]|%5D|%5d)|([0-9a-fA-F:]+))" +) -def _extract_rpc_operation(method_name: Any) -> str: +def _extract_rpc_method(method_name: Any) -> str: """Normalizes a gRPC method path into ``package.Service/Method``.""" if isinstance(method_name, bytes): method_name = method_name.decode("utf-8", errors="replace") @@ -46,6 +54,14 @@ def _extract_rpc_operation(method_name: Any) -> str: return normalized if normalized else "UNKNOWN" +_extract_rpc_operation = _extract_rpc_method + + +def _extract_logical_operation(rpc_method: str) -> str: + """Extracts the short logical ``gcp.storage.operation`` from a gRPC method path.""" + return _opentelemetry_metrics.normalize_logical_operation(rpc_method) + + def _resolve_status_and_error(exc: BaseException) -> Tuple[Optional[str], str]: """Resolves ``(rpc.response.status_code, error.type)`` from an exception.""" if isinstance(exc, asyncio.CancelledError): @@ -76,33 +92,48 @@ def _resolve_status_and_error(exc: BaseException) -> Tuple[Optional[str], str]: return status_code, error_type -def _resolve_network_path( - call: Any, default_network_path: Optional[str] = None -) -> Optional[str]: - """Resolves ``"directpath"`` or ``"cloudpath"`` from ``call.peer()`` or channel config. - - Note: Python's ``grpc.aio.Call`` does not currently expose ``peer()`` on - C-core async calls; when ``peer()`` is unavailable, falls back to the - channel's configured ``default_network_path``. - """ - peer_fn = getattr(call, "peer", None) - if callable(peer_fn): +def _extract_peer_ip(text: str) -> Optional[str]: + """Extracts the last valid peer IPv4/IPv6 address from a gRPC peer or debug string.""" + last_valid: Optional[str] = None + for match in _PEER_IP_RE.finditer(text): + candidate = match.group(1) or match.group(2) or match.group(3) try: - peer_val = peer_fn() - if isinstance(peer_val, bytes): - peer_val = peer_val.decode("utf-8", errors="replace") - if isinstance(peer_val, str): - if "2001:4860:8040:" in peer_val or peer_val.startswith("ipv4:34.126."): - return "directpath" - if peer_val.startswith(("ipv4:", "ipv6:")): - return "cloudpath" - except Exception: # pragma: NO COVER - pass - return default_network_path - - -def _mark_call_closing_cleanly(call: Any) -> None: - """Marks a wrapped gRPC call as closing cleanly before normal cancellation.""" + ipaddress.ip_address(candidate) + last_valid = candidate + except ValueError: + continue + return last_valid + + +def _classify_ip(ip_str: str) -> str: + """Classifies a validated IP address string into ``'directpath'`` or ``'cloudpath'``.""" + addr = ipaddress.ip_address(ip_str) + if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None: + addr = addr.ipv4_mapped + if isinstance(addr, ipaddress.IPv4Address): + return "directpath" if addr in _DIRECTPATH_IPV4_NET else "cloudpath" + return "directpath" if addr in _DIRECTPATH_IPV6_NET else "cloudpath" + + +def _inspect_metadata(md: Any) -> Tuple[bool, bool]: + """Inspects a gRPC metadata sequence for non-empty headers and GFE markers.""" + if not isinstance(md, (tuple, list, grpc.aio.Metadata)): + return False, False + has_entries = False + for entry in md: + has_entries = True + key = getattr(entry, "key", None) + if key is None and isinstance(entry, (tuple, list)) and len(entry) >= 1: + key = entry[0] + if isinstance(key, bytes): + key = key.decode("utf-8", errors="replace") + if isinstance(key, str) and key.lower() in _GFE_METADATA_KEYS: + return True, True + return has_entries, False + + +def _iter_wrapped_calls(call: Any) -> Iterator[Any]: + """Yields ``call`` and any inner wrapped gRPC call objects.""" seen = set() queue = deque([call]) steps = 0 @@ -112,11 +143,8 @@ def _mark_call_closing_cleanly(call: Any) -> None: if current is None or id(current) in seen: continue seen.add(id(current)) + yield current inst_dict = getattr(current, "__dict__", None) - if isinstance(current, _TrackedCallBase) or ( - isinstance(inst_dict, dict) and "_closing_cleanly" in inst_dict - ): - current._closing_cleanly = True if not isinstance(inst_dict, dict): continue for attr in ( @@ -136,6 +164,147 @@ def _mark_call_closing_cleanly(call: Any) -> None: pass +def _resolve_network_path( + call: Any = None, + exc: Optional[BaseException] = None, + direct_path_attempted: Optional[bool] = None, + received_server_response: bool = False, +) -> str: + """Resolves per-call ``network.path`` (``'directpath'``, ``'cloudpath'``, or ``'unknown'``). + + Detection priority: + 1. GFE response metadata headers (``x-goog-gfe-service-time``, + ``x-guploader-uploadid``) on the Cython call or ``RpcError`` -> ``'cloudpath'``. + 2. Remote peer IP from ``call.peer()`` or C-core ``debug_error_string()`` + matched against DirectPath CIDRs ``34.126.0.0/18`` and + ``2001:4860:8040::/42`` -> ``'directpath'`` or ``'cloudpath'``. + 3. Fallback when Python ``grpc.aio`` hides ``peer()`` and no GFE header is + present: if the server responded, ``'directpath'`` when DirectPath was + attempted on the channel else ``'cloudpath'``; if no server response was + received (e.g. DNS or pre-connection failure), ``'unknown'``. + """ + has_server_metadata = False + detected_peer_ip: Optional[str] = None + + if call is not None: + for current in _iter_wrapped_calls(call): + if direct_path_attempted is None and isinstance(current, _TrackedCallBase): + direct_path_attempted = current._direct_path_attempted + + inst_dict = getattr(current, "__dict__", None) + cython_call = getattr(current, "_cython_call", None) + status_obj = ( + getattr(cython_call, "_status", None) + if cython_call is not None + else None + ) + status_trailing = None + if status_obj is not None: + trailing_fn = getattr(status_obj, "trailing_metadata", None) + status_trailing = ( + trailing_fn() + if callable(trailing_fn) + else getattr(status_obj, "_trailing_metadata", None) + ) + + md_candidates = [ + getattr(cython_call, "_initial_metadata", None) + if cython_call is not None + else None, + getattr(cython_call, "_trailing_metadata", None) + if cython_call is not None + else None, + status_trailing, + inst_dict.get("_initial_metadata") + if isinstance(inst_dict, dict) + else None, + inst_dict.get("_trailing_metadata") + if isinstance(inst_dict, dict) + else None, + ] + for md in md_candidates: + entries, is_gfe = _inspect_metadata(md) + if is_gfe: + return "cloudpath" + if entries: + has_server_metadata = True + + if detected_peer_ip is None: + peer_fn = getattr(current, "peer", None) + if callable(peer_fn): + try: + peer_val = peer_fn() + if isinstance(peer_val, bytes): + peer_val = peer_val.decode("utf-8", errors="replace") + if isinstance(peer_val, str): + detected_peer_ip = _extract_peer_ip(peer_val) + except Exception: # pragma: NO COVER + pass + + if detected_peer_ip is None and status_obj is not None: + dbg_fn = getattr(status_obj, "debug_error_string", None) + dbg_val = ( + dbg_fn() + if callable(dbg_fn) + else getattr(status_obj, "_debug_error_string", None) + ) + if isinstance(dbg_val, bytes): + dbg_val = dbg_val.decode("utf-8", errors="replace") + if isinstance(dbg_val, str): + detected_peer_ip = _extract_peer_ip(dbg_val) + + if exc is not None: + for item in _opentelemetry_metrics._iter_exception_chain(exc): + for md_attr in ("initial_metadata", "trailing_metadata"): + md_fn = getattr(item, md_attr, None) + md = md_fn() if callable(md_fn) else md_fn + entries, is_gfe = _inspect_metadata(md) + if is_gfe: + return "cloudpath" + if entries: + has_server_metadata = True + + if detected_peer_ip is None: + dbg_fn = getattr(item, "debug_error_string", None) + dbg_val = dbg_fn() if callable(dbg_fn) else dbg_fn + if isinstance(dbg_val, bytes): + dbg_val = dbg_val.decode("utf-8", errors="replace") + if isinstance(dbg_val, str): + detected_peer_ip = _extract_peer_ip(dbg_val) + + if detected_peer_ip is not None: + return _classify_ip(detected_peer_ip) + + server_responded = received_server_response or has_server_metadata + if direct_path_attempted is False: + return "cloudpath" if server_responded else "unknown" + if direct_path_attempted is True: + return "directpath" if server_responded else "unknown" + return "unknown" + + +def _mark_call_closing_cleanly(call: Any) -> None: + """Marks a wrapped gRPC call as closing cleanly before normal cancellation.""" + for current in _iter_wrapped_calls(call): + inst_dict = getattr(current, "__dict__", None) + if isinstance(current, _TrackedCallBase) or ( + isinstance(inst_dict, dict) and "_closing_cleanly" in inst_dict + ): + current._closing_cleanly = True + + +def _bind_call_to_operation( + call: Any, op_state: Optional[_opentelemetry_metrics._OperationState] +) -> None: + """Binds a long-lived gRPC stream call to the currently active operation.""" + if op_state is None: + return + for current in _iter_wrapped_calls(call): + if isinstance(current, _TrackedCallBase): + current._op_state = op_state + current._start_time = op_state.start_time + + class _TrackedCallBase: """Base proxy for ``grpc.aio.Call`` objects that records attempt metrics once.""" @@ -143,43 +312,74 @@ def __init__( self, call: Any, recorder: _opentelemetry_metrics._MetricsRecorder, - operation: str, + rpc_method: str, + logical_operation: str, start_time: float, op_state: Optional[_opentelemetry_metrics._OperationState], - network_path: Optional[str] = None, + direct_path_attempted: bool = False, ) -> None: self._call = call self._recorder = recorder - self._operation = operation + self._rpc_method = rpc_method + self._logical_operation = logical_operation self._start_time = start_time - self._client_method = op_state.client_method if op_state is not None else None - self._network_path = network_path + self._op_state = op_state + self._direct_path_attempted = direct_path_attempted + self._received_response = False self._recorded = False self._closing_cleanly = False def __getattr__(self, name: str) -> Any: return getattr(self._call, name) - def _record_once(self, status_code: Optional[str], error_type: str) -> None: + def _record_once( + self, + status_code: Optional[str], + error_type: str, + exc: Optional[BaseException] = None, + received_response: bool = False, + ) -> None: if not self._recorded: self._recorded = True - duration = max(0.0, time.perf_counter() - self._start_time) - resolved_path = _resolve_network_path(self._call, self._network_path) + current_op = _opentelemetry_metrics.get_current_operation() + fallback_op = ( + self._op_state + if (self._op_state is not None and not self._op_state.finished) + else None + ) + active_op = current_op or fallback_op + effective_start = ( + current_op.start_time + if (current_op is not None and current_op is not self._op_state) + else self._start_time + ) + duration = max(0.0, time.perf_counter() - effective_start) + resolved_path = _resolve_network_path( + call=self._call, + exc=exc, + direct_path_attempted=self._direct_path_attempted, + received_server_response=received_response or self._received_response, + ) self._recorder.record_rpc_attempt( duration=duration, - operation=self._operation, + operation=self._logical_operation, status_code=status_code, error_type=error_type, - client_method=self._client_method, + rpc_method=self._rpc_method, network_path=resolved_path, + op_state=active_op, ) def _record_from_exception(self, exc: BaseException) -> None: if isinstance(exc, asyncio.CancelledError) and self._closing_cleanly: - self._record_once(status_code="OK", error_type="OK") + self._record_once( + status_code="OK", + error_type="OK", + received_response=self._received_response, + ) else: status_code, error_type = _resolve_status_and_error(exc) - self._record_once(status_code=status_code, error_type=error_type) + self._record_once(status_code=status_code, error_type=error_type, exc=exc) async def wait_for_connection(self) -> None: try: @@ -191,7 +391,11 @@ async def wait_for_connection(self) -> None: def cancel(self) -> bool: result = self._call.cancel() if self._closing_cleanly: - self._record_once(status_code="OK", error_type="OK") + self._record_once( + status_code="OK", + error_type="OK", + received_response=self._received_response, + ) else: self._record_once(status_code="CANCELLED", error_type="CANCELLED") return result @@ -234,7 +438,8 @@ async def _await_impl(self) -> Any: self._record_from_exception(exc) raise else: - self._record_once(status_code="OK", error_type="OK") + self._received_response = True + self._record_once(status_code="OK", error_type="OK", received_response=True) return response def __await__(self): @@ -248,11 +453,16 @@ async def _aiter_impl(self): completed_normally = False try: async for response in self._call: + self._received_response = True yield response completed_normally = True except GeneratorExit: if self._closing_cleanly: - self._record_once(status_code="OK", error_type="OK") + self._record_once( + status_code="OK", + error_type="OK", + received_response=self._received_response, + ) else: self._record_once(status_code="CANCELLED", error_type="CANCELLED") raise @@ -261,7 +471,9 @@ async def _aiter_impl(self): raise finally: if completed_normally: - self._record_once(status_code="OK", error_type="OK") + self._record_once( + status_code="OK", error_type="OK", received_response=True + ) def __aiter__(self): return self._aiter_impl() @@ -273,8 +485,11 @@ async def read(self) -> Any: self._record_from_exception(exc) raise else: + self._received_response = True if response is grpc.aio.EOF or response == grpc.aio.EOF: - self._record_once(status_code="OK", error_type="OK") + self._record_once( + status_code="OK", error_type="OK", received_response=True + ) return response @@ -340,13 +555,14 @@ class _MetricsUnaryUnaryInterceptor(grpc.aio.UnaryUnaryClientInterceptor): def __init__( self, recorder: _opentelemetry_metrics._MetricsRecorder, - network_path: Optional[str] = None, + direct_path_attempted: bool = False, ) -> None: self._recorder = recorder - self._network_path = network_path + self._direct_path_attempted = direct_path_attempted async def intercept_unary_unary(self, continuation, client_call_details, request): - operation = _extract_rpc_operation(getattr(client_call_details, "method", None)) + rpc_method = _extract_rpc_method(getattr(client_call_details, "method", None)) + logical_operation = _extract_logical_operation(rpc_method) op_state = _opentelemetry_metrics.get_current_operation() start_time = time.perf_counter() try: @@ -354,24 +570,29 @@ async def intercept_unary_unary(self, continuation, client_call_details, request except BaseException as exc: duration = max(0.0, time.perf_counter() - start_time) status_code, error_type = _resolve_status_and_error(exc) + resolved_path = _resolve_network_path( + exc=exc, + direct_path_attempted=self._direct_path_attempted, + received_server_response=False, + ) self._recorder.record_rpc_attempt( duration=duration, - operation=operation, + operation=logical_operation, status_code=status_code, error_type=error_type, - client_method=( - op_state.client_method if op_state is not None else None - ), - network_path=self._network_path, + rpc_method=rpc_method, + network_path=resolved_path, + op_state=op_state, ) raise return _TrackedUnaryUnaryCall( call, self._recorder, - operation, + rpc_method, + logical_operation, start_time, op_state, - network_path=self._network_path, + direct_path_attempted=self._direct_path_attempted, ) @@ -381,13 +602,14 @@ class _MetricsUnaryStreamInterceptor(grpc.aio.UnaryStreamClientInterceptor): def __init__( self, recorder: _opentelemetry_metrics._MetricsRecorder, - network_path: Optional[str] = None, + direct_path_attempted: bool = False, ) -> None: self._recorder = recorder - self._network_path = network_path + self._direct_path_attempted = direct_path_attempted async def intercept_unary_stream(self, continuation, client_call_details, request): - operation = _extract_rpc_operation(getattr(client_call_details, "method", None)) + rpc_method = _extract_rpc_method(getattr(client_call_details, "method", None)) + logical_operation = _extract_logical_operation(rpc_method) op_state = _opentelemetry_metrics.get_current_operation() start_time = time.perf_counter() try: @@ -395,24 +617,29 @@ async def intercept_unary_stream(self, continuation, client_call_details, reques except BaseException as exc: duration = max(0.0, time.perf_counter() - start_time) status_code, error_type = _resolve_status_and_error(exc) + resolved_path = _resolve_network_path( + exc=exc, + direct_path_attempted=self._direct_path_attempted, + received_server_response=False, + ) self._recorder.record_rpc_attempt( duration=duration, - operation=operation, + operation=logical_operation, status_code=status_code, error_type=error_type, - client_method=( - op_state.client_method if op_state is not None else None - ), - network_path=self._network_path, + rpc_method=rpc_method, + network_path=resolved_path, + op_state=op_state, ) raise return _TrackedUnaryStreamCall( call, self._recorder, - operation, + rpc_method, + logical_operation, start_time, op_state, - network_path=self._network_path, + direct_path_attempted=self._direct_path_attempted, ) @@ -422,15 +649,16 @@ class _MetricsStreamUnaryInterceptor(grpc.aio.StreamUnaryClientInterceptor): def __init__( self, recorder: _opentelemetry_metrics._MetricsRecorder, - network_path: Optional[str] = None, + direct_path_attempted: bool = False, ) -> None: self._recorder = recorder - self._network_path = network_path + self._direct_path_attempted = direct_path_attempted async def intercept_stream_unary( self, continuation, client_call_details, request_iterator ): - operation = _extract_rpc_operation(getattr(client_call_details, "method", None)) + rpc_method = _extract_rpc_method(getattr(client_call_details, "method", None)) + logical_operation = _extract_logical_operation(rpc_method) op_state = _opentelemetry_metrics.get_current_operation() start_time = time.perf_counter() try: @@ -438,24 +666,29 @@ async def intercept_stream_unary( except BaseException as exc: duration = max(0.0, time.perf_counter() - start_time) status_code, error_type = _resolve_status_and_error(exc) + resolved_path = _resolve_network_path( + exc=exc, + direct_path_attempted=self._direct_path_attempted, + received_server_response=False, + ) self._recorder.record_rpc_attempt( duration=duration, - operation=operation, + operation=logical_operation, status_code=status_code, error_type=error_type, - client_method=( - op_state.client_method if op_state is not None else None - ), - network_path=self._network_path, + rpc_method=rpc_method, + network_path=resolved_path, + op_state=op_state, ) raise return _TrackedStreamUnaryCall( call, self._recorder, - operation, + rpc_method, + logical_operation, start_time, op_state, - network_path=self._network_path, + direct_path_attempted=self._direct_path_attempted, ) @@ -465,15 +698,16 @@ class _MetricsStreamStreamInterceptor(grpc.aio.StreamStreamClientInterceptor): def __init__( self, recorder: _opentelemetry_metrics._MetricsRecorder, - network_path: Optional[str] = None, + direct_path_attempted: bool = False, ) -> None: self._recorder = recorder - self._network_path = network_path + self._direct_path_attempted = direct_path_attempted async def intercept_stream_stream( self, continuation, client_call_details, request_iterator ): - operation = _extract_rpc_operation(getattr(client_call_details, "method", None)) + rpc_method = _extract_rpc_method(getattr(client_call_details, "method", None)) + logical_operation = _extract_logical_operation(rpc_method) op_state = _opentelemetry_metrics.get_current_operation() start_time = time.perf_counter() try: @@ -481,37 +715,50 @@ async def intercept_stream_stream( except BaseException as exc: duration = max(0.0, time.perf_counter() - start_time) status_code, error_type = _resolve_status_and_error(exc) + resolved_path = _resolve_network_path( + exc=exc, + direct_path_attempted=self._direct_path_attempted, + received_server_response=False, + ) self._recorder.record_rpc_attempt( duration=duration, - operation=operation, + operation=logical_operation, status_code=status_code, error_type=error_type, - client_method=( - op_state.client_method if op_state is not None else None - ), - network_path=self._network_path, + rpc_method=rpc_method, + network_path=resolved_path, + op_state=op_state, ) raise return _TrackedStreamStreamCall( call, self._recorder, - operation, + rpc_method, + logical_operation, start_time, op_state, - network_path=self._network_path, + direct_path_attempted=self._direct_path_attempted, ) def _create_metrics_interceptors( recorder: _opentelemetry_metrics._MetricsRecorder, - network_path: Optional[str] = None, + direct_path_attempted: bool = False, ) -> Tuple[grpc.aio.ClientInterceptor, ...]: """Creates the 4 ``grpc.aio`` client interceptors for ``recorder``.""" return ( - _MetricsUnaryUnaryInterceptor(recorder, network_path=network_path), - _MetricsUnaryStreamInterceptor(recorder, network_path=network_path), - _MetricsStreamUnaryInterceptor(recorder, network_path=network_path), - _MetricsStreamStreamInterceptor(recorder, network_path=network_path), + _MetricsUnaryUnaryInterceptor( + recorder, direct_path_attempted=direct_path_attempted + ), + _MetricsUnaryStreamInterceptor( + recorder, direct_path_attempted=direct_path_attempted + ), + _MetricsStreamUnaryInterceptor( + recorder, direct_path_attempted=direct_path_attempted + ), + _MetricsStreamStreamInterceptor( + recorder, direct_path_attempted=direct_path_attempted + ), ) @@ -572,6 +819,12 @@ class AsyncGrpcClient: metrics. If ``None``, falls back to the ``GCP_STORAGE_PYTHON_ENABLE_OTEL_DEBUG_METRICS`` environment variable, or ``False`` if unset. + + :type meter_provider: :class:`~opentelemetry.metrics.MeterProvider` or None + :param meter_provider: + (Optional, Experimental) A custom OpenTelemetry ``MeterProvider`` + instance used to create metric instruments. If ``None``, falls back to + the globally registered OpenTelemetry ``MeterProvider``. """ def __init__( @@ -584,9 +837,11 @@ def __init__( attempt_direct_path_xds_over_interconnect=False, enable_metrics=None, enable_debug_metrics=None, + meter_provider=None, ): self._enable_metrics = enable_metrics self._enable_debug_metrics = enable_debug_metrics + self._meter_provider = meter_provider self._metrics_enabled = _opentelemetry_metrics.is_metrics_enabled( enable_metrics ) @@ -603,6 +858,7 @@ def __init__( self._metrics_recorder = _opentelemetry_metrics._MetricsRecorder( enable_metrics=enable_metrics, enable_debug_metrics=enable_debug_metrics, + meter_provider=meter_provider, api_endpoint=api_endpoint, rpc_system="grpc", ) @@ -655,7 +911,7 @@ def _create_anonymous_client(self, client_options, credentials): channel = grpc.aio.insecure_channel( client_options.api_endpoint, interceptors=_create_metrics_interceptors( - self._metrics_recorder, network_path="cloudpath" + self._metrics_recorder, direct_path_attempted=False ), ) else: @@ -698,13 +954,12 @@ def _create_async_grpc_client( extra_channel_kwargs = {} if self._metrics_recorder.standard_enabled: - network_path = ( - "directpath" - if (attempt_direct_path or attempt_direct_path_xds_over_interconnect) - else "cloudpath" + direct_path_attempted = bool( + attempt_direct_path or attempt_direct_path_xds_over_interconnect ) extra_channel_kwargs["interceptors"] = _create_metrics_interceptors( - self._metrics_recorder, network_path=network_path + self._metrics_recorder, + direct_path_attempted=direct_path_attempted, ) if attempt_direct_path_xds_over_interconnect: @@ -815,6 +1070,7 @@ async def delete_object( with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncGrpcClient.delete_object", + operation="DeleteObject", rpc_method="google.storage.v2.Storage/DeleteObject", ): _validate_metadata(metadata) @@ -901,6 +1157,7 @@ async def get_object( with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncGrpcClient.get_object", + operation="GetObject", rpc_method="google.storage.v2.Storage/GetObject", ): _validate_metadata(metadata) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py index 95a2ffe798f1..8f17350f33f5 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py @@ -33,6 +33,8 @@ ) from google.cloud.storage.asyncio.async_grpc_client import ( AsyncGrpcClient, + _bind_call_to_operation, + _resolve_network_path, ) from google.cloud.storage.asyncio.async_read_object_stream import ( _AsyncReadObjectStream, @@ -158,6 +160,7 @@ async def create_mrd( with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncMultiRangeDownloader.create_mrd", + operation="ReadObject", rpc_method=_BIDI_READ_OBJECT_RPC, ): mrd = cls( @@ -235,6 +238,21 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): if self.is_stream_open: await self.close() + def _current_network_path( + self, + exc: Optional[BaseException] = None, + received_server_response: bool = True, + ) -> str: + """Resolves ``network.path`` from the underlying Bidi read stream call.""" + call = getattr( + getattr(self.read_obj_str, "socket_like_rpc", None), "call", None + ) + return _resolve_network_path( + call=call, + exc=exc, + received_server_response=received_server_response, + ) + def _on_open_error(self, exc): """Extracts routing token and read handle on redirect error during open.""" logger.warning(f"Error occurred while opening MRD: {exc}") @@ -254,6 +272,7 @@ async def open( with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncMultiRangeDownloader.open", + operation="ReadObject", rpc_method=_BIDI_READ_OBJECT_RPC, ): if self._is_stream_open: @@ -287,6 +306,10 @@ def combined_on_error(exc): ) async def _do_open(): + current_op = _opentelemetry_metrics.get_current_operation() + attempts_before = ( + current_op.recorded_attempts if current_op is not None else 0 + ) current_metadata = list(metadata) if metadata else [] # Cleanup stream from previous failed attempt, if any. @@ -308,6 +331,8 @@ async def _do_open(): generation_number=self.generation, read_handle=self.read_handle, ) + if recorder is not None: + self.read_obj_str._metrics_recorder = recorder if self._routing_token: current_metadata.append( @@ -318,9 +343,31 @@ async def _do_open(): ) self._routing_token = None - await self.read_obj_str.open( - metadata=current_metadata if current_metadata else None - ) + try: + await self.read_obj_str.open( + metadata=current_metadata if current_metadata else None + ) + except Exception as exc: + if recorder is not None: + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + exc, + attempts_before=attempts_before, + will_retry=bool(retry_policy._predicate(exc)), + network_path=self._current_network_path( + exc=exc, received_server_response=False + ), + ) + raise + + if recorder is not None: + _opentelemetry_metrics.record_pending_retry( + recorder, + operation="ReadObject", + network_path=self._current_network_path( + received_server_response=True + ), + ) if self.read_obj_str.generation_number: self.generation = self.read_obj_str.generation_number @@ -336,7 +383,7 @@ async def _do_open(): await retry_policy(_do_open)() self._multiplexer = _StreamMultiplexer(self.read_obj_str) - def _create_stream_factory(self, state, metadata): + def _create_stream_factory(self, state, metadata, recorder=None): """Create a factory that opens a new stream with current routing state.""" async def factory(): @@ -350,6 +397,8 @@ async def factory(): generation_number=self.generation, read_handle=current_handle, ) + if recorder is not None: + stream._metrics_recorder = recorder current_metadata = list(metadata) if metadata else [] if current_token: @@ -423,6 +472,7 @@ async def download_ranges( with _opentelemetry_metrics.start_operation( recorder, client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + operation="ReadObject", rpc_method=_BIDI_READ_OBJECT_RPC, ): if len(read_ranges) > 1000: @@ -436,6 +486,13 @@ async def download_ranges( if not self._is_stream_open: raise ValueError("Underlying bidi-gRPC stream is not open") + _bind_call_to_operation( + getattr( + getattr(self.read_obj_str, "socket_like_rpc", None), "call", None + ), + _opentelemetry_metrics.get_current_operation(), + ) + if retry_policy is None: retry_policy = AsyncRetry(predicate=_is_read_retryable) @@ -509,7 +566,7 @@ async def generator(): else self._multiplexer.stream_generation ) stream_factory = self._create_stream_factory( - state, metadata + state, metadata, recorder=recorder ) await self._multiplexer.reopen_stream( broken_gen, stream_factory @@ -557,7 +614,10 @@ async def generator(): ttfb_recorded = True recorder.record_ttfb( time.perf_counter() - attempt_start, - operation=_BIDI_READ_OBJECT_RPC, + operation="ReadObject", + network_path=self._current_network_path( + received_server_response=True + ), ) for data_range in item.object_data_ranges: if data_range.range_end: @@ -570,7 +630,12 @@ async def generator(): strategy = _ReadResumptionStrategy() retry_manager = _BidiStreamRetryManager( - strategy, send_and_recv_via_multiplexer + strategy, + send_and_recv_via_multiplexer, + recorder=recorder, + network_path_getter=lambda: self._current_network_path( + received_server_response=True + ), ) try: @@ -585,7 +650,7 @@ async def generator(): ds.bytes_written for ds in download_states.values() ) recorder.record_response_body_size( - total_bytes, operation=_BIDI_READ_OBJECT_RPC + total_bytes, operation="ReadObject" ) if initial_state.get("read_handle"): diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py index 8322c3fa3d99..2850dcf52236 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py @@ -26,6 +26,7 @@ from google.cloud.storage.asyncio.async_grpc_client import ( AsyncGrpcClient, _mark_call_closing_cleanly, + _resolve_network_path, ) logger = logging.getLogger(__name__) @@ -166,11 +167,25 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: if response and response.read_handle: self.read_handle = response.read_handle + if self._metrics_recorder is not None: + _opentelemetry_metrics.record_pending_retry( + self._metrics_recorder, + operation="ReadObject", + network_path=self._get_network_path(received_server_response=True), + ) + self._is_stream_open = True except (asyncio.CancelledError, Exception): await self._close_socket_like_rpc() raise + def _get_network_path(self, received_server_response: bool = True) -> str: + """Resolves ``network.path`` for the active gRPC stream call.""" + return _resolve_network_path( + call=getattr(self.socket_like_rpc, "call", None), + received_server_response=received_server_response, + ) + def _mark_closing_cleanly(self) -> None: """Marks the underlying gRPC call as closing cleanly for metrics.""" _mark_call_closing_cleanly(getattr(self.socket_like_rpc, "call", None)) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py index deff956c7efb..ee69a598ff6b 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py @@ -28,10 +28,9 @@ from google.cloud.storage.asyncio.async_grpc_client import ( AsyncGrpcClient, _mark_call_closing_cleanly, + _resolve_network_path, ) -_BIDI_WRITE_OBJECT_RPC = "google.storage.v2.Storage/BidiWriteObject" - class _AsyncWriteObjectStream(_AsyncAbstractObjectStream): """Class representing a gRPC bidi-stream for writing data from a GCS @@ -189,9 +188,16 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: await self._close_socket_like_rpc() raise if self._metrics_recorder is not None: + net_path = self._get_network_path(received_server_response=True) + _opentelemetry_metrics.record_pending_retry( + self._metrics_recorder, + operation="WriteObject", + network_path=net_path, + ) self._metrics_recorder.record_ttfb( time.perf_counter() - attempt_start, - operation=_BIDI_WRITE_OBJECT_RPC, + operation="WriteObject", + network_path=net_path, ) self._is_stream_open = True @@ -216,6 +222,13 @@ async def _close_socket_like_rpc(self) -> None: except Exception: pass + def _get_network_path(self, received_server_response: bool = True) -> str: + """Resolves ``network.path`` for the active gRPC stream call.""" + return _resolve_network_path( + call=getattr(self.socket_like_rpc, "call", None), + received_server_response=received_server_response, + ) + def _mark_closing_cleanly(self) -> None: """Marks the underlying gRPC call as closing cleanly for metrics.""" _mark_call_closing_cleanly(getattr(self.socket_like_rpc, "call", None)) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/retry/bidi_stream_retry_manager.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/retry/bidi_stream_retry_manager.py index 947ee74c1183..9bee357674d7 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/retry/bidi_stream_retry_manager.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/retry/bidi_stream_retry_manager.py @@ -13,8 +13,9 @@ # limitations under the License. import logging -from typing import Any, AsyncIterator, Callable +from typing import Any, AsyncIterator, Callable, Optional +from google.cloud.storage import _opentelemetry_metrics from google.cloud.storage.asyncio.retry.base_strategy import ( _BaseResumptionStrategy, ) @@ -29,15 +30,21 @@ def __init__( self, strategy: _BaseResumptionStrategy, send_and_recv: Callable[..., AsyncIterator[Any]], + recorder: Optional[Any] = None, + network_path_getter: Optional[Callable[[], Optional[str]]] = None, ): """Initializes the retry manager. Args: strategy: The strategy for managing the state of a specific bidi operation (e.g., reads or writes). send_and_recv: An async callable that opens a new gRPC stream. + recorder: Optional `_MetricsRecorder` for retry metric accounting. + network_path_getter: Optional callable returning the current ``network.path``. """ self._strategy = strategy self._send_and_recv = send_and_recv + self._recorder = recorder + self._network_path_getter = network_path_getter async def execute(self, initial_state: Any, retry_policy): """ @@ -50,14 +57,38 @@ async def execute(self, initial_state: Any, retry_policy): state = initial_state async def attempt(): + current_op = _opentelemetry_metrics.get_current_operation() + attempts_before = current_op.recorded_attempts if current_op else 0 requests = self._strategy.generate_requests(state) stream = self._send_and_recv(requests, state) try: async for response in stream: self._strategy.update_state_from_response(response, state) + if self._recorder is not None: + _opentelemetry_metrics.record_pending_retry( + self._recorder, + network_path=( + self._network_path_getter() + if self._network_path_getter is not None + else None + ), + ) return except Exception as e: - if retry_policy._predicate(e): + will_retry = bool(retry_policy._predicate(e)) + if self._recorder is not None: + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + self._recorder, + e, + attempts_before=attempts_before, + will_retry=will_retry, + network_path=( + self._network_path_getter() + if self._network_path_getter is not None + else None + ), + ) + if will_retry: logger.warning( f"Bidi stream operation failed: {e}. Attempting state recovery and retry." ) diff --git a/packages/google-cloud-storage/tests/unit/asyncio/retry/test_bidi_stream_retry_manager.py b/packages/google-cloud-storage/tests/unit/asyncio/retry/test_bidi_stream_retry_manager.py index 71b60ed28ec7..175d9f4f1c6e 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/retry/test_bidi_stream_retry_manager.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/retry/test_bidi_stream_retry_manager.py @@ -154,3 +154,81 @@ async def mock_send_and_recv(*args, **kwargs): ) mock_strategy.recover_state_on_failure.assert_not_called() + + @pytest.mark.asyncio + async def test_execute_records_retry_metrics_with_and_without_network_path_getter( + self, monkeypatch + ): + from google.cloud.storage import _opentelemetry_metrics + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr( + _opentelemetry_metrics, "metrics", mock_metrics, raising=False + ) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + mock_strategy = mock.AsyncMock(spec=base_strategy._BaseResumptionStrategy) + attempt_count = 0 + + async def mock_send_and_recv(*args, **kwargs): + nonlocal attempt_count + attempt_count += 1 + if attempt_count == 1: + raise exceptions.ServiceUnavailable("503") + yield "ok" + + retry_manager = manager._BidiStreamRetryManager( + strategy=mock_strategy, + send_and_recv=mock_send_and_recv, + recorder=recorder, + network_path_getter=lambda: "directpath", + ) + retry_policy = AsyncRetry(predicate=_is_retriable, initial=0.001) + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ): + with mock.patch("asyncio.sleep", new_callable=mock.AsyncMock): + await retry_manager.execute(initial_state={}, retry_policy=retry_policy) + + recorder.retries_counter.add.assert_called_once() + _, retry_kw = recorder.retries_counter.add.call_args + assert retry_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert retry_kw["attributes"]["error.type"] == "UNAVAILABLE" + assert retry_kw["attributes"]["network.path"] == "directpath" + + # Without network_path_getter on non-retryable failure + recorder.retries_counter.add.reset_mock() + + async def fail_send_and_recv(*args, **kwargs): + if False: + yield + raise exceptions.PermissionDenied("403") + + retry_manager_no_getter = manager._BidiStreamRetryManager( + strategy=mock_strategy, + send_and_recv=fail_send_and_recv, + recorder=recorder, + network_path_getter=None, + ) + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ): + with pytest.raises(exceptions.PermissionDenied): + await retry_manager_no_getter.execute( + initial_state={}, retry_policy=DEFAULT_TEST_RETRY + ) + recorder.retries_counter.add.assert_not_called() diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py index 245c920e2784..48a4e06163ff 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -892,20 +892,28 @@ async def test_writer_metrics_recording(self, mock_appendable_writer): mock_recorder.record_logical_operation.reset_mock() await writer.append(b"hello") mock_recorder.record_request_body_size.assert_called_once_with( - 5, operation="google.storage.v2.Storage/BidiWriteObject" + 5, operation="WriteObject" ) mock_recorder.record_logical_operation.assert_called_once() assert ( mock_recorder.record_logical_operation.call_args.kwargs["client_method"] == "storage.asyncio.AsyncAppendableObjectWriter.append" ) + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["operation"] + == "WriteObject" + ) + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["rpc_method"] + == "google.storage.v2.Storage/BidiWriteObject" + ) # append_from_file suppresses per-chunk append metrics and records total size once mock_recorder.record_logical_operation.reset_mock() mock_recorder.record_request_body_size.reset_mock() await writer.append_from_file(io.BytesIO(b"abcdefghij"), block_size=4) mock_recorder.record_request_body_size.assert_called_once_with( - 10, operation="google.storage.v2.Storage/BidiWriteObject" + 10, operation="WriteObject" ) mock_recorder.record_logical_operation.assert_called_once() assert ( @@ -926,6 +934,10 @@ async def test_writer_metrics_recording(self, mock_appendable_writer): mock_recorder.record_logical_operation.call_args.kwargs["client_method"] == f"storage.asyncio.AsyncAppendableObjectWriter.{method_name}" ) + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["operation"] + == "WriteObject" + ) mock_recorder.record_logical_operation.reset_mock() mock_stream.recv.return_value = storage_type.BidiWriteObjectResponse( @@ -938,3 +950,113 @@ async def test_writer_metrics_recording(self, mock_appendable_writer): mock_recorder.record_logical_operation.call_args.kwargs["client_method"] == "storage.asyncio.AsyncAppendableObjectWriter.close" ) + + @pytest.mark.asyncio + async def test_writer_retry_metrics_accounting( + self, mock_appendable_writer, monkeypatch + ): + from google.api_core.retry_async import AsyncRetry + + from google.cloud.storage import _opentelemetry_metrics + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr( + _opentelemetry_metrics, "metrics", mock_metrics, raising=False + ) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + mock_client = mock_appendable_writer["mock_client"] + mock_client._metrics_recorder = recorder + mock_stream = mock_appendable_writer["mock_stream"] + + fast_retry = AsyncRetry( + predicate=_is_write_retryable, + initial=0.001, + maximum=0.001, + on_error=mock.Mock(), + ) + writer = self._make_one(mock_client) + + # 1. open() retry + mock_stream.open = AsyncMock( + side_effect=[exceptions.ServiceUnavailable("503 open"), None] + ) + await writer.open(retry_policy=fast_retry) + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "UNAVAILABLE" + ) + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"][ + "gcp.storage.operation" + ] + == "WriteObject" + ) + + # 2. append() retry + recorder.retries_counter.add.reset_mock() + mock_stream.open = AsyncMock(return_value=None) + mock_stream.send = AsyncMock( + side_effect=[exceptions.ServiceUnavailable("503 append"), None] + ) + await writer.append(b"abc", retry_policy=fast_retry) + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "UNAVAILABLE" + ) + + # 3. close(finalize_on_close=False) retry + recorder.retries_counter.add.reset_mock() + writer.offset = 0 + mock_stream.persisted_size = 0 + redirect = BidiWriteObjectRedirectedError( + routing_token="rt_close", + write_handle=storage_type.BidiWriteHandle(handle=b"hc"), + ) + mock_stream.close = AsyncMock( + side_effect=[exceptions.Aborted("redirect", errors=[redirect]), None] + ) + await writer.close(finalize_on_close=False, retry_policy=fast_retry) + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "ABORTED" + ) + + # 4. finalize() retry + recorder.retries_counter.add.reset_mock() + writer._is_stream_open = True + writer.offset = 0 + mock_stream.close = AsyncMock(return_value=None) + mock_stream.send = AsyncMock(return_value=None) + mock_stream.recv = AsyncMock( + side_effect=[ + exceptions.InternalServerError("500 finalize"), + storage_type.BidiWriteObjectResponse( + resource=storage_type.Object(size=3) + ), + ] + ) + await writer.finalize(retry_policy=fast_retry) + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "INTERNAL" + ) + + # 5. open() failure when metrics are disabled (recorder is None) + mock_client._metrics_recorder = None + writer_disabled = self._make_one(mock_client) + mock_stream.open = AsyncMock(side_effect=exceptions.NotFound("404")) + with pytest.raises(exceptions.NotFound): + await writer_disabled.open() diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py index 2166a26498be..4d016095a74f 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py @@ -13,6 +13,7 @@ # limitations under the License. import asyncio +import types from unittest import mock import grpc @@ -666,12 +667,16 @@ async def test_metrics_interceptors_and_client_lifecycle( mock_gapic_client.transport.close = mock.AsyncMock() mock_async_storage_client.return_value = mock_gapic_client + mock_provider = mock.MagicMock() client = async_grpc_client.AsyncGrpcClient( credentials=_make_credentials(), client_options={"api_endpoint": "custom.storage.googleapis.com:443"}, enable_metrics=True, + meter_provider=mock_provider, ) assert client.metrics_enabled is True + assert client._meter_provider is mock_provider + assert client._metrics_recorder._meter_provider is mock_provider assert ( client._metrics_recorder.server_address == "custom.storage.googleapis.com" ) @@ -692,6 +697,7 @@ async def test_metrics_interceptors_and_client_lifecycle( del_kwargs["client_method"] == "storage.asyncio.AsyncGrpcClient.delete_object" ) + assert del_kwargs["operation"] == "DeleteObject" assert del_kwargs["rpc_method"] == "google.storage.v2.Storage/DeleteObject" assert del_kwargs["error_type"] == "OK" @@ -702,6 +708,7 @@ async def test_metrics_interceptors_and_client_lifecycle( assert ( get_kwargs["client_method"] == "storage.asyncio.AsyncGrpcClient.get_object" ) + assert get_kwargs["operation"] == "GetObject" assert get_kwargs["rpc_method"] == "google.storage.v2.Storage/GetObject" assert get_kwargs["error_type"] == "OK" @@ -955,15 +962,19 @@ async def read(self): ) assert ( async_grpc_client._resolve_network_path( - mock.MagicMock(peer=lambda: 123), "cloudpath" + mock.MagicMock(peer=lambda: 123), + direct_path_attempted=False, + received_server_response=True, ) == "cloudpath" ) assert ( async_grpc_client._resolve_network_path( - mock.MagicMock(peer=lambda: "unix:/tmp/sock"), "cloudpath" + mock.MagicMock(peer=lambda: "unix:/tmp/sock"), + direct_path_attempted=False, + received_server_response=False, ) - == "cloudpath" + == "unknown" ) class _NoneCodeRpcErr(Exception): @@ -1019,3 +1030,369 @@ def code(self): assert ( recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "UNAVAILABLE" ) + + def test_resolve_network_path_comprehensive(self): + # 1. _inspect_metadata edge cases + assert async_grpc_client._inspect_metadata(123) == (False, False) + assert async_grpc_client._inspect_metadata(()) == (False, False) + assert async_grpc_client._inspect_metadata([object()]) == (True, False) + assert async_grpc_client._inspect_metadata([()]) == (True, False) + + # 2. GFE headers override direct_path_attempted=True -> "cloudpath" + call_cy_init = types.SimpleNamespace( + _cython_call=types.SimpleNamespace( + _initial_metadata=( + types.SimpleNamespace(key="x-goog-gfe-service-time", value="10"), + ), + _trailing_metadata=None, + _status=None, + ) + ) + assert ( + async_grpc_client._resolve_network_path( + call=call_cy_init, direct_path_attempted=True + ) + == "cloudpath" + ) + + call_cy_status_fn = types.SimpleNamespace( + _cython_call=types.SimpleNamespace( + _initial_metadata=(), + _trailing_metadata=None, + _status=types.SimpleNamespace( + trailing_metadata=lambda: ((b"X-GUploader-UploadID", b"up1"),) + ), + ) + ) + assert ( + async_grpc_client._resolve_network_path( + call=call_cy_status_fn, direct_path_attempted=True + ) + == "cloudpath" + ) + + call_cy_status_attr = types.SimpleNamespace( + _cython_call=types.SimpleNamespace( + _initial_metadata=None, + _trailing_metadata=None, + _status=types.SimpleNamespace( + _trailing_metadata=(("x-goog-gfe-service-time", "1"),), + _debug_error_string="no ip here", + ), + ) + ) + assert ( + async_grpc_client._resolve_network_path( + call=call_cy_status_attr, direct_path_attempted=True + ) + == "cloudpath" + ) + + # Non-GFE server metadata sets has_server_metadata=True + call_non_gfe = types.SimpleNamespace( + _initial_metadata=[("x-custom-server", "ok")] + ) + assert ( + async_grpc_client._resolve_network_path( + call=call_non_gfe, + direct_path_attempted=True, + received_server_response=False, + ) + == "directpath" + ) + + # GFE metadata on RpcError (grpc.aio.Metadata) overrides direct_path_attempted=True + class _GfeRpcErr(Exception): + def initial_metadata(self): + return grpc.aio.Metadata() + + def trailing_metadata(self): + return grpc.aio.Metadata(("x-goog-gfe-service-time", "5")) + + assert ( + async_grpc_client._resolve_network_path( + exc=_GfeRpcErr(), direct_path_attempted=True + ) + == "cloudpath" + ) + + class _NonGfeAttrErr(Exception): + initial_metadata = None + trailing_metadata = [("x-server", "1")] + debug_error_string = "ipv4:142.250.10.1:443" + + assert ( + async_grpc_client._resolve_network_path( + exc=_NonGfeAttrErr(), direct_path_attempted=True + ) + == "cloudpath" + ) + + # Wrapped call + chained exception when peer IP already detected on outer layer + inner_call = types.SimpleNamespace(peer=lambda: "ipv4:142.250.1.1:443") + outer_call_with_peer = types.SimpleNamespace( + peer=lambda: "ipv4:34.126.1.1:443", _call=inner_call + ) + chained_exc = _NonGfeAttrErr("outer") + chained_exc.__cause__ = RuntimeError("inner") + assert ( + async_grpc_client._resolve_network_path( + call=outer_call_with_peer, exc=chained_exc + ) + == "directpath" + ) + + class _NonGfeNoIpErr(Exception): + initial_metadata = None + trailing_metadata = [("x-server", "1")] + debug_error_string = None + + assert ( + async_grpc_client._resolve_network_path( + exc=_NonGfeNoIpErr(), direct_path_attempted=True + ) + == "directpath" + ) + + # 3. DirectPath IPv4 (/18) and IPv6 (/42) CIDR boundaries + IPv4-mapped IPv6 + for peer_uri, expected in ( + ("ipv4:34.126.0.1:443", "directpath"), + ("ipv4:34.126.63.255:443", "directpath"), + ("ipv4:34.126.64.1:443", "cloudpath"), + ("ipv6:[2001:4860:8040::1]:443", "directpath"), + ("ipv6:%5B2001:4860:8040:ffff::1%5D:443", "directpath"), + ("ipv6:2001:4860:8040::2", "directpath"), + ("ipv6:[2001:4860:8080::1]:443", "cloudpath"), + ("ipv6:[::ffff:34.126.1.2]:443", "directpath"), + ("ipv6:[::ffff:142.250.1.2]:443", "cloudpath"), + ): + c = types.SimpleNamespace(peer=lambda p=peer_uri: p) + assert async_grpc_client._resolve_network_path(call=c) == expected + + # Invalid IP ignored -> falls back + invalid_ip_call = types.SimpleNamespace(peer=lambda: "ipv4:999.999.999.999:443") + assert ( + async_grpc_client._resolve_network_path( + call=invalid_ip_call, + direct_path_attempted=False, + received_server_response=False, + ) + == "unknown" + ) + + # 4. C-core debug_error_string (bytes & str, status & exc) + call_dbg_bytes = types.SimpleNamespace( + _cython_call=types.SimpleNamespace( + _initial_metadata=(), + _trailing_metadata=None, + _status=types.SimpleNamespace( + trailing_metadata=lambda: (), + debug_error_string=lambda: b'{"peer":"ipv4:34.126.5.5:443"}', + ), + ) + ) + assert ( + async_grpc_client._resolve_network_path(call=call_dbg_bytes) == "directpath" + ) + + class _BytesDbgErr(Exception): + def debug_error_string(self): + return b"ipv6:%5b2001:4860:8040::9%5d:443" + + assert ( + async_grpc_client._resolve_network_path(exc=_BytesDbgErr()) == "directpath" + ) + + # 5. Fallback matrix & wrapped _TrackedCallBase discovery + >16 chain cap + assert ( + async_grpc_client._resolve_network_path( + direct_path_attempted=True, received_server_response=True + ) + == "directpath" + ) + assert ( + async_grpc_client._resolve_network_path( + direct_path_attempted=True, received_server_response=False + ) + == "unknown" + ) + assert ( + async_grpc_client._resolve_network_path( + direct_path_attempted=None, received_server_response=True + ) + == "unknown" + ) + + tracked = async_grpc_client._TrackedUnaryUnaryCall( + types.SimpleNamespace(), + mock.MagicMock(), + "google.storage.v2.Storage/GetObject", + "GetObject", + 0.0, + None, + direct_path_attempted=True, + ) + outer = types.SimpleNamespace(_call=tracked) + assert ( + async_grpc_client._resolve_network_path( + call=outer, + direct_path_attempted=None, + received_server_response=True, + ) + == "directpath" + ) + + # Deep chain >16 steps terminates cleanly + head = types.SimpleNamespace() + cur = head + for _ in range(20): + nxt = types.SimpleNamespace() + cur._call = nxt + cur = nxt + assert len(list(async_grpc_client._iter_wrapped_calls(head))) == 16 + + @pytest.mark.asyncio + async def test_unary_retry_and_bidi_active_operation_metrics(self, monkeypatch): + from google.api_core import exceptions as api_exceptions + from google.api_core.retry_async import AsyncRetry + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr( + _opentelemetry_metrics, "metrics", mock_metrics, raising=False + ) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + u_u, _, _, s_s = async_grpc_client._create_metrics_interceptors( + recorder, direct_path_attempted=True + ) + details = mock.MagicMock(method="/google.storage.v2.Storage/GetObject") + + # 1. End-to-end Unary-Unary retry (UNAVAILABLE -> OK) + calls_made = 0 + + class _SimpleAwaitable: + def __init__(self, val=None, exc=None): + self._val = val + self._exc = exc + + async def _run(self): + if self._exc: + raise self._exc + return self._val + + def __await__(self): + return self._run().__await__() + + async def unary_continuation(_details, _req): + nonlocal calls_made + calls_made += 1 + if calls_made == 1: + return _SimpleAwaitable(exc=api_exceptions.ServiceUnavailable("503")) + return _SimpleAwaitable(val="object_meta") + + retry = AsyncRetry( + predicate=lambda e: isinstance(e, api_exceptions.ServiceUnavailable), + initial=0.001, + maximum=0.001, + ) + + async def invoke_rpc(): + call = await u_u.intercept_unary_unary(unary_continuation, details, "req") + return await call + + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncGrpcClient.get_object", + operation="GetObject", + rpc_method="google.storage.v2.Storage/GetObject", + ): + res = await retry(invoke_rpc)() + + assert res == "object_meta" + assert recorder.attempts_counter.add.call_count == 2 + assert recorder.errors_counter.add.call_count == 1 + assert recorder.retries_counter.add.call_count == 1 + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "UNAVAILABLE" + ) + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"][ + "gcp.storage.operation" + ] + == "GetObject" + ) + assert recorder.operations_counter.add.call_count == 1 + assert ( + recorder.operations_counter.add.call_args.kwargs["attributes"]["error.type"] + == "OK" + ) + assert ( + recorder.client_request_duration.record.call_args.kwargs["attributes"][ + "rpc.method" + ] + == "google.storage.v2.Storage/GetObject" + ) + + # 2. Bidi stream opened in open() op and failing mid-stream in download_ranges() op + bidi_details = mock.MagicMock( + method="/google.storage.v2.Storage/BidiReadObject" + ) + raw_bidi = mock.MagicMock() + raw_bidi.read = mock.AsyncMock( + side_effect=api_exceptions.Aborted("Idle stream closed") + ) + + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.open", + operation="ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) as open_op: + tracked_bidi = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=raw_bidi), bidi_details, iter([]) + ) + # Simulate stream opened 100s ago + tracked_bidi._start_time -= 100.0 + + assert open_op.recorded_attempts == 0 + assert open_op.finished is True + async_grpc_client._bind_call_to_operation(tracked_bidi, None) + + recorder.rpc_call_duration.record.reset_mock() + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + operation="ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) as dl_op: + with pytest.raises(api_exceptions.Aborted): + await tracked_bidi.read() + assert dl_op.recorded_attempts == 1 + assert dl_op.last_failure_error_type == "ABORTED" + recorded_dur = recorder.rpc_call_duration.record.call_args.args[0] + assert recorded_dur < 10.0 + + # 3. _bind_call_to_operation for background recv loop task without active ContextVar + tracked_bg = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=mock.MagicMock()), bidi_details, iter([]) + ) + tracked_bg._start_time -= 100.0 + wrapped_bg = types.SimpleNamespace(_call=tracked_bg) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + operation="ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) as bg_op: + async_grpc_client._bind_call_to_operation(wrapped_bg, bg_op) + assert tracked_bg._op_state is bg_op + assert tracked_bg._start_time == bg_op.start_time diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py index 5f6f3992f383..2340f66b5631 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py @@ -927,14 +927,24 @@ async def test_mrd_metrics_recording( buf = BytesIO() await mrd.download_ranges([(0, 4, buf)]) mock_recorder.record_ttfb.assert_called_once() + assert mock_recorder.record_ttfb.call_args.kwargs["operation"] == "ReadObject" + assert "network_path" in mock_recorder.record_ttfb.call_args.kwargs mock_recorder.record_response_body_size.assert_called_once_with( - 4, operation="google.storage.v2.Storage/BidiReadObject" + 4, operation="ReadObject" ) mock_recorder.record_logical_operation.assert_called_once() assert ( mock_recorder.record_logical_operation.call_args.kwargs["client_method"] == "storage.asyncio.AsyncMultiRangeDownloader.download_ranges" ) + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["operation"] + == "ReadObject" + ) + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["rpc_method"] + == "google.storage.v2.Storage/BidiReadObject" + ) # Verify download_ranges with custom metadata marks stream clean before reopen # and records TTFB only once even across multiple response chunks @@ -973,8 +983,137 @@ async def test_mrd_metrics_recording( stream._mark_closing_cleanly.assert_called() mock_recorder.record_ttfb.assert_called_once() mock_recorder.record_response_body_size.assert_called_once_with( - 4, operation="google.storage.v2.Storage/BidiReadObject" + 4, operation="ReadObject" ) await mrd.close() stream._mark_closing_cleanly.assert_called() + + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader.generate_random_56_bit_integer" + ) + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader._AsyncReadObjectStream" + ) + @pytest.mark.asyncio + async def test_mrd_retry_metrics_accounting( + self, mock_cls_async_read_object_stream, mock_random_int, monkeypatch + ): + from google.api_core.retry_async import AsyncRetry + + from google.cloud.storage import _opentelemetry_metrics + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr( + _opentelemetry_metrics, "metrics", mock_metrics, raising=False + ) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + mock_client = mock.MagicMock() + mock_client._metrics_recorder = recorder + mock_client.grpc_client = mock.AsyncMock() + + # 1. MRD.open() retry (attempt 1 fails with 503, attempt 2 succeeds) + failed_open_stream = mock.MagicMock() + failed_open_stream.open = AsyncMock( + side_effect=exceptions.ServiceUnavailable("503 open failed") + ) + failed_open_stream.is_stream_open = False + + open_stream = mock.MagicMock() + open_stream.open = AsyncMock() + open_stream.close = AsyncMock() + open_stream.send = AsyncMock() + open_stream.generation_number = _TEST_GENERATION_NUMBER + open_stream.persisted_size = _TEST_OBJECT_SIZE + open_stream.read_handle = _TEST_READ_HANDLE + open_stream.is_finalized = True + open_stream.full_obj_server_crc32c = None + open_stream.is_stream_open = True + + # 2. MRD.download_ranges() mid-stream retry (attempt 1 Aborted, attempt 2 succeeds) + open_stream.recv = AsyncMock( + side_effect=exceptions.Aborted("Idle stream closed") + ) + recovered_stream = mock.MagicMock() + recovered_stream.open = AsyncMock() + recovered_stream.close = AsyncMock() + recovered_stream.send = AsyncMock() + recovered_stream.generation_number = _TEST_GENERATION_NUMBER + recovered_stream.persisted_size = _TEST_OBJECT_SIZE + recovered_stream.read_handle = _TEST_READ_HANDLE + recovered_stream.is_finalized = True + recovered_stream.full_obj_server_crc32c = None + recovered_stream.is_stream_open = True + recovered_stream.recv = AsyncMock( + side_effect=[ + _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=b"data", crc32c=google_crc32c.value(b"data") + ), + range_end=True, + read_range=_storage_v2.ReadRange( + read_offset=0, read_length=4, read_id=123 + ), + ) + ] + ) + ] + ) + + mock_cls_async_read_object_stream.side_effect = [ + failed_open_stream, + open_stream, + recovered_stream, + ] + mock_random_int.return_value = 123 + + user_on_error = mock.Mock() + fast_retry = AsyncRetry(initial=0.001, maximum=0.001, on_error=user_on_error) + mrd = AsyncMultiRangeDownloader( + mock_client, _TEST_BUCKET_NAME, _TEST_OBJECT_NAME + ) + await mrd.open(retry_policy=fast_retry) + + user_on_error.assert_called_once() + recorder.retries_counter.add.assert_called_once() + _, open_retry_kw = recorder.retries_counter.add.call_args + assert open_retry_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert open_retry_kw["attributes"]["error.type"] == "UNAVAILABLE" + + recorder.retries_counter.add.reset_mock() + buf = BytesIO() + await mrd.download_ranges( + [(0, 4, buf)], + retry_policy=AsyncRetry( + predicate=lambda e: True, initial=0.001, maximum=0.001 + ), + ) + assert buf.getvalue() == b"data" + recorder.retries_counter.add.assert_called_once() + _, dl_retry_kw = recorder.retries_counter.add.call_args + assert dl_retry_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert dl_retry_kw["attributes"]["error.type"] == "ABORTED" + + # 3. MRD.open() failure when metrics are disabled (recorder is None) + disabled_client = mock.MagicMock() + disabled_client._metrics_recorder = None + disabled_client.grpc_client = mock.AsyncMock() + fail_stream = mock.MagicMock() + fail_stream.open = AsyncMock(side_effect=exceptions.NotFound("404")) + mock_cls_async_read_object_stream.side_effect = [fail_stream] + mrd_disabled = AsyncMultiRangeDownloader( + disabled_client, _TEST_BUCKET_NAME, _TEST_OBJECT_NAME + ) + with pytest.raises(exceptions.NotFound): + await mrd_disabled.open() diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py index 393fa7b2b709..3c5687581c1c 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py @@ -537,3 +537,62 @@ async def test_open_unfinalized_object_skips_checksum( assert read_obj_stream.is_finalized is False assert read_obj_stream.full_obj_server_crc32c is None + + +@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc") +@mock.patch( + "google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client" +) +@pytest.mark.asyncio +async def test_open_records_pending_retry_and_close_marks_clean( + mock_client, mock_cls_async_bidi_rpc, monkeypatch +): + from google.cloud.storage import _opentelemetry_metrics + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + socket_like_rpc = AsyncMock() + mock_call = mock.MagicMock(_closing_cleanly=False) + mock_call.peer = lambda: "ipv4:34.126.1.1:443" + socket_like_rpc.call = mock_call + mock_cls_async_bidi_rpc.return_value = socket_like_rpc + socket_like_rpc.open = AsyncMock() + recv_response = mock.MagicMock(spec=_storage_v2.BidiReadObjectResponse) + recv_response.metadata = None + recv_response.read_handle = _TEST_READ_HANDLE + socket_like_rpc.recv = AsyncMock(return_value=recv_response) + + read_obj_stream = _AsyncReadObjectStream( + client=mock_client, + bucket_name=_TEST_BUCKET_NAME, + object_name=_TEST_OBJECT_NAME, + metrics_recorder=recorder, + ) + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.open", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) as op_state: + op_state.mark_attempt_failed("UNAVAILABLE") + await read_obj_stream.open() + + recorder.retries_counter.add.assert_called_once() + _, retry_kw = recorder.retries_counter.add.call_args + assert retry_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert retry_kw["attributes"]["error.type"] == "UNAVAILABLE" + assert retry_kw["attributes"]["network.path"] == "directpath" + + read_obj_stream.requests_done = AsyncMock() + await read_obj_stream.close() + assert mock_call._closing_cleanly is True diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py index 056bff5c4eb4..06b6a7cf63ed 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py @@ -419,10 +419,8 @@ async def test_open_records_write_ttfb_and_close_marks_clean( ) await stream.open() mock_recorder.record_ttfb.assert_called_once() - assert ( - mock_recorder.record_ttfb.call_args.kwargs["operation"] - == "google.storage.v2.Storage/BidiWriteObject" - ) + assert mock_recorder.record_ttfb.call_args.kwargs["operation"] == "WriteObject" + assert "network_path" in mock_recorder.record_ttfb.call_args.kwargs await stream.close() assert mock_call._closing_cleanly is True diff --git a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py index 79a2e76eaee5..f96cb39a7ca5 100644 --- a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py +++ b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py @@ -230,9 +230,14 @@ def test_otel_missing_disables_metrics(monkeypatch): def test_get_common_attributes(): """Verify common attributes conform to GCS OTel specification.""" attrs = _opentelemetry_metrics.get_common_attributes() + assert set(attrs.keys()) == { + "gcp.client.service", + "gcp.client.version", + "gcp.client.artifact", + } + assert "gcp.client.repo" not in attrs assert attrs["gcp.client.service"] == "storage" assert attrs["gcp.client.version"] == __version__ - assert attrs["gcp.client.repo"] == "googleapis/google-cloud-python" assert attrs["gcp.client.artifact"] == "google-cloud-storage" attrs["gcp.client.service"] = "mutated" @@ -316,12 +321,13 @@ def test_histogram_bucket_boundaries(): assert list(latency) == sorted(latency) sizes = _opentelemetry_metrics.SIZE_HISTOGRAM_BOUNDARIES - assert len(sizes) == 45 + assert len(sizes) == 48 assert sizes[0] == 0 assert sizes[1] == 128 * 1024 assert sizes[32] == 4 * 1024 * 1024 - assert sizes[33] == 8 * 1024 * 1024 - assert sizes[-1] == 16 * 1024 * 1024 * 1024 + assert sizes[33] == int(4.25 * 1024 * 1024) + assert sizes[34] == int(4.75 * 1024 * 1024) + assert sizes[-1] == 8593866752 assert list(sizes) == sorted(sizes) @@ -1111,7 +1117,7 @@ def test_metrics_recorder_recording_and_shutdown(monkeypatch): "By", "Application payload bytes delivered to the caller.", ), - "gcp.storage.client.operation.ttfb": ( + "gcp.storage.client.ttfb": ( "s", "Time elapsed from attempt initiation to receipt of the first response byte.", ), @@ -1129,6 +1135,10 @@ def test_metrics_recorder_recording_and_shutdown(monkeypatch): "{attempt}", "Number of physical network transport attempts executed.", ), + "gcp.storage.client.retries": ( + "{retry}", + "Number of transport retries executed after a failed attempt.", + ), "gcp.storage.client.errors": ( "{error}", "Number of physical network transport attempts terminating with a non-OK status.", @@ -1137,7 +1147,10 @@ def test_metrics_recorder_recording_and_shutdown(monkeypatch): # Record within a logical operation context with _opentelemetry_metrics.start_operation( - recorder, "storage.Blob.upload_from_string", "storage.objects.insert" + recorder, + "storage.Blob.upload_from_string", + "WriteObject", + rpc_method="google.storage.v2.Storage/WriteObject", ): recorder.record_http_attempt( duration=0.05, @@ -1153,10 +1166,30 @@ def test_metrics_recorder_recording_and_shutdown(monkeypatch): url_template="/upload/storage/v1/b/{bucket}/o", status_code=200, error_type="OK", + network_path="cloudpath", ) recorder.record_request_body_size(1024) recorder.record_response_body_size(512) - recorder.record_ttfb(0.01) + recorder.record_ttfb(0.01, network_path="cloudpath") + + # Logical operation metrics: duration has rpc.method; operations counter does not + recorder.client_request_duration.record.assert_called_once() + _, req_dur_kwargs = recorder.client_request_duration.record.call_args + assert req_dur_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert ( + req_dur_kwargs["attributes"]["rpc.method"] + == "google.storage.v2.Storage/WriteObject" + ) + assert ( + req_dur_kwargs["attributes"]["gcp.client.method"] + == "storage.Blob.upload_from_string" + ) + assert req_dur_kwargs["attributes"]["error.type"] == "OK" + + recorder.operations_counter.add.assert_called_once() + _, op_cnt_kwargs = recorder.operations_counter.add.call_args + assert op_cnt_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert "rpc.method" not in op_cnt_kwargs["attributes"] assert recorder.http_request_duration.record.call_count == 2 _, http_dur_kwargs = recorder.http_request_duration.record.call_args_list[0] @@ -1173,79 +1206,101 @@ def test_metrics_recorder_recording_and_shutdown(monkeypatch): assert recorder.attempts_counter.add.call_count == 2 _, attempt_kwargs = recorder.attempts_counter.add.call_args_list[0] - assert ( - attempt_kwargs["attributes"]["gcp.storage.operation"] - == "storage.objects.insert" - ) + assert attempt_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" assert attempt_kwargs["attributes"]["http.response.status_code"] == 503 assert attempt_kwargs["attributes"]["error.type"] == "UNAVAILABLE" assert attempt_kwargs["attributes"]["network.path"] == "cloudpath" assert "url.template" not in attempt_kwargs["attributes"] assert "http.request.method" not in attempt_kwargs["attributes"] + assert "gcp.client.method" not in attempt_kwargs["attributes"] - # Only the 503 attempt records an error + # Retry counter incremented once for the second attempt after 503 UNAVAILABLE + recorder.retries_counter.add.assert_called_once() + _, http_retry_kwargs = recorder.retries_counter.add.call_args + assert http_retry_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert http_retry_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert http_retry_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.client.method" not in http_retry_kwargs["attributes"] + + # Only the 503 attempt records an error (no gcp.client.method on errors) recorder.errors_counter.add.assert_called_once() _, err_kwargs = recorder.errors_counter.add.call_args assert err_kwargs["attributes"]["error.type"] == "UNAVAILABLE" - assert err_kwargs["attributes"]["gcp.storage.operation"] == "storage.objects.insert" - assert ( - err_kwargs["attributes"]["gcp.client.method"] - == "storage.Blob.upload_from_string" - ) + assert err_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert err_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.client.method" not in err_kwargs["attributes"] assert "url.template" not in err_kwargs["attributes"] assert "http.response.status_code" not in err_kwargs["attributes"] recorder.request_body_size.record.assert_called_once() recorder.response_body_size.record.assert_called_once() - recorder.operation_ttfb.record.assert_called_once() + recorder.ttfb.record.assert_called_once() + _, ttfb_kwargs = recorder.ttfb.record.call_args + assert ttfb_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert ttfb_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.client.method" not in ttfb_kwargs["attributes"] - # gRPC attempt recording (OK and non-OK) + # gRPC attempt recording (UNAVAILABLE -> OK retry sequence) recorder.errors_counter.add.reset_mock() recorder.attempts_counter.add.reset_mock() - recorder.record_rpc_attempt( - duration=0.02, - operation="google.storage.v2.Storage/BidiReadObject", - status_code="OK", - error_type="OK", - client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", - network_path="directpath", - ) - recorder.record_rpc_attempt( - duration=0.03, - operation="google.storage.v2.Storage/BidiReadObject", - status_code="UNAVAILABLE", - error_type="UNAVAILABLE", - client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", - ) + recorder.retries_counter.add.reset_mock() + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ): + recorder.record_rpc_attempt( + duration=0.03, + operation="ReadObject", + status_code="UNAVAILABLE", + error_type="UNAVAILABLE", + rpc_method="google.storage.v2.Storage/BidiReadObject", + network_path="directpath", + ) + recorder.record_rpc_attempt( + duration=0.02, + operation="ReadObject", + status_code="OK", + error_type="OK", + rpc_method="google.storage.v2.Storage/BidiReadObject", + network_path="directpath", + ) assert recorder.rpc_call_duration.record.call_count == 2 - _, rpc_dur_kwargs = recorder.rpc_call_duration.record.call_args_list[0] + _, rpc_dur_kwargs = recorder.rpc_call_duration.record.call_args_list[1] assert ( - rpc_dur_kwargs["attributes"]["gcp.storage.operation"] + rpc_dur_kwargs["attributes"]["rpc.method"] == "google.storage.v2.Storage/BidiReadObject" ) - assert ( - rpc_dur_kwargs["attributes"]["gcp.client.method"] - == "storage.asyncio.AsyncMultiRangeDownloader.download_ranges" - ) assert rpc_dur_kwargs["attributes"]["rpc.response.status_code"] == "OK" + assert rpc_dur_kwargs["attributes"]["error.type"] == "OK" assert rpc_dur_kwargs["attributes"]["network.path"] == "directpath" + assert "gcp.storage.operation" not in rpc_dur_kwargs["attributes"] + assert "gcp.client.method" not in rpc_dur_kwargs["attributes"] assert recorder.attempts_counter.add.call_count == 2 - _, rpc_attempt_kwargs = recorder.attempts_counter.add.call_args_list[0] - assert ( - rpc_attempt_kwargs["attributes"]["gcp.storage.operation"] - == "google.storage.v2.Storage/BidiReadObject" - ) + _, rpc_attempt_kwargs = recorder.attempts_counter.add.call_args_list[1] + assert rpc_attempt_kwargs["attributes"]["gcp.storage.operation"] == "ReadObject" assert rpc_attempt_kwargs["attributes"]["rpc.response.status_code"] == "OK" assert rpc_attempt_kwargs["attributes"]["error.type"] == "OK" assert rpc_attempt_kwargs["attributes"]["network.path"] == "directpath" assert "gcp.client.method" not in rpc_attempt_kwargs["attributes"] + assert "rpc.method" not in rpc_attempt_kwargs["attributes"] recorder.errors_counter.add.assert_called_once() _, rpc_err_kwargs = recorder.errors_counter.add.call_args + assert rpc_err_kwargs["attributes"]["gcp.storage.operation"] == "ReadObject" assert rpc_err_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert rpc_err_kwargs["attributes"]["network.path"] == "directpath" + assert "gcp.client.method" not in rpc_err_kwargs["attributes"] assert "rpc.response.status_code" not in rpc_err_kwargs["attributes"] + recorder.retries_counter.add.assert_called_once() + _, rpc_retry_kwargs = recorder.retries_counter.add.call_args + assert rpc_retry_kwargs["attributes"]["gcp.storage.operation"] == "ReadObject" + assert rpc_retry_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert rpc_retry_kwargs["attributes"]["network.path"] == "directpath" + # Record outside of start_operation with status_code=None and callable .code() returning None recorder.record_http_attempt( duration=0.01, @@ -1334,9 +1389,205 @@ def test_metrics_recorder_disabled_no_ops(monkeypatch): # All recording methods are safe no-ops when disabled recorder.record_logical_operation(0.1, "m", "op") + recorder.record_retry("ReadObject", "UNAVAILABLE", "cloudpath") recorder.record_http_attempt(0.1, "GET", "/storage/v1/b") recorder.record_rpc_attempt(0.1, "google.storage.v2.Storage/ReadObject") recorder.record_request_body_size(100) recorder.record_response_body_size(100) recorder.record_ttfb(0.01) recorder.shutdown() + + +def test_normalize_logical_operation_and_resolve_rpc_method(): + assert _opentelemetry_metrics.normalize_logical_operation(None) == "UNKNOWN" + assert _opentelemetry_metrics.normalize_logical_operation("") == "UNKNOWN" + assert _opentelemetry_metrics.normalize_logical_operation(123) == "UNKNOWN" # type: ignore[arg-type] + assert _opentelemetry_metrics.normalize_logical_operation("///") == "UNKNOWN" + assert ( + _opentelemetry_metrics.normalize_logical_operation( + "google.storage.v2.Storage/BidiReadObject" + ) + == "ReadObject" + ) + assert ( + _opentelemetry_metrics.normalize_logical_operation( + "/google.storage.v2.Storage/BidiWriteObject" + ) + == "WriteObject" + ) + assert ( + _opentelemetry_metrics.normalize_logical_operation( + "google.storage.v2.Storage/GetObject" + ) + == "GetObject" + ) + assert ( + _opentelemetry_metrics.normalize_logical_operation("GetNotificationConfig") + == "GetNotificationConfig" + ) + + assert _opentelemetry_metrics.resolve_rpc_method(None, None) == "UNKNOWN" + assert _opentelemetry_metrics.resolve_rpc_method("", "") == "UNKNOWN" + assert _opentelemetry_metrics.resolve_rpc_method("///", None) == "UNKNOWN" + assert _opentelemetry_metrics.resolve_rpc_method("UNKNOWN", None) == "UNKNOWN" + assert ( + _opentelemetry_metrics.resolve_rpc_method( + "ReadObject", "google.storage.v2.Storage/BidiReadObject" + ) + == "google.storage.v2.Storage/BidiReadObject" + ) + assert ( + _opentelemetry_metrics.resolve_rpc_method("ReadObject") + == "google.storage.v2.Storage/ReadObject" + ) + assert ( + _opentelemetry_metrics.resolve_rpc_method("GetNotificationConfig") + == "google.storage.v1.Storage/GetNotificationConfig" + ) + assert ( + _opentelemetry_metrics.resolve_rpc_method("storage.objects.get") + == "storage.objects.get" + ) + + +def test_iter_exception_chain_traverses_errors_list(): + wrapped = api_exceptions.Aborted( + "aborted", + errors=[socket.timeout("nested timeout in errors")], + ) + chain = list(_opentelemetry_metrics._iter_exception_chain(wrapped)) + assert any(isinstance(item, socket.timeout) for item in chain) + + class _NonSeqErrorsErr(Exception): + errors = 123 + + assert _opentelemetry_metrics.classify_error_type(_NonSeqErrorsErr()) == "UNKNOWN" + + +def test_operation_state_and_retry_helpers(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + disabled_recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=False) + + # No-ops when recorder is None, disabled, or no active operation + _opentelemetry_metrics.record_pending_retry(None) + _opentelemetry_metrics.record_pending_retry(disabled_recorder) + _opentelemetry_metrics.record_pending_retry(recorder) + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + None, RuntimeError("e"), attempts_before=0, will_retry=True + ) + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + disabled_recorder, RuntimeError("e"), attempts_before=0, will_retry=True + ) + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, RuntimeError("e"), attempts_before=0, will_retry=True + ) + + # Standalone record_retry and record_ttfb outside start_operation without optional attrs + recorder.record_retry(operation=None, error_type="UNAVAILABLE", network_path=None) + _, retry_no_op_kw = recorder.retries_counter.add.call_args + assert "gcp.storage.operation" not in retry_no_op_kw["attributes"] + assert "network.path" not in retry_no_op_kw["attributes"] + + recorder.record_ttfb(0.02, operation=None, network_path=None) + _, ttfb_no_op_kw = recorder.ttfb.record.call_args + assert "gcp.storage.operation" not in ttfb_no_op_kw["attributes"] + assert "network.path" not in ttfb_no_op_kw["attributes"] + + recorder.retries_counter.add.reset_mock() + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.open", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) as op_state: + assert op_state is not None + op_state.mark_attempt_failed("OK") + op_state.mark_attempt_failed("") + assert op_state.last_failure_error_type is None + + # Pending retry when no failure occurred is a no-op + _opentelemetry_metrics.record_pending_retry(recorder) + recorder.retries_counter.add.assert_not_called() + + # When transport already recorded the attempt, handle_unrecorded_retryable_failure skips + op_state.recorded_attempts = 1 + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.ServiceUnavailable("503"), + attempts_before=0, + will_retry=True, + network_path="directpath", + ) + assert op_state.recorded_attempts == 1 + assert op_state.last_failure_error_type is None + + # Unrecorded failure 1 (will_retry=True) -> marks failure, no retry yet + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.ServiceUnavailable("503"), + attempts_before=1, + will_retry=True, + network_path="unknown", + ) + assert op_state.recorded_attempts == 2 + assert op_state.last_failure_error_type == "UNAVAILABLE" + recorder.retries_counter.add.assert_not_called() + + # Consecutive unrecorded failure 2 (will_retry=True) -> emits retry for failure 1, marks failure 2 + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.DeadlineExceeded("timeout"), + attempts_before=2, + will_retry=True, + network_path="cloudpath", + ) + assert op_state.recorded_attempts == 3 + assert op_state.last_failure_error_type == "DEADLINE_EXCEEDED" + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "UNAVAILABLE" + ) + + # Unrecorded terminal failure 3 (will_retry=False) -> emits retry for failure 2, leaves error cleared + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.PermissionDenied("403"), + attempts_before=3, + will_retry=False, + network_path="cloudpath", + ) + assert op_state.recorded_attempts == 4 + assert op_state.last_failure_error_type is None + assert recorder.retries_counter.add.call_count == 2 + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "DEADLINE_EXCEEDED" + ) + + # Mark failure and consume via record_pending_retry + op_state.mark_attempt_failed("ABORTED") + _opentelemetry_metrics.record_pending_retry(recorder, network_path="directpath") + assert recorder.retries_counter.add.call_count == 3 + _, pending_kw = recorder.retries_counter.add.call_args + assert pending_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert pending_kw["attributes"]["error.type"] == "ABORTED" + assert pending_kw["attributes"]["network.path"] == "directpath" + + import contextvars + + copied_ctx = contextvars.copy_context() + + # Once start_operation exits, state.finished is True even in copied contexts + assert op_state.finished is True + assert copied_ctx.run(_opentelemetry_metrics.get_current_operation) is None From e3eac79797918a20292ed0c396d41e07efe66f7d Mon Sep 17 00:00:00 2001 From: Shradha Katyal Date: Mon, 26 Oct 2026 09:57:39 +0000 Subject: [PATCH 7/7] fix(storage): fix Async gRPC TTFB, Bidi stream duration, and broken stream error metrics - Sync core OTel metrics View scoping, URI parsing hardening, and CONNECTION_ERROR consolidation - Record gcp.storage.client.ttfb once per RPC/stream attempt at the gRPC interceptor layer - Preserve physical stream start time across _bind_call_to_operation for healthy Bidi stream completions - Stop marking broken streams as clean in _StreamMultiplexer.reopen_stream and prevent stale teardown double-retries [Generated-by: AI] --- .../cloud/storage/_opentelemetry_metrics.py | 49 +++-- .../storage/asyncio/_stream_multiplexer.py | 3 - .../storage/asyncio/async_grpc_client.py | 72 ++++++- .../asyncio/async_multi_range_downloader.py | 12 -- .../asyncio/async_write_object_stream.py | 7 - .../unit/asyncio/test_async_grpc_client.py | 86 +++++++- .../test_async_multi_range_downloader.py | 188 +++++++++++++++++- .../asyncio/test_async_write_object_stream.py | 32 ++- .../unit/asyncio/test_stream_multiplexer.py | 10 +- .../tests/unit/test__opentelemetry_metrics.py | 30 ++- 10 files changed, 413 insertions(+), 76 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py index f9583d5baba4..f511eff47eea 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py +++ b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py @@ -187,6 +187,8 @@ def is_debug_metrics_enabled( # --------------------------------------------------------------------------- # 5. Standard Common Attributes & Meter Provider # --------------------------------------------------------------------------- +_METER_NAME = "google.cloud.storage" + _COMMON_ATTRIBUTES: Dict[str, Any] = { "gcp.client.service": "storage", "gcp.client.version": __version__, @@ -227,7 +229,7 @@ def get_meter( ): return None return metrics.get_meter( - "google.cloud.storage", + _METER_NAME, __version__, meter_provider=meter_provider, ) @@ -309,6 +311,7 @@ def get_histogram_views() -> List[Any]: views.append( View( instrument_name=name, + meter_name=_METER_NAME, aggregation=ExplicitBucketHistogramAggregation( boundaries=LATENCY_HISTOGRAM_BOUNDARIES ), @@ -318,6 +321,7 @@ def get_histogram_views() -> List[Any]: views.append( View( instrument_name=name, + meter_name=_METER_NAME, aggregation=ExplicitBucketHistogramAggregation( boundaries=SIZE_HISTOGRAM_BOUNDARIES ), @@ -507,10 +511,8 @@ def classify_error_type( {"NameResolutionError"}, ): return "DNS_FAILURE" - if isinstance(item, ConnectionRefusedError): - return "CONNECTION_REFUSED" - if isinstance(item, ConnectionResetError): - return "CONNECTION_RESET" + if isinstance(item, (ConnectionRefusedError, ConnectionResetError)): + return "CONNECTION_ERROR" # 8. Timeouts (Note: urllib3.exceptions.NewConnectionError subclasses ConnectTimeoutError # even for non-timeout connection failures, so exclude NewConnectionError here.) @@ -537,7 +539,7 @@ def classify_error_type( ): return "TIMEOUT" - # 9. Generic Connectivity Failure + # 9. Generic Connection / Network Transport Failure for item in chain: if ( _is_subclass_by_module( @@ -552,7 +554,7 @@ def classify_error_type( ) or isinstance(item, (OSError, http.client.HTTPException)) ): - return "CONNECTIVITY" + return "CONNECTION_ERROR" # 10. Explicit HTTP status_code fallback if valid_status and status_code is not None: @@ -708,7 +710,10 @@ def normalize_url_template(url_or_path: Optional[str]) -> str: if not url_or_path or not isinstance(url_or_path, str): return "UNKNOWN" - parsed = urlsplit(url_or_path) + try: + parsed = urlsplit(url_or_path) + except ValueError: + return "UNKNOWN" path = parsed.path or "" query = parsed.query or "" @@ -993,12 +998,30 @@ def _extract_server_address(api_endpoint: Optional[str]) -> str: endpoint = api_endpoint.strip() if not endpoint: return DEFAULT_SERVER_ADDRESS - if ":///" in endpoint: - endpoint = "https://" + endpoint.split(":///", 1)[1].lstrip("/") - elif "://" not in endpoint: + for grpc_prefix in ( + "dns:", + "google-c2p:", + "google-c2p-experimental:", + "xds:", + "ipv4:", + "ipv6:", + ): + if endpoint.startswith(grpc_prefix): + remainder = endpoint[len(grpc_prefix) :] + if remainder.startswith("//"): + remainder = remainder[2:] + remainder = ( + remainder.split("/", 1)[1] if "/" in remainder else remainder + ) + endpoint = remainder.lstrip("/") + break + if "://" not in endpoint: endpoint = "https://" + endpoint.lstrip("/") - parsed = urlsplit(endpoint) - return parsed.hostname or DEFAULT_SERVER_ADDRESS + try: + parsed = urlsplit(endpoint) + return parsed.hostname or DEFAULT_SERVER_ADDRESS + except ValueError: + return DEFAULT_SERVER_ADDRESS class _MetricsRecorder: diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py index 7f5bb8f3c88b..b9617e1164d9 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py @@ -194,9 +194,6 @@ async def reopen_stream( async with self._reopen_lock: if self._stream_generation != broken_generation: return - mark_clean = getattr(self._stream, "_mark_closing_cleanly", None) - if callable(mark_clean) and not inspect.iscoroutinefunction(mark_clean): - mark_clean() self._stop_recv_loop() if self._recv_task: try: diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py index a9bb0135f40a..56c7ff30cdde 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py @@ -302,7 +302,6 @@ def _bind_call_to_operation( for current in _iter_wrapped_calls(call): if isinstance(current, _TrackedCallBase): current._op_state = op_state - current._start_time = op_state.start_time class _TrackedCallBase: @@ -323,21 +322,54 @@ def __init__( self._rpc_method = rpc_method self._logical_operation = logical_operation self._start_time = start_time + self._initial_op_state = op_state self._op_state = op_state + self._started_after_failure = ( + op_state is not None and op_state.last_failure_error_type is not None + ) self._direct_path_attempted = direct_path_attempted self._received_response = False + self._ttfb_recorded = False self._recorded = False self._closing_cleanly = False def __getattr__(self, name: str) -> Any: return getattr(self._call, name) + def _record_ttfb_once( + self, + duration: Optional[float] = None, + network_path: Optional[str] = None, + ) -> None: + if not self._ttfb_recorded: + self._ttfb_recorded = True + ttfb_duration = ( + max(0.0, time.perf_counter() - self._start_time) + if duration is None + else duration + ) + resolved_path = ( + _resolve_network_path( + call=self._call, + direct_path_attempted=self._direct_path_attempted, + received_server_response=True, + ) + if network_path is None + else network_path + ) + self._recorder.record_ttfb( + duration=ttfb_duration, + operation=self._logical_operation, + network_path=resolved_path, + ) + def _record_once( self, status_code: Optional[str], error_type: str, exc: Optional[BaseException] = None, received_response: bool = False, + record_ttfb: bool = False, ) -> None: if not self._recorded: self._recorded = True @@ -348,11 +380,14 @@ def _record_once( else None ) active_op = current_op or fallback_op - effective_start = ( - current_op.start_time - if (current_op is not None and current_op is not self._op_state) - else self._start_time - ) + if ( + error_type != "OK" + and active_op is not None + and active_op is not self._initial_op_state + ): + effective_start = active_op.start_time + else: + effective_start = self._start_time duration = max(0.0, time.perf_counter() - effective_start) resolved_path = _resolve_network_path( call=self._call, @@ -360,6 +395,19 @@ def _record_once( direct_path_attempted=self._direct_path_attempted, received_server_response=received_response or self._received_response, ) + if record_ttfb: + self._record_ttfb_once( + duration=duration, + network_path=resolved_path, + ) + is_stale_teardown = ( + active_op is not None + and active_op.last_failure_error_type is not None + and not self._started_after_failure + ) + saved_prior_error = ( + active_op.consume_retry_error() if is_stale_teardown else None + ) self._recorder.record_rpc_attempt( duration=duration, operation=self._logical_operation, @@ -369,6 +417,8 @@ def _record_once( network_path=resolved_path, op_state=active_op, ) + if is_stale_teardown: + active_op.last_failure_error_type = saved_prior_error def _record_from_exception(self, exc: BaseException) -> None: if isinstance(exc, asyncio.CancelledError) and self._closing_cleanly: @@ -439,7 +489,12 @@ async def _await_impl(self) -> Any: raise else: self._received_response = True - self._record_once(status_code="OK", error_type="OK", received_response=True) + self._record_once( + status_code="OK", + error_type="OK", + received_response=True, + record_ttfb=True, + ) return response def __await__(self): @@ -454,6 +509,7 @@ async def _aiter_impl(self): try: async for response in self._call: self._received_response = True + self._record_ttfb_once() yield response completed_normally = True except GeneratorExit: @@ -490,6 +546,8 @@ async def read(self) -> Any: self._record_once( status_code="OK", error_type="OK", received_response=True ) + else: + self._record_ttfb_once() return response diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py index 8f17350f33f5..060fcf923435 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py @@ -16,7 +16,6 @@ import asyncio import logging -import time from io import BytesIO from typing import Any, Dict, List, Optional, Tuple @@ -540,8 +539,6 @@ def send_and_recv_via_multiplexer( async def generator(): nonlocal attempt_count, last_broken_generation attempt_count += 1 - attempt_start = time.perf_counter() - ttfb_recorded = False if attempt_count > 1: logger.info( @@ -610,15 +607,6 @@ async def generator(): # Track completion if item.object_data_ranges: - if not ttfb_recorded and recorder is not None: - ttfb_recorded = True - recorder.record_ttfb( - time.perf_counter() - attempt_start, - operation="ReadObject", - network_path=self._current_network_path( - received_server_response=True - ), - ) for data_range in item.object_data_ranges: if data_range.range_end: pending_read_ids.discard( diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py index ee69a598ff6b..882c5df59a29 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py @@ -13,7 +13,6 @@ # limitations under the License. import asyncio -import time from typing import Any, List, Optional, Tuple import grpc @@ -180,7 +179,6 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: metadata=final_metadata, ) - attempt_start = time.perf_counter() try: await self.socket_like_rpc.open() # this is actually 1 send response = await self.socket_like_rpc.recv() @@ -194,11 +192,6 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: operation="WriteObject", network_path=net_path, ) - self._metrics_recorder.record_ttfb( - time.perf_counter() - attempt_start, - operation="WriteObject", - network_path=net_path, - ) self._is_stream_open = True if response.persisted_size: diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py index 4d016095a74f..0b0a72a6f3b0 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py @@ -829,9 +829,12 @@ def __await__(self): assert await tracked_uu == "resp" recorder.record_rpc_attempt.assert_called_once() assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + recorder.record_ttfb.assert_called_once() + assert recorder.record_ttfb.call_args.kwargs["operation"] == "GetObject" # 2. Stream-Unary error and continuation error recorder.record_rpc_attempt.reset_mock() + recorder.record_ttfb.reset_mock() raw_su = _AwaitableCall(exc=_FakeRpcError()) tracked_su = await s_u.intercept_stream_unary( mock.AsyncMock(return_value=raw_su), details, iter([]) @@ -845,6 +848,7 @@ def __await__(self): assert ( recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "UNAVAILABLE" ) + recorder.record_ttfb.assert_not_called() for interceptor, method_name in ( (u_u, "intercept_unary_unary"), @@ -883,20 +887,35 @@ async def read(self): return item recorder.record_rpc_attempt.reset_mock() + recorder.record_ttfb.reset_mock() tracked_us = await u_s.intercept_unary_stream( mock.AsyncMock(return_value=_StreamCall(["a", "b"])), details, "req" ) collected = [x async for x in tracked_us] assert collected == ["a", "b"] assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + recorder.record_ttfb.assert_called_once() recorder.record_rpc_attempt.reset_mock() + recorder.record_ttfb.reset_mock() tracked_ss = await s_s.intercept_stream_stream( - mock.AsyncMock(return_value=_StreamCall(["x"])), details, iter([]) + mock.AsyncMock(return_value=_StreamCall(["x", "y"])), details, iter([]) ) assert await tracked_ss.read() == "x" + assert await tracked_ss.read() == "y" assert await tracked_ss.read() == grpc.aio.EOF assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + recorder.record_ttfb.assert_called_once() + + # Immediate EOF without messages records attempt OK without TTFB + recorder.record_rpc_attempt.reset_mock() + recorder.record_ttfb.reset_mock() + tracked_empty = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=_StreamCall([])), details, iter([]) + ) + assert await tracked_empty.read() == grpc.aio.EOF + recorder.record_rpc_attempt.assert_called_once() + recorder.record_ttfb.assert_not_called() # Clean close vs abnormal cancel (including _interceptors_task, cycle, and non-dict object) recorder.record_rpc_attempt.reset_mock() @@ -1318,6 +1337,7 @@ async def invoke_rpc(): assert res == "object_meta" assert recorder.attempts_counter.add.call_count == 2 + assert recorder.ttfb.record.call_count == 1 assert recorder.errors_counter.add.call_count == 1 assert recorder.retries_counter.add.call_count == 1 assert ( @@ -1381,18 +1401,66 @@ async def invoke_rpc(): recorded_dur = recorder.rpc_call_duration.record.call_args.args[0] assert recorded_dur < 10.0 - # 3. _bind_call_to_operation for background recv loop task without active ContextVar + # 3. _bind_call_to_operation preserves _start_time: + # - background recv failure uses bound op start_time + # - clean stream close records full physical stream lifetime from _start_time + raw_bg_fail = mock.MagicMock() + raw_bg_fail.read = mock.AsyncMock( + side_effect=api_exceptions.Aborted("Background recv failed") + ) tracked_bg = await s_s.intercept_stream_stream( - mock.AsyncMock(return_value=mock.MagicMock()), bidi_details, iter([]) + mock.AsyncMock(return_value=raw_bg_fail), bidi_details, iter([]) ) tracked_bg._start_time -= 100.0 + original_bg_start = tracked_bg._start_time wrapped_bg = types.SimpleNamespace(_call=tracked_bg) - with _opentelemetry_metrics.start_operation( - recorder, + bg_op = _opentelemetry_metrics._OperationState( client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", operation="ReadObject", rpc_method="google.storage.v2.Storage/BidiReadObject", - ) as bg_op: - async_grpc_client._bind_call_to_operation(wrapped_bg, bg_op) - assert tracked_bg._op_state is bg_op - assert tracked_bg._start_time == bg_op.start_time + ) + async_grpc_client._bind_call_to_operation(wrapped_bg, bg_op) + assert tracked_bg._op_state is bg_op + assert tracked_bg._start_time == original_bg_start + + recorder.rpc_call_duration.record.reset_mock() + with pytest.raises(api_exceptions.Aborted): + await tracked_bg.read() + assert recorder.rpc_call_duration.record.call_args.args[0] < 10.0 + + # 4. Healthy Bidi stream closed after _bind_call_to_operation and inside finalize() + # records full physical stream lifetime (>= 100.0s) + raw_healthy = mock.MagicMock() + raw_healthy.cancel = mock.MagicMock(return_value=True) + raw_healthy.read = mock.AsyncMock(return_value=grpc.aio.EOF) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.open", + operation="WriteObject", + rpc_method="google.storage.v2.Storage/BidiWriteObject", + ): + tracked_healthy = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=raw_healthy), + mock.MagicMock(method="/google.storage.v2.Storage/BidiWriteObject"), + iter([]), + ) + tracked_healthy._start_time -= 100.0 + + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.append", + operation="WriteObject", + rpc_method="google.storage.v2.Storage/BidiWriteObject", + ) as append_op: + async_grpc_client._bind_call_to_operation(tracked_healthy, append_op) + + recorder.rpc_call_duration.record.reset_mock() + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.finalize", + operation="WriteObject", + rpc_method="google.storage.v2.Storage/BidiWriteObject", + ): + async_grpc_client._mark_call_closing_cleanly(tracked_healthy) + tracked_healthy.cancel() + assert recorder.rpc_call_duration.record.call_args.args[0] >= 100.0 diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py index 2340f66b5631..411cef7cab45 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py @@ -926,9 +926,6 @@ async def test_mrd_metrics_recording( mock_recorder.record_logical_operation.reset_mock() buf = BytesIO() await mrd.download_ranges([(0, 4, buf)]) - mock_recorder.record_ttfb.assert_called_once() - assert mock_recorder.record_ttfb.call_args.kwargs["operation"] == "ReadObject" - assert "network_path" in mock_recorder.record_ttfb.call_args.kwargs mock_recorder.record_response_body_size.assert_called_once_with( 4, operation="ReadObject" ) @@ -947,7 +944,6 @@ async def test_mrd_metrics_recording( ) # Verify download_ranges with custom metadata marks stream clean before reopen - # and records TTFB only once even across multiple response chunks chunk1 = _storage_v2.BidiReadObjectResponse( object_data_ranges=[ _storage_v2.ObjectRangeData( @@ -975,13 +971,11 @@ async def test_mrd_metrics_recording( ] ) stream.recv = AsyncMock(side_effect=[chunk1, chunk2]) - mock_recorder.record_ttfb.reset_mock() mock_recorder.record_response_body_size.reset_mock() stream._mark_closing_cleanly.reset_mock() buf2 = BytesIO() await mrd.download_ranges([(0, 4, buf2)], metadata=[("x-custom", "val")]) stream._mark_closing_cleanly.assert_called() - mock_recorder.record_ttfb.assert_called_once() mock_recorder.record_response_body_size.assert_called_once_with( 4, operation="ReadObject" ) @@ -1117,3 +1111,185 @@ async def test_mrd_retry_metrics_accounting( ) with pytest.raises(exceptions.NotFound): await mrd_disabled.open() + + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader.generate_random_56_bit_integer" + ) + @pytest.mark.asyncio + async def test_mrd_bidi_stream_interceptor_ttfb_duration_and_reopen_metrics( + self, mock_random_int, monkeypatch + ): + import grpc + from google.api_core.retry_async import AsyncRetry + + from google.cloud.storage import _opentelemetry_metrics + from google.cloud.storage.asyncio import async_grpc_client + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr( + _opentelemetry_metrics, "metrics", mock_metrics, raising=False + ) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + _, _, _, s_s = async_grpc_client._create_metrics_interceptors( + recorder, direct_path_attempted=False + ) + details = mock.MagicMock(method="/google.storage.v2.Storage/BidiReadObject") + + class _FakeBidiCall: + def __init__(self): + self._queue = asyncio.Queue() + self._done = False + self._pump_task = None + + def done(self): + return self._done + + def add_done_callback(self, callback): + pass + + def cancel(self): + self._done = True + self._pump_task.cancel() + self._queue.put_nowait(grpc.aio.EOF) + return True + + async def write(self, request): + if request.read_object_spec.bucket: + self._queue.put_nowait( + _storage_v2.BidiReadObjectResponse( + metadata=_storage_v2.Object( + generation=_TEST_GENERATION_NUMBER, + size=_TEST_OBJECT_SIZE, + ), + read_handle=_storage_v2.BidiReadHandle( + handle=_TEST_READ_HANDLE + ), + ) + ) + if request.read_ranges: + for r in request.read_ranges: + self._queue.put_nowait( + _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=b"data", + crc32c=google_crc32c.value(b"data"), + ), + range_end=True, + read_range=_storage_v2.ReadRange( + read_offset=r.read_offset, + read_length=4, + read_id=r.read_id, + ), + ) + ] + ) + ) + + async def done_writing(self): + self._queue.put_nowait(grpc.aio.EOF) + + async def read(self): + return await self._queue.get() + + created_tracked_calls = [] + raw_calls = [ + _FakeBidiCall(), + _FakeBidiCall(), + _FakeBidiCall(), + ] + + async def start_bidi_rpc(request_iterator=None, metadata=None): + raw = raw_calls.pop(0) + tracked = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=raw), details, request_iterator + ) + + async def _pump_requests(): + async for req in request_iterator: + await tracked.write(req) + await tracked.done_writing() + + raw._pump_task = asyncio.create_task(_pump_requests()) + created_tracked_calls.append(tracked) + return tracked + + mock_client = mock.MagicMock() + mock_client._metrics_recorder = recorder + bidi_key = object() + mock_client.grpc_client._client._transport.bidi_read_object = bidi_key + mock_client.grpc_client._client._transport._wrapped_methods = { + bidi_key: start_bidi_rpc + } + mock_random_int.side_effect = [101, 102, 201, 202] + + # Scenario A: Healthy MRD lifecycle across 2 download_ranges() calls + close() + mrd = await AsyncMultiRangeDownloader.create_mrd( + mock_client, _TEST_BUCKET_NAME, _TEST_OBJECT_NAME + ) + created_tracked_calls[0]._start_time -= 50.0 + assert recorder.ttfb.record.call_count == 1 + + b1, b2 = BytesIO(), BytesIO() + await mrd.download_ranges([(0, 4, b1)]) + await mrd.download_ranges([(0, 4, b2)]) + # Same physical stream: TTFB is still recorded only once + assert recorder.ttfb.record.call_count == 1 + assert b1.getvalue() == b"data" + assert b2.getvalue() == b"data" + + await mrd.close() + assert recorder.attempts_counter.add.call_count == 1 + assert ( + recorder.attempts_counter.add.call_args.kwargs["attributes"]["error.type"] + == "OK" + ) + assert recorder.rpc_call_duration.record.call_args.args[0] >= 50.0 + + # Scenario B: Broken stream during download_ranges() triggers reopen_stream() + recorder.ttfb.record.reset_mock() + recorder.attempts_counter.add.reset_mock() + recorder.retries_counter.add.reset_mock() + recorder.rpc_call_duration.record.reset_mock() + + mrd2 = await AsyncMultiRangeDownloader.create_mrd( + mock_client, _TEST_BUCKET_NAME, _TEST_OBJECT_NAME + ) + assert recorder.ttfb.record.call_count == 1 + mrd2.read_obj_str.send = AsyncMock( + side_effect=exceptions.ServiceUnavailable("send broke") + ) + + b3 = BytesIO() + await mrd2.download_ranges( + [(0, 4, b3)], + retry_policy=AsyncRetry( + predicate=lambda e: True, initial=0.001, maximum=0.001 + ), + ) + assert b3.getvalue() == b"data" + # Stream 1 + reopened Stream 2 => 2 TTFB points and 1 retry + assert recorder.ttfb.record.call_count == 2 + assert recorder.retries_counter.add.call_count == 1 + # Broken stream 1 was NOT falsely marked OK by reopen_stream() + stream1_err = recorder.attempts_counter.add.call_args_list[0].kwargs[ + "attributes" + ]["error.type"] + assert stream1_err == "CANCELLED" + + await mrd2.close() + assert recorder.attempts_counter.add.call_count == 2 + stream2_err = recorder.attempts_counter.add.call_args_list[1].kwargs[ + "attributes" + ]["error.type"] + assert stream2_err == "OK" diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py index 06b6a7cf63ed..120e9fa23ea7 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py @@ -399,20 +399,32 @@ async def test_methods_require_open_raises(self, mock_client): async def test_open_records_write_ttfb_and_close_marks_clean( self, mock_rpc_cls, mock_client ): + from google.cloud.storage.asyncio import async_grpc_client + mock_recorder = MagicMock() mock_recorder.standard_enabled = True - mock_rpc = mock_rpc_cls.return_value - mock_rpc.open = AsyncMock() - mock_rpc.send = AsyncMock() - mock_rpc.recv = AsyncMock( + _, _, _, s_s = async_grpc_client._create_metrics_interceptors( + mock_recorder, direct_path_attempted=False + ) + details = MagicMock(method="/google.storage.v2.Storage/BidiWriteObject") + raw_call = MagicMock() + raw_call.read = AsyncMock( side_effect=[ MagicMock(persisted_size=0, resource=None, write_handle=b"h1"), + MagicMock(persisted_size=4, resource=None, write_handle=b"h1"), grpc.aio.EOF, ] ) + tracked_call = await s_s.intercept_stream_stream( + AsyncMock(return_value=raw_call), details, iter([]) + ) + + mock_rpc = mock_rpc_cls.return_value + mock_rpc.open = AsyncMock() + mock_rpc.send = AsyncMock() + mock_rpc.recv = AsyncMock(side_effect=tracked_call.read) mock_rpc.close = AsyncMock() - mock_call = MagicMock(_closing_cleanly=False) - mock_rpc.call = mock_call + mock_rpc.call = tracked_call stream = _AsyncWriteObjectStream( mock_client, BUCKET, OBJECT, metrics_recorder=mock_recorder @@ -422,8 +434,14 @@ async def test_open_records_write_ttfb_and_close_marks_clean( assert mock_recorder.record_ttfb.call_args.kwargs["operation"] == "WriteObject" assert "network_path" in mock_recorder.record_ttfb.call_args.kwargs + # Subsequent recv on the same physical stream must not emit duplicate TTFB + await stream.recv() + mock_recorder.record_ttfb.assert_called_once() + await stream.close() - assert mock_call._closing_cleanly is True + assert tracked_call._closing_cleanly is True + mock_recorder.record_rpc_attempt.assert_called_once() + assert mock_recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" @mock.patch("google.cloud.storage.asyncio.async_write_object_stream.AsyncBidiRpc") @pytest.mark.asyncio diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py b/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py index 9c75b9e8e1e1..2c54d985535e 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py @@ -13,7 +13,7 @@ # limitations under the License. import asyncio -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import grpc import pytest @@ -399,6 +399,7 @@ class TestReopenStream: async def test_reopen_bumps_generation_and_replaces_stream(self): # Given a multiplexer with a registered queue old_stream = AsyncMock() + old_stream._mark_closing_cleanly = MagicMock() old_stream.recv = AsyncMock(side_effect=asyncio.Event().wait) mux = _StreamMultiplexer(old_stream) mux.register({1}) @@ -410,7 +411,8 @@ async def test_reopen_bumps_generation_and_replaces_stream(self): # When the stream is reopened with the correct generation await mux.reopen_stream(0, factory) - # Then the generation is bumped and the stream is replaced + # Then the generation is bumped, broken stream is not marked clean, and stream is replaced + old_stream._mark_closing_cleanly.assert_not_called() assert mux.stream_generation == 1 assert mux._stream is new_stream factory.assert_called_once() @@ -560,6 +562,7 @@ class TestClose: async def test_close_cancels_recv_loop(self): # Given a multiplexer with an active receive loop mock_stream = AsyncMock() + mock_stream._mark_closing_cleanly = MagicMock() mock_stream.recv = AsyncMock(side_effect=asyncio.Event().wait) mux = _StreamMultiplexer(mock_stream) mux.register({1}) @@ -569,7 +572,8 @@ async def test_close_cancels_recv_loop(self): # When closing the multiplexer await mux.close() - # Then the receive loop task is cancelled + # Then the stream is marked clean and the receive loop task is cancelled + mock_stream._mark_closing_cleanly.assert_called_once() assert recv_task.cancelled() @pytest.mark.asyncio diff --git a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py index f96cb39a7ca5..2e10d433cbe5 100644 --- a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py +++ b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py @@ -312,6 +312,7 @@ def test_get_meter(monkeypatch): def test_histogram_bucket_boundaries(): """Verify custom latency and byte-size bucket boundaries match specification.""" latency = _opentelemetry_metrics.LATENCY_HISTOGRAM_BOUNDARIES + assert len(latency) == 165 assert latency[0] == 0.0 assert latency[1] == 0.002 assert latency[49] == 0.098 @@ -340,9 +341,10 @@ def __init__(self, boundaries): self.boundaries = boundaries class FakeView: - def __init__(self, instrument_name, aggregation): + def __init__(self, instrument_name, aggregation, meter_name=None): self.instrument_name = instrument_name self.aggregation = aggregation + self.meter_name = meter_name fake_view_mod.ExplicitBucketHistogramAggregation = FakeAggregation fake_view_mod.View = FakeView @@ -361,6 +363,7 @@ def __init__(self, instrument_name, aggregation): views = _opentelemetry_metrics.get_histogram_views() assert len(views) == 6 + assert all(v.meter_name == "google.cloud.storage" for v in views) latency_views = [ v for v in views @@ -611,37 +614,37 @@ def code(self): wrapped_refused = requests.exceptions.ConnectionError(refused_err) assert ( _opentelemetry_metrics.classify_error_type(wrapped_refused) - == "CONNECTION_REFUSED" + == "CONNECTION_ERROR" ) reset_err = ConnectionResetError(104, "Connection reset by peer") wrapped_reset = requests.exceptions.ConnectionError(reset_err) assert ( - _opentelemetry_metrics.classify_error_type(wrapped_reset) == "CONNECTION_RESET" + _opentelemetry_metrics.classify_error_type(wrapped_reset) == "CONNECTION_ERROR" ) - # 10. Generic Connectivity + # 10. Generic Connection Errors assert ( _opentelemetry_metrics.classify_error_type( requests.exceptions.ConnectionError("connection aborted") ) - == "CONNECTIVITY" + == "CONNECTION_ERROR" ) assert ( _opentelemetry_metrics.classify_error_type( requests.exceptions.ChunkedEncodingError("incomplete read") ) - == "CONNECTIVITY" + == "CONNECTION_ERROR" ) assert ( _opentelemetry_metrics.classify_error_type(OSError("network unreachable")) - == "CONNECTIVITY" + == "CONNECTION_ERROR" ) assert ( _opentelemetry_metrics.classify_error_type( http.client.IncompleteRead(b"partial", 10) ) - == "CONNECTIVITY" + == "CONNECTION_ERROR" ) # 11. Explicit status_code fallback when exception is generic @@ -875,6 +878,7 @@ def code(self): ("https://storage.googleapis.com/storage/v2", "UNKNOWN"), ("https://storage.googleapis.com/storage/v2/other?uploadId=xyz", "UNKNOWN"), ("https://storage.googleapis.com/custom/unrecognized/path/here", "UNKNOWN"), + ("https://[::1/storage/v1/b", "UNKNOWN"), ], ) def test_normalize_url_template(raw_url, expected): @@ -1061,6 +1065,14 @@ async def run_all(): ("https://[::1]:8080", "::1"), ("http://localhost:9023/storage/v1", "localhost"), ("https://", "storage.googleapis.com"), + ("https://[::1", "storage.googleapis.com"), + ("[::1", "storage.googleapis.com"), + ("dns://8.8.8.8/custom.endpoint.com:443", "custom.endpoint.com"), + ("dns:custom.endpoint.com:443", "custom.endpoint.com"), + ("dns://custom.endpoint.com:443", "custom.endpoint.com"), + ("dns://8.8.8.8/", "storage.googleapis.com"), + ("ipv4:127.0.0.1:443", "127.0.0.1"), + ("ipv6:[::1]:443", "::1"), ], ) def test_extract_server_address(endpoint, expected): @@ -1340,7 +1352,7 @@ class _NonTupleArgsErr(Exception): _opentelemetry_metrics.classify_error_type( urllib3.exceptions.NewConnectionError(None, "conn err") ) - == "CONNECTIVITY" + == "CONNECTION_ERROR" ) # Shutdown flushes custom meter_provider and swallows force_flush errors gracefully