diff --git a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py index 8592cc24c65d..f511eff47eea 100644 --- a/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py +++ b/packages/google-cloud-storage/google/cloud/storage/_opentelemetry_metrics.py @@ -14,9 +14,31 @@ """Manages OpenTelemetry metrics instruments and gating for GCS client.""" +import asyncio +import concurrent.futures +import contextlib +import http.client import logging import os -from typing import Any, Dict, Optional +import re +import socket +import ssl +import time +from collections import deque +from contextvars import ContextVar +from dataclasses import dataclass, field +from typing import ( + Any, + Deque, + Dict, + FrozenSet, + Iterator, + List, + Optional, + Set, + Tuple, +) +from urllib.parse import urlsplit from google.cloud.storage.version import __version__ @@ -38,14 +60,43 @@ _DEFAULT_ENABLE_METRICS = False _DEFAULT_ENABLE_DEBUG_METRICS = False +DEFAULT_SERVER_ADDRESS = "storage.googleapis.com" + +# --------------------------------------------------------------------------- +# 3. Standard Metric Instrument Names +# --------------------------------------------------------------------------- +METRIC_RPC_CLIENT_CALL_DURATION = "rpc.client.call.duration" +METRIC_HTTP_CLIENT_REQUEST_DURATION = "http.client.request.duration" +METRIC_GCP_CLIENT_REQUEST_DURATION = "gcp.client.request.duration" +METRIC_STORAGE_CLIENT_OPERATIONS = "gcp.storage.client.operations" +METRIC_STORAGE_CLIENT_ATTEMPTS = "gcp.storage.client.attempts" +METRIC_STORAGE_CLIENT_RETRIES = "gcp.storage.client.retries" +METRIC_STORAGE_REQUEST_BODY_SIZE = "gcp.storage.client.request.body.size" +METRIC_STORAGE_RESPONSE_BODY_SIZE = "gcp.storage.client.response.body.size" +METRIC_STORAGE_CLIENT_TTFB = "gcp.storage.client.ttfb" +METRIC_STORAGE_CLIENT_ERRORS = "gcp.storage.client.errors" + +_LATENCY_METRIC_NAMES = ( + METRIC_RPC_CLIENT_CALL_DURATION, + METRIC_HTTP_CLIENT_REQUEST_DURATION, + METRIC_GCP_CLIENT_REQUEST_DURATION, + METRIC_STORAGE_CLIENT_TTFB, +) + +_SIZE_METRIC_NAMES = ( + METRIC_STORAGE_REQUEST_BODY_SIZE, + METRIC_STORAGE_RESPONSE_BODY_SIZE, +) + # --------------------------------------------------------------------------- -# 3. Optional OpenTelemetry Dependency Check +# 4. Optional OpenTelemetry Dependency Check # --------------------------------------------------------------------------- try: from opentelemetry import metrics HAS_OPENTELEMETRY_METRICS = True except ImportError: + metrics = None # type: ignore[assignment] HAS_OPENTELEMETRY_METRICS = False logger.debug( "OpenTelemetry metrics package (opentelemetry-api >= 1.12.0) is not " @@ -134,12 +185,13 @@ def is_debug_metrics_enabled( # --------------------------------------------------------------------------- -# 4. Standard Common Attributes & Meter Provider +# 5. Standard Common Attributes & Meter Provider # --------------------------------------------------------------------------- +_METER_NAME = "google.cloud.storage" + _COMMON_ATTRIBUTES: Dict[str, Any] = { "gcp.client.service": "storage", "gcp.client.version": __version__, - "gcp.client.repo": "googleapis/google-cloud-python", "gcp.client.artifact": "google-cloud-storage", } @@ -168,13 +220,1192 @@ def get_meter( Optional[Any]: The OpenTelemetry Meter instance if standard or debug metrics are enabled, or None otherwise. """ - if not ( - is_metrics_enabled(enable_metrics) - or is_debug_metrics_enabled(enable_debug_metrics) + if ( + not ( + is_metrics_enabled(enable_metrics) + or is_debug_metrics_enabled(enable_debug_metrics) + ) + or metrics is None ): return None return metrics.get_meter( - "google.cloud.storage", + _METER_NAME, __version__, meter_provider=meter_provider, ) + + +# --------------------------------------------------------------------------- +# 6. Custom Histogram Bucket Boundaries & Views +# --------------------------------------------------------------------------- +def _build_latency_buckets() -> Tuple[float, ...]: + """Builds the custom latency histogram bucket boundaries in seconds. + + Uses 2ms resolution for the first 50 buckets (0.000s to 0.098s), then starts + at 0.100s with 10ms resolution and doubles the increment every 10 buckets + while boundary < 300.0s. + """ + boundaries: List[float] = [] + boundary = 0.0 + increment = 0.002 + + for _ in range(50): + boundaries.append(round(boundary, 6)) + boundary += increment + + increment = 0.010 + i = 0 + while boundary < 300.0: + if i != 0 and i % 10 == 0: + increment *= 2 + boundaries.append(round(boundary, 6)) + boundary += increment + i += 1 + + return tuple(boundaries) + + +def _build_size_buckets() -> Tuple[int, ...]: + """Builds the custom byte-size histogram bucket boundaries in bytes. + + Uses 128 KiB increments up to 4 MiB, then doubles the increment on each + subsequent bucket while ``boundary <= 16 GiB`` (48 total buckets, matching + Go and Java SDKs). + """ + kb = 1024 + mb = 1024 * kb + gb = 1024 * mb + + boundaries: List[int] = [] + boundary = 0 + increment = 128 * kb + + while boundary <= 16 * gb: + boundaries.append(boundary) + boundary += increment + if boundary >= 4 * mb: + increment *= 2 + + return tuple(boundaries) + + +LATENCY_HISTOGRAM_BOUNDARIES: Tuple[float, ...] = _build_latency_buckets() +SIZE_HISTOGRAM_BOUNDARIES: Tuple[int, ...] = _build_size_buckets() + + +def get_histogram_views() -> List[Any]: + """Returns OpenTelemetry SDK Views for custom latency and size histograms. + + If ``opentelemetry-sdk`` is not installed, returns an empty list. + """ + try: + from opentelemetry.sdk.metrics.view import ( + ExplicitBucketHistogramAggregation, + View, + ) + except ImportError: + return [] + + views: List[Any] = [] + for name in _LATENCY_METRIC_NAMES: + views.append( + View( + instrument_name=name, + meter_name=_METER_NAME, + aggregation=ExplicitBucketHistogramAggregation( + boundaries=LATENCY_HISTOGRAM_BOUNDARIES + ), + ) + ) + for name in _SIZE_METRIC_NAMES: + views.append( + View( + instrument_name=name, + meter_name=_METER_NAME, + aggregation=ExplicitBucketHistogramAggregation( + boundaries=SIZE_HISTOGRAM_BOUNDARIES + ), + ) + ) + return views + + +# --------------------------------------------------------------------------- +# 7. Canonical Error Classification (`error.type`) +# --------------------------------------------------------------------------- +_HTTP_STATUS_TO_ERROR_TYPE: Dict[int, str] = { + 400: "INVALID_ARGUMENT", + 401: "UNAUTHENTICATED", + 403: "PERMISSION_DENIED", + 404: "NOT_FOUND", + 408: "DEADLINE_EXCEEDED", + 409: "ABORTED", + 412: "FAILED_PRECONDITION", + 416: "OUT_OF_RANGE", + 429: "RESOURCE_EXHAUSTED", + 499: "CANCELLED", + 500: "INTERNAL", + 501: "UNIMPLEMENTED", + 503: "UNAVAILABLE", + 504: "DEADLINE_EXCEEDED", +} + + +def _is_valid_status_code(val: Any) -> bool: + """Returns True if ``val`` is a valid HTTP error status code (int in 400..599).""" + return isinstance(val, int) and not isinstance(val, bool) and 400 <= val < 600 + + +def _iter_exception_chain(exc: BaseException) -> Iterator[BaseException]: + """Yields ``exc`` and any nested causes, contexts, reasons, or exception args.""" + visited: Set[int] = {id(exc)} + queue: Deque[BaseException] = deque([exc]) + while queue: + current = queue.popleft() + yield current + + for attr_name in ( + "__cause__", + "__context__", + "cause", + "reason", + "original_exception", + ): + nested = getattr(current, attr_name, None) + if isinstance(nested, BaseException) and id(nested) not in visited: + visited.add(id(nested)) + queue.append(nested) + + for seq_attr in ("args", "errors"): + seq = getattr(current, seq_attr, ()) + if isinstance(seq, (tuple, list)): + for item in seq: + if isinstance(item, BaseException) and id(item) not in visited: + visited.add(id(item)) + queue.append(item) + + +def _is_subclass_by_module( + cls: type, module_prefix: str, class_names: Set[str] +) -> bool: + """Checks whether ``cls`` or any of its base classes match module/name criteria.""" + mro = getattr(cls, "__mro__", None) + if isinstance(mro, tuple): + for base in mro: + mod = getattr(base, "__module__", "") or "" + if mod == module_prefix or mod.startswith(module_prefix + "."): + if getattr(base, "__name__", None) in class_names: + return True + return False + + +def classify_error_type( + exc: Optional[BaseException] = None, + status_code: Optional[int] = None, +) -> str: + """Maps an exception or HTTP status code to a low-cardinality ``error.type``. + + Args: + exc: Optional exception raised during an operation or transport attempt. + status_code: Optional HTTP status code from the response. + + Returns: + str: Canonical low-cardinality error classification string. + """ + valid_status = _is_valid_status_code(status_code) + if exc is None: + if not valid_status or status_code is None: + return "OK" + return _HTTP_STATUS_TO_ERROR_TYPE.get(status_code, str(status_code)) + + chain = list(_iter_exception_chain(exc)) + + # 1. Checksum Mismatch (DataCorruption) + for item in chain: + if _is_subclass_by_module( + item.__class__, "google.cloud.storage", {"DataCorruption"} + ) or _is_subclass_by_module( + item.__class__, "google.resumable_media", {"DataCorruption"} + ): + return "CHECKSUM_MISMATCH" + + # 2. Authentication / Credentials Failure (google.auth.exceptions) + for item in chain: + if _is_subclass_by_module( + item.__class__, + "google.auth.exceptions", + { + "RefreshError", + "TransportError", + "DefaultCredentialsError", + "GoogleAuthError", + }, + ): + return "AUTHENTICATION_ERROR" + + # 3. Cancellation + for item in chain: + if isinstance( + item, (asyncio.CancelledError, concurrent.futures.CancelledError) + ): + return "CANCELLED" + + # 4. HTTP response status code on exception (checked before grpc_status_code so + # HTTP 409 Conflict consistently maps to ABORTED across HTTP and Veneer metrics) + for item in chain: + resp = getattr(item, "response", None) + resp_status = getattr(resp, "status_code", None) + if _is_valid_status_code(resp_status): + return _HTTP_STATUS_TO_ERROR_TYPE.get(resp_status, str(resp_status)) + + if item.__class__.__name__ == "Conflict" and _is_subclass_by_module( + item.__class__, "google.api_core.exceptions", {"Conflict"} + ): + return _HTTP_STATUS_TO_ERROR_TYPE[409] + + # 5. gRPC / GoogleAPICallError status code + for item in chain: + grpc_code = getattr(item, "grpc_status_code", None) + if grpc_code is not None: + name = getattr(grpc_code, "name", None) + if name: + return str(name) + return str(grpc_code) + + # Direct grpc.RpcError (has callable .code()) + code_attr = getattr(item, "code", None) + if callable(code_attr): + try: + rpc_status = code_attr() + name = getattr(rpc_status, "name", None) + if isinstance(name, str) and name: + return name + except Exception: # pragma: NO COVER + pass + + if _is_valid_status_code(code_attr): + return _HTTP_STATUS_TO_ERROR_TYPE.get(code_attr, str(code_attr)) + + # 6. TLS / SSL Failure + for item in chain: + if ( + isinstance(item, ssl.SSLError) + or _is_subclass_by_module( + item.__class__, + "requests.exceptions", + {"SSLError"}, + ) + or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"SSLError"}, + ) + ): + return "TLS_FAILURE" + + # 7. Specific Socket / Network Failures (DNS, Connection Refused, Connection Reset) + for item in chain: + if isinstance(item, socket.gaierror) or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"NameResolutionError"}, + ): + return "DNS_FAILURE" + if isinstance(item, (ConnectionRefusedError, ConnectionResetError)): + return "CONNECTION_ERROR" + + # 8. Timeouts (Note: urllib3.exceptions.NewConnectionError subclasses ConnectTimeoutError + # even for non-timeout connection failures, so exclude NewConnectionError here.) + for item in chain: + if item.__class__.__name__ != "NewConnectionError": + if isinstance( + item, + ( + socket.timeout, + TimeoutError, + asyncio.TimeoutError, + concurrent.futures.TimeoutError, + ), + ): + return "TIMEOUT" + if _is_subclass_by_module( + item.__class__, + "requests.exceptions", + {"Timeout", "ConnectTimeout", "ReadTimeout"}, + ) or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"TimeoutError", "ReadTimeoutError", "ConnectTimeoutError"}, + ): + return "TIMEOUT" + + # 9. Generic Connection / Network Transport Failure + for item in chain: + if ( + _is_subclass_by_module( + item.__class__, + "requests.exceptions", + {"ConnectionError", "ChunkedEncodingError", "RequestException"}, + ) + or _is_subclass_by_module( + item.__class__, + "urllib3.exceptions", + {"HTTPError"}, + ) + or isinstance(item, (OSError, http.client.HTTPException)) + ): + return "CONNECTION_ERROR" + + # 10. Explicit HTTP status_code fallback + if valid_status and status_code is not None: + return _HTTP_STATUS_TO_ERROR_TYPE.get(status_code, str(status_code)) + + # 11. Client Validation Errors + for item in chain: + if ( + isinstance(item, (ValueError, TypeError)) + or _is_subclass_by_module( + item.__class__, "google.cloud.storage", {"InvalidResponse"} + ) + or _is_subclass_by_module( + item.__class__, "google.resumable_media", {"InvalidResponse"} + ) + ): + return "CLIENT_VALIDATION_ERROR" + + return "UNKNOWN" + + +# --------------------------------------------------------------------------- +# 8. HTTP `url.template` Route Normalization +# --------------------------------------------------------------------------- +_URL_TEMPLATE_PATTERNS: Tuple[Tuple[re.Pattern[str], str], ...] = ( + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/rewriteTo/b/[^/]+/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/copyTo/b/[^/]+/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}/copyTo/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/moveTo/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/compose$"), + "/storage/v1/b/{bucket}/o/{object}/compose", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/restore$"), + "/storage/v1/b/{bucket}/o/{object}/restore", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/acl/[^/]+$"), + "/storage/v1/b/{bucket}/o/{object}/acl/{entity}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/acl$"), + "/storage/v1/b/{bucket}/o/{object}/acl", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/[^/]+/iam(?:/.*)?$"), + "/storage/v1/b/{bucket}/o/{object}/iam", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o/.+$"), + "/storage/v1/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/o$"), + "/storage/v1/b/{bucket}/o", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/acl/[^/]+$"), + "/storage/v1/b/{bucket}/acl/{entity}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/acl$"), + "/storage/v1/b/{bucket}/acl", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/defaultObjectAcl/[^/]+$"), + "/storage/v1/b/{bucket}/defaultObjectAcl/{entity}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/defaultObjectAcl$"), + "/storage/v1/b/{bucket}/defaultObjectAcl", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/iam(?:/.*)?$"), + "/storage/v1/b/{bucket}/iam", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/notificationConfigs/[^/]+$"), + "/storage/v1/b/{bucket}/notificationConfigs/{notification}", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/notificationConfigs$"), + "/storage/v1/b/{bucket}/notificationConfigs", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/lockRetentionPolicy$"), + "/storage/v1/b/{bucket}/lockRetentionPolicy", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+/restore$"), + "/storage/v1/b/{bucket}/restore", + ), + ( + re.compile(r"^/storage/v1/b/[^/]+$"), + "/storage/v1/b/{bucket}", + ), + ( + re.compile(r"^/storage/v1/b$"), + "/storage/v1/b", + ), + ( + re.compile(r"^/storage/v1/projects/[^/]+/serviceAccount$"), + "/storage/v1/projects/{project}/serviceAccount", + ), + ( + re.compile(r"^/storage/v1/projects/[^/]+/hmacKeys/[^/]+$"), + "/storage/v1/projects/{project}/hmacKeys/{access_id}", + ), + ( + re.compile(r"^/storage/v1/projects/[^/]+/hmacKeys$"), + "/storage/v1/projects/{project}/hmacKeys", + ), + ( + re.compile(r"^/upload/storage/v1/b/[^/]+/o(?:/[^/]+)?$"), + "/upload/storage/v1/b/{bucket}/o", + ), + ( + re.compile(r"^/download/storage/v1/b/[^/]+/o/.+$"), + "/download/storage/v1/b/{bucket}/o/{object}", + ), + ( + re.compile(r"^/batch/storage/v1$"), + "/batch/storage/v1", + ), +) + +_XML_MPU_RESERVED_PREFIXES = {"storage", "upload", "download", "batch", "b", "projects"} +_XML_MPU_QUERY_KEYS = {"uploads", "uploadId", "partNumber"} +_XML_MULTI_SEGMENT_RE = re.compile(r"^/([^/]+)/(.+)$") + + +def normalize_url_template(url_or_path: Optional[str]) -> str: + """Normalizes a raw HTTP URL or path into a bounded ``url.template`` string. + + Bucket names, object keys, project IDs, and query strings are stripped to + prevent cardinality explosion. Unrecognized paths fall back to ``"UNKNOWN"``. + + Args: + url_or_path: Full request URL or path string. + + Returns: + str: Low-cardinality ``url.template`` value. + """ + if not url_or_path or not isinstance(url_or_path, str): + return "UNKNOWN" + + try: + parsed = urlsplit(url_or_path) + except ValueError: + return "UNKNOWN" + path = parsed.path or "" + query = parsed.query or "" + + path = "/" + path.strip("/") + if path == "/": + return "UNKNOWN" + + # Normalize relative paths passed to Connection.api_request before /storage/v1 prefixing + if path == "/b" or path.startswith(("/b/", "/projects/")): + path = "/storage/v1" + path + + for pattern, template in _URL_TEMPLATE_PATTERNS: + if pattern.match(path): + return template + + # XML API Multipart Upload routes: /{bucket}/{object} with XML MPU query params + if query: + query_keys = {param.partition("=")[0] for param in query.split("&") if param} + if query_keys & _XML_MPU_QUERY_KEYS: + match_multi = _XML_MULTI_SEGMENT_RE.match(path) + if match_multi and match_multi.group(1) not in _XML_MPU_RESERVED_PREFIXES: + return "/{bucket}/{object}" + + return "UNKNOWN" + + +# --------------------------------------------------------------------------- +# 9. Logical Operation Normalization, Context & Suppression (`ContextVar`) +# --------------------------------------------------------------------------- +_GRPC_LOGICAL_OPERATION_OVERRIDES: Dict[str, str] = { + "BidiReadObject": "ReadObject", + "BidiWriteObject": "WriteObject", +} + +_RPC_V2_METHODS: FrozenSet[str] = frozenset( + { + "BidiReadObject", + "BidiWriteObject", + "CancelResumableWrite", + "ComposeObject", + "CreateBucket", + "DeleteBucket", + "DeleteObject", + "GetBucket", + "GetIamPolicy", + "GetObject", + "ListBuckets", + "ListObjects", + "LockBucketRetentionPolicy", + "MoveObject", + "QueryWriteStatus", + "ReadObject", + "RestoreObject", + "RewriteObject", + "SetIamPolicy", + "StartResumableWrite", + "TestIamPermissions", + "UpdateBucket", + "UpdateObject", + "WriteObject", + } +) + + +def normalize_logical_operation(operation: Optional[str]) -> str: + """Normalizes an RPC path or method name into a short logical ``gcp.storage.operation``.""" + if not operation or not isinstance(operation, str): + return "UNKNOWN" + cleaned = operation.strip().strip("/") + if not cleaned: + return "UNKNOWN" + short_name = cleaned.rsplit("/", 1)[-1] if "/" in cleaned else cleaned + return _GRPC_LOGICAL_OPERATION_OVERRIDES.get(short_name, short_name) + + +def resolve_rpc_method( + operation: Optional[str], rpc_method: Optional[str] = None +) -> str: + """Resolves the full RPC path (e.g. ``google.storage.v2.Storage/ReadObject``) for ``rpc.method``.""" + raw = rpc_method if (rpc_method and isinstance(rpc_method, str)) else operation + if not raw or not isinstance(raw, str): + return "UNKNOWN" + cleaned = raw.strip().strip("/") + if not cleaned or cleaned == "UNKNOWN": + return "UNKNOWN" + if "/" in cleaned: + return cleaned + if cleaned in _RPC_V2_METHODS: + return f"google.storage.v2.Storage/{cleaned}" + if "." in cleaned: + return cleaned + return f"google.storage.v1.Storage/{cleaned}" + + +@dataclass +class _OperationState: + """Represents the active logical GCS Veneer operation in the current context.""" + + client_method: str + operation: str + rpc_method: str + suppressed: bool = False + start_time: float = field(default_factory=time.perf_counter) + finished: bool = False + last_failure_error_type: Optional[str] = None + recorded_attempts: int = 0 + + def mark_attempt_failed(self, error_type: str) -> None: + """Records the error type of a failed attempt for a subsequent retry.""" + if error_type and error_type != "OK": + self.last_failure_error_type = error_type + + def consume_retry_error(self) -> Optional[str]: + """Consumes and returns the prior failed attempt's error type, if any.""" + prior = self.last_failure_error_type + self.last_failure_error_type = None + return prior + + +_OP_STATE: ContextVar[Optional[_OperationState]] = ContextVar( + "_gcs_otel_op_state", default=None +) + + +def get_current_operation() -> Optional[_OperationState]: + """Returns the current logical `_OperationState` in this execution context, if any.""" + op = _OP_STATE.get() + if op is not None and op.finished: + return None + return op + + +@contextlib.contextmanager +def start_operation( + recorder: Optional["_MetricsRecorder"], + client_method: str, + operation: str, + suppress: bool = False, + rpc_method: Optional[str] = None, +) -> Iterator[Optional[_OperationState]]: + """Context manager tracking a logical user-facing GCS operation. + + If a parent logical operation is already active in the current context, + nested calls become a no-op so only the outermost user-invoked method + emits ``gcp.client.request.duration`` and ``gcp.storage.client.operations``. + + Args: + recorder: Optional `_MetricsRecorder` configured on the client. + client_method: Fully qualified Python method name + (e.g. ``"storage.Blob.download_to_filename"``). + operation: Short logical GCS operation name + (e.g. ``"ReadObject"``, ``"GetObject"``). + suppress: If True, suppresses logical operation and payload size + emission for this context and any nested calls (used by + ``transfer_manager`` workers and ``BlobReader`` prefetches). + rpc_method: Optional full RPC path + (e.g. ``"google.storage.v2.Storage/BidiReadObject"``). If omitted, + inferred from ``operation``. + """ + existing = get_current_operation() + if existing is not None: + if suppress and not existing.suppressed: + suppressed_child = _OperationState( + client_method=existing.client_method, + operation=existing.operation, + rpc_method=existing.rpc_method, + suppressed=True, + start_time=existing.start_time, + ) + child_token = _OP_STATE.set(suppressed_child) + try: + yield suppressed_child + finally: + suppressed_child.finished = True + _OP_STATE.reset(child_token) + else: + yield existing + return + + recorder_active = ( + recorder is not None and getattr(recorder, "standard_enabled", False) is True + ) + if not recorder_active and not suppress: + yield None + return + + resolved_operation = normalize_logical_operation(operation) + resolved_rpc_method = resolve_rpc_method( + resolved_operation, rpc_method=rpc_method or operation + ) + start_time = time.perf_counter() + state = _OperationState( + client_method=client_method, + operation=resolved_operation, + rpc_method=resolved_rpc_method, + suppressed=suppress, + start_time=start_time, + ) + token = _OP_STATE.set(state) + error_type = "OK" + try: + yield state + except BaseException as exc: + error_type = classify_error_type(exc) + raise + finally: + state.finished = True + duration = max(0.0, time.perf_counter() - start_time) + _OP_STATE.reset(token) + if recorder_active and not suppress and recorder is not None: + recorder.record_logical_operation( + duration=duration, + client_method=client_method, + operation=resolved_operation, + error_type=error_type, + rpc_method=resolved_rpc_method, + ) + + +def record_pending_retry( + recorder: Optional["_MetricsRecorder"], + operation: Optional[str] = None, + network_path: Optional[str] = None, +) -> None: + """Emits a pending ``gcp.storage.client.retries`` increment if a prior attempt failed.""" + if recorder is None or not getattr(recorder, "standard_enabled", False): + return + current_op = get_current_operation() + if current_op is None: + return + prior_error = current_op.consume_retry_error() + if prior_error is not None: + recorder.record_retry( + operation=operation or current_op.operation, + error_type=prior_error, + network_path=network_path, + ) + + +def handle_unrecorded_retryable_failure( + recorder: Optional["_MetricsRecorder"], + exc: BaseException, + attempts_before: int, + will_retry: bool, + network_path: Optional[str] = None, +) -> None: + """Tracks retry state when a failed attempt bypassed transport-level attempt recording.""" + if recorder is None or not getattr(recorder, "standard_enabled", False): + return + current_op = get_current_operation() + if current_op is None: + return + if current_op.recorded_attempts > attempts_before: + return + current_op.recorded_attempts += 1 + prior_error = current_op.consume_retry_error() + if prior_error is not None: + recorder.record_retry( + operation=current_op.operation, + error_type=prior_error, + network_path=network_path, + ) + if will_retry: + current_op.mark_attempt_failed(classify_error_type(exc)) + + +def get_active_recorder(obj: Any) -> Optional["_MetricsRecorder"]: + """Returns ``obj._metrics_recorder`` if it is an active ``_MetricsRecorder``.""" + recorder = getattr(obj, "_metrics_recorder", None) + if recorder is not None and getattr(recorder, "standard_enabled", False) is True: + return recorder + return None + + +# --------------------------------------------------------------------------- +# 10. Server Address Parsing & `_MetricsRecorder` +# --------------------------------------------------------------------------- +def _extract_server_address(api_endpoint: Optional[str]) -> str: + """Extracts the low-cardinality hostname from ``api_endpoint``.""" + if not api_endpoint or not isinstance(api_endpoint, str): + return DEFAULT_SERVER_ADDRESS + endpoint = api_endpoint.strip() + if not endpoint: + return DEFAULT_SERVER_ADDRESS + for grpc_prefix in ( + "dns:", + "google-c2p:", + "google-c2p-experimental:", + "xds:", + "ipv4:", + "ipv6:", + ): + if endpoint.startswith(grpc_prefix): + remainder = endpoint[len(grpc_prefix) :] + if remainder.startswith("//"): + remainder = remainder[2:] + remainder = ( + remainder.split("/", 1)[1] if "/" in remainder else remainder + ) + endpoint = remainder.lstrip("/") + break + if "://" not in endpoint: + endpoint = "https://" + endpoint.lstrip("/") + try: + parsed = urlsplit(endpoint) + return parsed.hostname or DEFAULT_SERVER_ADDRESS + except ValueError: + return DEFAULT_SERVER_ADDRESS + + +class _MetricsRecorder: + """Holds OpenTelemetry metric instruments and records standard GCS metrics.""" + + def __init__( + self, + enable_metrics: Optional[bool] = None, + enable_debug_metrics: Optional[bool] = None, + meter_provider: Optional[Any] = None, + api_endpoint: Optional[str] = None, + rpc_system: str = "http", + ) -> None: + self.standard_enabled: bool = is_metrics_enabled(enable_metrics) + self.debug_enabled: bool = is_debug_metrics_enabled(enable_debug_metrics) + self.enabled: bool = self.standard_enabled or self.debug_enabled + self._meter_provider: Optional[Any] = meter_provider + self.rpc_system: str = rpc_system + self.server_address: str = _extract_server_address(api_endpoint) + self.meter: Optional[Any] = get_meter( + enable_metrics=enable_metrics, + enable_debug_metrics=enable_debug_metrics, + meter_provider=meter_provider, + ) + + self._base_attributes: Dict[str, Any] = { + **_COMMON_ATTRIBUTES, + "server.address": self.server_address, + "rpc.system.name": self.rpc_system, + } + + if self.standard_enabled and self.meter is not None: + self.rpc_call_duration = self.meter.create_histogram( + METRIC_RPC_CLIENT_CALL_DURATION, + unit="s", + description="Duration of an individual gRPC attempt in seconds (excluding retries).", + ) + self.http_request_duration = self.meter.create_histogram( + METRIC_HTTP_CLIENT_REQUEST_DURATION, + unit="s", + description="Duration of an individual HTTP request attempt in seconds (excluding retries).", + ) + self.client_request_duration = self.meter.create_histogram( + METRIC_GCP_CLIENT_REQUEST_DURATION, + unit="s", + description="End-to-end latency of a logical user-facing Veneer operation in seconds.", + ) + self.operations_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_OPERATIONS, + unit="{operation}", + description="Number of logical GCS client operations.", + ) + self.attempts_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_ATTEMPTS, + unit="{attempt}", + description="Number of physical network transport attempts executed.", + ) + self.retries_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_RETRIES, + unit="{retry}", + description="Number of transport retries executed after a failed attempt.", + ) + self.request_body_size = self.meter.create_histogram( + METRIC_STORAGE_REQUEST_BODY_SIZE, + unit="By", + description="Application payload bytes submitted by the caller for an upload or write operation.", + ) + self.response_body_size = self.meter.create_histogram( + METRIC_STORAGE_RESPONSE_BODY_SIZE, + unit="By", + description="Application payload bytes delivered to the caller.", + ) + self.ttfb = self.meter.create_histogram( + METRIC_STORAGE_CLIENT_TTFB, + unit="s", + description="Time elapsed from attempt initiation to receipt of the first response byte.", + ) + self.operation_ttfb = self.ttfb + self.errors_counter = self.meter.create_counter( + METRIC_STORAGE_CLIENT_ERRORS, + unit="{error}", + description="Number of physical network transport attempts terminating with a non-OK status.", + ) + else: + self.rpc_call_duration = None + self.http_request_duration = None + self.client_request_duration = None + self.operations_counter = None + self.attempts_counter = None + self.retries_counter = None + self.request_body_size = None + self.response_body_size = None + self.ttfb = None + self.operation_ttfb = None + self.errors_counter = None + + def get_base_attributes(self) -> Dict[str, Any]: + """Returns a copy of the base attributes for this recorder instance.""" + return self._base_attributes.copy() + + def _build_operation_attributes( + self, + operation: Optional[str] = None, + client_method: Optional[str] = None, + current_op: Optional[_OperationState] = None, + ) -> Dict[str, Any]: + """Builds base attributes enriched with current or explicit operation metadata.""" + attrs = self._base_attributes.copy() + if current_op is None: + current_op = get_current_operation() + raw_op = operation or (current_op.operation if current_op else None) + resolved_method = client_method or ( + current_op.client_method if current_op else None + ) + if raw_op is not None: + attrs["gcp.storage.operation"] = normalize_logical_operation(raw_op) + if resolved_method is not None: + attrs["gcp.client.method"] = resolved_method + return attrs + + def record_logical_operation( + self, + duration: float, + client_method: str, + operation: str, + error_type: str = "OK", + rpc_method: Optional[str] = None, + ) -> None: + """Records end-to-end logical operation duration and count.""" + if not self.standard_enabled or self.meter is None: + return + logical_op = normalize_logical_operation(operation) + full_rpc_method = resolve_rpc_method( + logical_op, rpc_method=rpc_method or operation + ) + op_attrs = { + **self._base_attributes, + "gcp.storage.operation": logical_op, + "gcp.client.method": client_method, + "error.type": error_type, + } + duration_attrs = { + **op_attrs, + "rpc.method": full_rpc_method, + } + self.client_request_duration.record( + max(0.0, duration), attributes=duration_attrs + ) + self.operations_counter.add(1, attributes=op_attrs) + + def record_retry( + self, + operation: Optional[str] = None, + error_type: str = "UNKNOWN", + network_path: Optional[str] = None, + ) -> None: + """Records a transport retry attempt with the prior failed attempt's error type.""" + if not self.standard_enabled or self.meter is None: + return + current_op = get_current_operation() + raw_op = operation or (current_op.operation if current_op else None) + retry_attrs: Dict[str, Any] = { + **self._base_attributes, + "error.type": error_type, + } + if raw_op is not None: + retry_attrs["gcp.storage.operation"] = normalize_logical_operation(raw_op) + if network_path is not None: + retry_attrs["network.path"] = network_path + self.retries_counter.add(1, attributes=retry_attrs) + + def record_http_attempt( + self, + duration: float, + http_method: str, + url_template: str, + status_code: Optional[int] = None, + error_type: str = "OK", + operation: Optional[str] = None, + client_method: Optional[str] = None, + network_path: Optional[str] = None, + ) -> None: + """Records a single physical HTTP request attempt.""" + if not self.standard_enabled or self.meter is None: + return + del client_method + current_op = get_current_operation() + raw_op = operation or (current_op.operation if current_op else None) + resolved_op = ( + normalize_logical_operation(raw_op) if raw_op is not None else None + ) + if current_op is not None: + current_op.recorded_attempts += 1 + prior_error = current_op.consume_retry_error() + if prior_error is not None: + self.record_retry( + operation=resolved_op, + error_type=prior_error, + network_path=network_path, + ) + if error_type != "OK": + current_op.mark_attempt_failed(error_type) + + valid_code = ( + isinstance(status_code, int) + and not isinstance(status_code, bool) + and 100 <= status_code < 600 + ) + + # 1. http.client.request.duration attributes + http_attrs: Dict[str, Any] = { + **self._base_attributes, + "http.request.method": http_method.upper() if http_method else "UNKNOWN", + "url.template": url_template, + "error.type": error_type, + } + if valid_code: + http_attrs["http.response.status_code"] = status_code + if network_path is not None: + http_attrs["network.path"] = network_path + + # 2. gcp.storage.client.attempts attributes + attempt_attrs: Dict[str, Any] = { + **self._base_attributes, + "error.type": error_type, + } + if resolved_op is not None: + attempt_attrs["gcp.storage.operation"] = resolved_op + if valid_code: + attempt_attrs["http.response.status_code"] = status_code + if network_path is not None: + attempt_attrs["network.path"] = network_path + + self.http_request_duration.record(max(0.0, duration), attributes=http_attrs) + self.attempts_counter.add(1, attributes=attempt_attrs) + + # 3. gcp.storage.client.errors attributes + if error_type != "OK": + error_attrs: Dict[str, Any] = { + **self._base_attributes, + "error.type": error_type, + } + if resolved_op is not None: + error_attrs["gcp.storage.operation"] = resolved_op + if network_path is not None: + error_attrs["network.path"] = network_path + self.errors_counter.add(1, attributes=error_attrs) + + def record_rpc_attempt( + self, + duration: float, + operation: Optional[str] = None, + status_code: Optional[str] = None, + error_type: str = "OK", + rpc_method: Optional[str] = None, + network_path: Optional[str] = None, + op_state: Optional[_OperationState] = None, + ) -> None: + """Records a single physical gRPC call attempt.""" + if not self.standard_enabled or self.meter is None: + return + active_op = op_state if op_state is not None else get_current_operation() + raw_rpc = ( + rpc_method or operation or (active_op.rpc_method if active_op else None) + ) + resolved_rpc_method = resolve_rpc_method(operation, rpc_method=raw_rpc) + raw_op = operation or rpc_method or (active_op.operation if active_op else None) + resolved_op = normalize_logical_operation(raw_op) + + if active_op is not None: + active_op.recorded_attempts += 1 + prior_error = active_op.consume_retry_error() + if prior_error is not None: + self.record_retry( + operation=resolved_op, + error_type=prior_error, + network_path=network_path, + ) + if error_type != "OK": + active_op.mark_attempt_failed(error_type) + + # 1. rpc.client.call.duration attributes + rpc_attrs: Dict[str, Any] = { + **self._base_attributes, + "rpc.method": resolved_rpc_method, + "error.type": error_type, + } + if status_code is not None: + rpc_attrs["rpc.response.status_code"] = status_code + if network_path is not None: + rpc_attrs["network.path"] = network_path + + # 2. gcp.storage.client.attempts attributes + attempt_attrs: Dict[str, Any] = { + **self._base_attributes, + "gcp.storage.operation": resolved_op, + "error.type": error_type, + } + if status_code is not None: + attempt_attrs["rpc.response.status_code"] = status_code + if network_path is not None: + attempt_attrs["network.path"] = network_path + + self.rpc_call_duration.record(max(0.0, duration), attributes=rpc_attrs) + self.attempts_counter.add(1, attributes=attempt_attrs) + + # 3. gcp.storage.client.errors attributes + if error_type != "OK": + error_attrs: Dict[str, Any] = { + **self._base_attributes, + "gcp.storage.operation": resolved_op, + "error.type": error_type, + } + if network_path is not None: + error_attrs["network.path"] = network_path + self.errors_counter.add(1, attributes=error_attrs) + + def record_request_body_size( + self, + size: int, + operation: Optional[str] = None, + client_method: Optional[str] = None, + ) -> None: + """Records the payload byte size of an upload or write operation.""" + if not self.standard_enabled or self.meter is None: + return + if not isinstance(size, int) or isinstance(size, bool): + return + current_op = get_current_operation() + if current_op is not None and current_op.suppressed: + return + attrs = self._build_operation_attributes( + operation=operation, client_method=client_method, current_op=current_op + ) + self.request_body_size.record(max(0, size), attributes=attrs) + + def record_response_body_size( + self, + size: int, + operation: Optional[str] = None, + client_method: Optional[str] = None, + ) -> None: + """Records the payload byte size of a download or read operation.""" + if not self.standard_enabled or self.meter is None: + return + if not isinstance(size, int) or isinstance(size, bool): + return + current_op = get_current_operation() + if current_op is not None and current_op.suppressed: + return + attrs = self._build_operation_attributes( + operation=operation, client_method=client_method, current_op=current_op + ) + self.response_body_size.record(max(0, size), attributes=attrs) + + def record_ttfb( + self, + duration: float, + operation: Optional[str] = None, + network_path: Optional[str] = None, + ) -> None: + """Records time to first response byte (TTFB) in seconds.""" + if not self.standard_enabled or self.meter is None: + return + current_op = get_current_operation() + if current_op is not None and current_op.suppressed: + return + raw_op = operation or (current_op.operation if current_op else None) + attrs: Dict[str, Any] = self._base_attributes.copy() + if raw_op is not None: + attrs["gcp.storage.operation"] = normalize_logical_operation(raw_op) + if network_path is not None: + attrs["network.path"] = network_path + self.ttfb.record(max(0.0, duration), attributes=attrs) + + def shutdown(self) -> None: + """Flushes pending metrics on a custom MeterProvider if configured.""" + if self._meter_provider is not None and hasattr( + self._meter_provider, "force_flush" + ): + try: + self._meter_provider.force_flush() + except Exception as exc: + logger.debug( + "Failed to force_flush custom MeterProvider during client shutdown: %s", + exc, + ) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py index 6251266b7240..b9617e1164d9 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +import inspect import logging from typing import Awaitable, Callable, Dict, Optional, Set @@ -211,6 +212,9 @@ async def reopen_stream( self._ensure_recv_loop() async def close(self) -> None: + mark_clean = getattr(self._stream, "_mark_closing_cleanly", None) + if callable(mark_clean) and not inspect.iscoroutinefunction(mark_clean): + mark_clean() self._stop_recv_loop() if self._recv_task: try: diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py index 1af54d6ffe43..974dfa194b63 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_appendable_object_writer.py @@ -24,9 +24,10 @@ from google.cloud import _storage_v2 from google.cloud._storage_v2.types import BidiWriteObjectRedirectedError from google.cloud._storage_v2.types.storage import BidiWriteObjectRequest -from google.cloud.storage import Blob +from google.cloud.storage import Blob, _opentelemetry_metrics from google.cloud.storage.asyncio.async_grpc_client import ( AsyncGrpcClient, + _resolve_network_path, ) from google.cloud.storage.asyncio.async_write_object_stream import ( _AsyncWriteObjectStream, @@ -46,6 +47,7 @@ _MAX_CHUNK_SIZE_BYTES = 2 * 1024 * 1024 # 2 MiB _DEFAULT_FLUSH_INTERVAL_BYTES = 16 * 1024 * 1024 # 16 MiB +_BIDI_WRITE_OBJECT_RPC = "google.storage.v2.Storage/BidiWriteObject" _BIDI_WRITE_REDIRECTED_TYPE_URL = ( "type.googleapis.com/google.storage.v2.BidiWriteObjectRedirectedError" ) @@ -219,6 +221,7 @@ def __init__( self.object_resource: Optional[_storage_v2.Object] = None self._flush_count = 0 self.blob: Optional[Blob] = None + self._suppress_append_body_size: bool = False @classmethod def from_blob( @@ -274,6 +277,21 @@ def from_blob( instance.blob = blob return instance + def _current_network_path( + self, + exc: Optional[BaseException] = None, + received_server_response: bool = True, + ) -> str: + """Resolves ``network.path`` from the underlying Bidi write stream call.""" + call = getattr( + getattr(self.write_obj_stream, "socket_like_rpc", None), "call", None + ) + return _resolve_network_path( + call=call, + exc=exc, + received_server_response=received_server_response, + ) + async def state_lookup(self) -> int: """Returns the persisted_size @@ -283,17 +301,26 @@ async def state_lookup(self) -> int: :raises ValueError: If the stream is not open (i.e., `open()` has not been called). """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before state_lookup().") + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.state_lookup", + operation="WriteObject", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + if not self._is_stream_open: + raise ValueError( + "Stream is not open. Call open() before state_lookup()." + ) - await self.write_obj_stream.send( - _storage_v2.BidiWriteObjectRequest( - state_lookup=True, + await self.write_obj_stream.send( + _storage_v2.BidiWriteObjectRequest( + state_lookup=True, + ) ) - ) - response = await self.write_obj_stream.recv() - self.persisted_size = response.persisted_size - return self.persisted_size + response = await self.write_obj_stream.recv() + self.persisted_size = response.persisted_size + return self.persisted_size def _on_open_error(self, exc): """Extracts routing token and write handle on redirect error during open.""" @@ -340,59 +367,97 @@ async def open( :raises ValueError: If the stream is already open. """ - if self._is_stream_open: - raise ValueError("Underlying bidi-gRPC stream is already open") - - retry_policy = self._merge_retry_policy(retry_policy) - - async def _do_open(): - current_metadata = list(metadata) if metadata else [] + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.open", + operation="WriteObject", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + if self._is_stream_open: + raise ValueError("Underlying bidi-gRPC stream is already open") - # Cleanup stream from previous failed attempt, if any. - if self.write_obj_stream: - if self.write_obj_stream.is_stream_open: - try: - await self.write_obj_stream.close() - except Exception as e: - logger.warning( - f"Error closing previous write stream during open retry. Got exception: {e}" - ) - self.write_obj_stream = None - self._is_stream_open = False + retry_policy = self._merge_retry_policy(retry_policy) - self.write_obj_stream = _AsyncWriteObjectStream( - client=self.client.grpc_client, - bucket_name=self.bucket_name, - object_name=self.object_name, - blob=self.blob, - generation_number=self.generation, - write_handle=self.write_handle, - routing_token=self._routing_token, - storage_class=self.storage_class, - ) + async def _do_open(): + current_op = _opentelemetry_metrics.get_current_operation() + attempts_before = ( + current_op.recorded_attempts if current_op is not None else 0 + ) + current_metadata = list(metadata) if metadata else [] + + # Cleanup stream from previous failed attempt, if any. + if self.write_obj_stream: + if self.write_obj_stream.is_stream_open: + try: + await self.write_obj_stream.close() + except Exception as e: + logger.warning( + f"Error closing previous write stream during open retry. Got exception: {e}" + ) + self.write_obj_stream = None + self._is_stream_open = False - if self._routing_token: - current_metadata.append( - ("x-goog-request-params", f"routing_token={self._routing_token}") + self.write_obj_stream = _AsyncWriteObjectStream( + client=self.client.grpc_client, + bucket_name=self.bucket_name, + object_name=self.object_name, + blob=self.blob, + generation_number=self.generation, + write_handle=self.write_handle, + routing_token=self._routing_token, + storage_class=self.storage_class, ) + if recorder is not None: + self.write_obj_stream._metrics_recorder = recorder + + if self._routing_token: + current_metadata.append( + ( + "x-goog-request-params", + f"routing_token={self._routing_token}", + ) + ) - await self.write_obj_stream.open( - metadata=current_metadata if current_metadata else None - ) + try: + await self.write_obj_stream.open( + metadata=current_metadata if current_metadata else None + ) + except Exception as exc: + if recorder is not None: + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + exc, + attempts_before=attempts_before, + will_retry=bool(retry_policy._predicate(exc)), + network_path=self._current_network_path( + exc=exc, received_server_response=False + ), + ) + raise + + if recorder is not None: + _opentelemetry_metrics.record_pending_retry( + recorder, + operation="WriteObject", + network_path=self._current_network_path( + received_server_response=True + ), + ) - if self.write_obj_stream.generation_number: - self.generation = self.write_obj_stream.generation_number - if self.write_obj_stream.write_handle: - self.write_handle = self.write_obj_stream.write_handle - if self.write_obj_stream.persisted_size is not None: - self.persisted_size = self.write_obj_stream.persisted_size - # set offset while opening - self.offset = self.persisted_size + if self.write_obj_stream.generation_number: + self.generation = self.write_obj_stream.generation_number + if self.write_obj_stream.write_handle: + self.write_handle = self.write_obj_stream.write_handle + if self.write_obj_stream.persisted_size is not None: + self.persisted_size = self.write_obj_stream.persisted_size + # set offset while opening + self.offset = self.persisted_size - self._is_stream_open = True - self._routing_token = None + self._is_stream_open = True + self._routing_token = None - await retry_policy(_do_open)() + await retry_policy(_do_open)() async def append( self, @@ -425,106 +490,129 @@ async def append( :raises ValueError: If the stream is not open. """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before append().") - if not data: - logger.debug("No data provided to append; returning without action.") - return - - if retry_policy is None: - retry_policy = AsyncRetry(predicate=_is_write_retryable) - - strategy = _WriteResumptionStrategy() - buffer = io.BytesIO(data) - attempt_count = 0 - - def send_and_recv_generator( - requests: List[BidiWriteObjectRequest], - state: Dict[str, _WriteState], - metadata: Optional[List[Tuple[str, str]]] = None, + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.append", + operation="WriteObject", + rpc_method=_BIDI_WRITE_OBJECT_RPC, ): - async def generator(): - nonlocal attempt_count - nonlocal requests - attempt_count += 1 - resp = None - write_state = state["write_state"] - # If this is a retry or redirect, we must re-open the stream - if attempt_count > 1 or write_state.routing_token: - logger.info( - f"Re-opening the stream with attempt_count: {attempt_count}" - ) - - current_metadata = list(metadata) if metadata else [] - if write_state.routing_token: - current_metadata.append( - ( - "x-goog-request-params", - f"routing_token={write_state.routing_token}", - ) + if not self._is_stream_open: + raise ValueError("Stream is not open. Call open() before append().") + if not data: + logger.debug("No data provided to append; returning without action.") + return + + if retry_policy is None: + retry_policy = AsyncRetry(predicate=_is_write_retryable) + + strategy = _WriteResumptionStrategy() + buffer = io.BytesIO(data) + attempt_count = 0 + + def send_and_recv_generator( + requests: List[BidiWriteObjectRequest], + state: Dict[str, _WriteState], + metadata: Optional[List[Tuple[str, str]]] = None, + ): + async def generator(): + nonlocal attempt_count + nonlocal requests + attempt_count += 1 + resp = None + write_state = state["write_state"] + # If this is a retry or redirect, we must re-open the stream + if attempt_count > 1 or write_state.routing_token: + logger.info( + f"Re-opening the stream with attempt_count: {attempt_count}" ) - self._routing_token = write_state.routing_token - - self._is_stream_open = False - await self.open(metadata=current_metadata) - - write_state.persisted_size = self.persisted_size - write_state.write_handle = self.write_handle - write_state.routing_token = None - write_state.user_buffer.seek(write_state.persisted_size) - write_state.bytes_sent = write_state.persisted_size - write_state.bytes_since_last_flush = 0 - self.bytes_appended_since_last_flush = 0 - - requests = strategy.generate_requests(state) - - for chunk_req in requests: - await self.write_obj_stream.send(chunk_req) - if chunk_req.flush: - self._flush_count += 1 + current_metadata = list(metadata) if metadata else [] + if write_state.routing_token: + current_metadata.append( + ( + "x-goog-request-params", + f"routing_token={write_state.routing_token}", + ) + ) + self._routing_token = write_state.routing_token + + self._is_stream_open = False + await self.open(metadata=current_metadata) + if recorder is not None: + _opentelemetry_metrics.record_pending_retry( + recorder, + operation="WriteObject", + network_path=self._current_network_path( + received_server_response=True + ), + ) - resp = None - if chunk_req.state_lookup: - # TODO: if there's error, it'll raise error - # and will be handled by `recover_state_on_failure` - resp = await self.write_obj_stream.recv() - - if resp: - if resp.persisted_size is not None: - self.persisted_size = resp.persisted_size - state["write_state"].persisted_size = resp.persisted_size - self.offset = self.persisted_size - if resp.write_handle: - self.write_handle = resp.write_handle - state["write_state"].write_handle = resp.write_handle - - yield resp - - return generator() - - # State initialization - write_state = _WriteState( - _MAX_CHUNK_SIZE_BYTES, - buffer, - self.flush_interval, - enable_checksum=enable_checksum, - ) - write_state.write_handle = self.write_handle - write_state.persisted_size = self.persisted_size - # offset is set during `open()` call. - write_state.bytes_sent = self.offset or 0 - write_state.bytes_since_last_flush = self.bytes_appended_since_last_flush - - retry_manager = _BidiStreamRetryManager( - _WriteResumptionStrategy(), - lambda r, s: send_and_recv_generator(r, s, metadata), - ) - await retry_manager.execute({"write_state": write_state}, retry_policy) + write_state.persisted_size = self.persisted_size + write_state.write_handle = self.write_handle + write_state.routing_token = None + + write_state.user_buffer.seek(write_state.persisted_size) + write_state.bytes_sent = write_state.persisted_size + write_state.bytes_since_last_flush = 0 + self.bytes_appended_since_last_flush = 0 + + requests = strategy.generate_requests(state) + + for chunk_req in requests: + await self.write_obj_stream.send(chunk_req) + if chunk_req.flush: + self._flush_count += 1 + + resp = None + if chunk_req.state_lookup: + # TODO: if there's error, it'll raise error + # and will be handled by `recover_state_on_failure` + resp = await self.write_obj_stream.recv() + + if resp: + if resp.persisted_size is not None: + self.persisted_size = resp.persisted_size + state[ + "write_state" + ].persisted_size = resp.persisted_size + self.offset = self.persisted_size + if resp.write_handle: + self.write_handle = resp.write_handle + state["write_state"].write_handle = resp.write_handle + + yield resp + + return generator() + + # State initialization + write_state = _WriteState( + _MAX_CHUNK_SIZE_BYTES, + buffer, + self.flush_interval, + enable_checksum=enable_checksum, + ) + write_state.write_handle = self.write_handle + write_state.persisted_size = self.persisted_size + # offset is set during `open()` call. + write_state.bytes_sent = self.offset or 0 + write_state.bytes_since_last_flush = self.bytes_appended_since_last_flush + + retry_manager = _BidiStreamRetryManager( + _WriteResumptionStrategy(), + lambda r, s: send_and_recv_generator(r, s, metadata), + recorder=recorder, + network_path_getter=lambda: self._current_network_path( + received_server_response=True + ), + ) + await retry_manager.execute({"write_state": write_state}, retry_policy) - # Sync local markers - self.bytes_appended_since_last_flush = write_state.bytes_since_last_flush - self.offset = write_state.bytes_sent + # Sync local markers + self.bytes_appended_since_last_flush = write_state.bytes_since_last_flush + self.offset = write_state.bytes_sent + if recorder is not None and not self._suppress_append_body_size: + recorder.record_request_body_size(len(data), operation="WriteObject") async def simple_flush(self) -> None: """Flushes the data to the server. @@ -535,15 +623,24 @@ async def simple_flush(self) -> None: :raises ValueError: If the stream is not open (i.e., `open()` has not been called). """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before simple_flush().") + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.simple_flush", + operation="WriteObject", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + if not self._is_stream_open: + raise ValueError( + "Stream is not open. Call open() before simple_flush()." + ) - await self.write_obj_stream.send( - _storage_v2.BidiWriteObjectRequest( - flush=True, + await self.write_obj_stream.send( + _storage_v2.BidiWriteObjectRequest( + flush=True, + ) ) - ) - self.bytes_appended_since_last_flush = 0 + self.bytes_appended_since_last_flush = 0 async def flush(self) -> int: """Flushes the data to the server. @@ -554,20 +651,27 @@ async def flush(self) -> int: :raises ValueError: If the stream is not open (i.e., `open()` has not been called). """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before flush().") + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.flush", + operation="WriteObject", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + if not self._is_stream_open: + raise ValueError("Stream is not open. Call open() before flush().") - await self.write_obj_stream.send( - _storage_v2.BidiWriteObjectRequest( - flush=True, - state_lookup=True, + await self.write_obj_stream.send( + _storage_v2.BidiWriteObjectRequest( + flush=True, + state_lookup=True, + ) ) - ) - response = await self.write_obj_stream.recv() - self.persisted_size = response.persisted_size - self.offset = self.persisted_size - self.bytes_appended_since_last_flush = 0 - return self.persisted_size + response = await self.write_obj_stream.recv() + self.persisted_size = response.persisted_size + self.offset = self.persisted_size + self.bytes_appended_since_last_flush = 0 + return self.persisted_size async def close( self, @@ -612,51 +716,83 @@ async def close( server. """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before close().") - - if full_object_checksum is not None and not finalize_on_close: - raise ValueError( - "full_object_checksum can only be provided when finalize_on_close is True." - ) + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.close", + operation="WriteObject", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + if not self._is_stream_open: + raise ValueError("Stream is not open. Call open() before close().") - if finalize_on_close: - return await self.finalize( - full_object_checksum=full_object_checksum, - retry_policy=retry_policy, - ) + if full_object_checksum is not None and not finalize_on_close: + raise ValueError( + "full_object_checksum can only be provided when finalize_on_close is True." + ) - retry_policy = self._merge_retry_policy(retry_policy) + if finalize_on_close: + return await self.finalize( + full_object_checksum=full_object_checksum, + retry_policy=retry_policy, + ) - attempt_count = 0 - expected_offset = self.offset + retry_policy = self._merge_retry_policy(retry_policy) - async def _do_close(): - nonlocal attempt_count - attempt_count += 1 + attempt_count = 0 + expected_offset = self.offset - if attempt_count > 1: - logger.info( - f"Re-opening the stream for close retry attempt: {attempt_count}" + async def _do_close(): + nonlocal attempt_count + attempt_count += 1 + current_op = _opentelemetry_metrics.get_current_operation() + attempts_before = ( + current_op.recorded_attempts if current_op is not None else 0 ) - self._is_stream_open = False - await self.open() - if ( - self.offset is not None - and expected_offset is not None - and self.offset != expected_offset - ): - raise exceptions.InternalServerError( - f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" - ) - await self.write_obj_stream.close() - return self.persisted_size + try: + if attempt_count > 1: + logger.info( + f"Re-opening the stream for close retry attempt: {attempt_count}" + ) + self._is_stream_open = False + await self.open() + if recorder is not None: + _opentelemetry_metrics.record_pending_retry( + recorder, + operation="WriteObject", + network_path=self._current_network_path( + received_server_response=True + ), + ) + if ( + self.offset is not None + and expected_offset is not None + and self.offset != expected_offset + ): + raise exceptions.InternalServerError( + f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" + ) - try: - return await retry_policy(_do_close)() - finally: - self._is_stream_open = False + await self.write_obj_stream.close() + except Exception as exc: + if recorder is not None: + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + exc, + attempts_before=attempts_before, + will_retry=bool(retry_policy._predicate(exc)), + network_path=self._current_network_path( + exc=exc, received_server_response=False + ), + ) + raise + return self.persisted_size + + try: + return await retry_policy(_do_close)() + finally: + self._is_stream_open = False async def finalize( self, @@ -699,67 +835,101 @@ async def finalize( full_object_checksum does not match the checksum computed by the server. """ - if not self._is_stream_open: - raise ValueError("Stream is not open. Call open() before finalize().") - - if full_object_checksum is None: - finalize_req = _storage_v2.BidiWriteObjectRequest(finish_write=True) - elif isinstance(full_object_checksum, bool) or not isinstance( - full_object_checksum, int + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.finalize", + operation="WriteObject", + rpc_method=_BIDI_WRITE_OBJECT_RPC, ): - raise TypeError("full_object_checksum must be an integer.") - elif not (0 <= full_object_checksum <= 0xFFFFFFFF): - raise ValueError("full_object_checksum must be a 32-bit unsigned integer.") - else: - finalize_req = _storage_v2.BidiWriteObjectRequest( - finish_write=True, - object_checksums=_storage_v2.ObjectChecksums( - crc32c=full_object_checksum - ), - ) - - retry_policy = self._merge_retry_policy(retry_policy) + if not self._is_stream_open: + raise ValueError("Stream is not open. Call open() before finalize().") + + if full_object_checksum is None: + finalize_req = _storage_v2.BidiWriteObjectRequest(finish_write=True) + elif isinstance(full_object_checksum, bool) or not isinstance( + full_object_checksum, int + ): + raise TypeError("full_object_checksum must be an integer.") + elif not (0 <= full_object_checksum <= 0xFFFFFFFF): + raise ValueError( + "full_object_checksum must be a 32-bit unsigned integer." + ) + else: + finalize_req = _storage_v2.BidiWriteObjectRequest( + finish_write=True, + object_checksums=_storage_v2.ObjectChecksums( + crc32c=full_object_checksum + ), + ) - attempt_count = 0 - expected_offset = self.offset + retry_policy = self._merge_retry_policy(retry_policy) - async def _do_finalize(): - nonlocal attempt_count - attempt_count += 1 + attempt_count = 0 + expected_offset = self.offset - if attempt_count > 1: - logger.info( - f"Re-opening the stream for finalize retry attempt: {attempt_count}" + async def _do_finalize(): + nonlocal attempt_count + attempt_count += 1 + current_op = _opentelemetry_metrics.get_current_operation() + attempts_before = ( + current_op.recorded_attempts if current_op is not None else 0 ) - self._is_stream_open = False - await self.open() - if ( - self.offset is not None - and expected_offset is not None - and self.offset != expected_offset - ): - raise exceptions.InternalServerError( - f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" - ) - await self.write_obj_stream.send(finalize_req) - response = await self.write_obj_stream.recv() - self.object_resource = response.resource - self.persisted_size = self.object_resource.size - return self.object_resource - - try: - return await retry_policy(_do_finalize)() - finally: - if self.write_obj_stream: try: - await self.write_obj_stream.close() - except Exception as e: - logger.debug( - f"Stream close during finalize cleanup resulted in: {e}" - ) - self._is_stream_open = False - self.offset = None + if attempt_count > 1: + logger.info( + f"Re-opening the stream for finalize retry attempt: {attempt_count}" + ) + self._is_stream_open = False + await self.open() + if recorder is not None: + _opentelemetry_metrics.record_pending_retry( + recorder, + operation="WriteObject", + network_path=self._current_network_path( + received_server_response=True + ), + ) + if ( + self.offset is not None + and expected_offset is not None + and self.offset != expected_offset + ): + raise exceptions.InternalServerError( + f"Unrecoverable data loss during reconnect. Expected offset {expected_offset}, got {self.offset}" + ) + + await self.write_obj_stream.send(finalize_req) + response = await self.write_obj_stream.recv() + except Exception as exc: + if recorder is not None: + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + exc, + attempts_before=attempts_before, + will_retry=bool(retry_policy._predicate(exc)), + network_path=self._current_network_path( + exc=exc, received_server_response=False + ), + ) + raise + self.object_resource = response.resource + self.persisted_size = self.object_resource.size + return self.object_resource + + try: + return await retry_policy(_do_finalize)() + finally: + if self.write_obj_stream: + try: + await self.write_obj_stream.close() + except Exception as e: + logger.debug( + f"Stream close during finalize cleanup resulted in: {e}" + ) + self._is_stream_open = False + self.offset = None @property def is_stream_open(self) -> bool: @@ -792,5 +962,21 @@ async def append_from_file( :param file_obj: A file handle opened in binary mode for reading. """ - while block := file_obj.read(block_size): - await self.append(block) + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.append_from_file", + operation="WriteObject", + rpc_method=_BIDI_WRITE_OBJECT_RPC, + ): + total_bytes = 0 + prev_suppress = self._suppress_append_body_size + self._suppress_append_body_size = True + try: + while block := file_obj.read(block_size): + await self.append(block) + total_bytes += len(block) + finally: + self._suppress_append_body_size = prev_suppress + if recorder is not None and total_bytes > 0: + recorder.record_request_body_size(total_bytes, operation="WriteObject") diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py index 21278cd624bf..56c7ff30cdde 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py @@ -14,6 +14,13 @@ """An async client for interacting with Google Cloud Storage using the gRPC API.""" +import asyncio +import ipaddress +import re +import time +from collections import deque +from typing import Any, Iterator, Optional, Tuple + import grpc from google.auth import credentials as auth_credentials @@ -28,6 +35,790 @@ _rewrite_host_for_interconnect, ) +_GRPC_STATUS_NAMES = {sc.name for sc in grpc.StatusCode} +_DIRECTPATH_IPV4_NET = ipaddress.ip_network("34.126.0.0/18") +_DIRECTPATH_IPV6_NET = ipaddress.ip_network("2001:4860:8040::/42") +_GFE_METADATA_KEYS = frozenset({"x-goog-gfe-service-time", "x-guploader-uploadid"}) +_PEER_IP_RE = re.compile( + r"ipv4:([0-9.]+)|ipv6:(?:(?:\[|%5B|%5b)([0-9a-fA-F:.]+)(?:\]|%5D|%5d)|([0-9a-fA-F:]+))" +) + + +def _extract_rpc_method(method_name: Any) -> str: + """Normalizes a gRPC method path into ``package.Service/Method``.""" + if isinstance(method_name, bytes): + method_name = method_name.decode("utf-8", errors="replace") + if not isinstance(method_name, str): + return "UNKNOWN" + normalized = method_name.strip().strip("/") + return normalized if normalized else "UNKNOWN" + + +_extract_rpc_operation = _extract_rpc_method + + +def _extract_logical_operation(rpc_method: str) -> str: + """Extracts the short logical ``gcp.storage.operation`` from a gRPC method path.""" + return _opentelemetry_metrics.normalize_logical_operation(rpc_method) + + +def _resolve_status_and_error(exc: BaseException) -> Tuple[Optional[str], str]: + """Resolves ``(rpc.response.status_code, error.type)`` from an exception.""" + if isinstance(exc, asyncio.CancelledError): + return "CANCELLED", "CANCELLED" + + status_code: Optional[str] = None + for item in _opentelemetry_metrics._iter_exception_chain(exc): + grpc_code = getattr(item, "grpc_status_code", None) + if grpc_code is not None: + name = getattr(grpc_code, "name", None) + status_code = str(name) if name else str(grpc_code) + break + + code_attr = getattr(item, "code", None) + if callable(code_attr): + try: + rpc_status = code_attr() + name = getattr(rpc_status, "name", None) + if name: + status_code = str(name) + break + except Exception: # pragma: NO COVER + pass + + error_type = _opentelemetry_metrics.classify_error_type(exc) + if status_code is None and error_type in _GRPC_STATUS_NAMES: + status_code = error_type + return status_code, error_type + + +def _extract_peer_ip(text: str) -> Optional[str]: + """Extracts the last valid peer IPv4/IPv6 address from a gRPC peer or debug string.""" + last_valid: Optional[str] = None + for match in _PEER_IP_RE.finditer(text): + candidate = match.group(1) or match.group(2) or match.group(3) + try: + ipaddress.ip_address(candidate) + last_valid = candidate + except ValueError: + continue + return last_valid + + +def _classify_ip(ip_str: str) -> str: + """Classifies a validated IP address string into ``'directpath'`` or ``'cloudpath'``.""" + addr = ipaddress.ip_address(ip_str) + if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None: + addr = addr.ipv4_mapped + if isinstance(addr, ipaddress.IPv4Address): + return "directpath" if addr in _DIRECTPATH_IPV4_NET else "cloudpath" + return "directpath" if addr in _DIRECTPATH_IPV6_NET else "cloudpath" + + +def _inspect_metadata(md: Any) -> Tuple[bool, bool]: + """Inspects a gRPC metadata sequence for non-empty headers and GFE markers.""" + if not isinstance(md, (tuple, list, grpc.aio.Metadata)): + return False, False + has_entries = False + for entry in md: + has_entries = True + key = getattr(entry, "key", None) + if key is None and isinstance(entry, (tuple, list)) and len(entry) >= 1: + key = entry[0] + if isinstance(key, bytes): + key = key.decode("utf-8", errors="replace") + if isinstance(key, str) and key.lower() in _GFE_METADATA_KEYS: + return True, True + return has_entries, False + + +def _iter_wrapped_calls(call: Any) -> Iterator[Any]: + """Yields ``call`` and any inner wrapped gRPC call objects.""" + seen = set() + queue = deque([call]) + steps = 0 + while queue and steps < 16: + steps += 1 + current = queue.popleft() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + yield current + inst_dict = getattr(current, "__dict__", None) + if not isinstance(inst_dict, dict): + continue + for attr in ( + "_call", + "call", + "_wrapped", + "_last_returned_call_from_interceptors", + ): + inner = inst_dict.get(attr) + if inner is not None: + queue.append(inner) + task = inst_dict.get("_interceptors_task") + if task is not None and hasattr(task, "done") and task.done(): + try: + queue.append(task.result()) + except Exception: # pragma: NO COVER + pass + + +def _resolve_network_path( + call: Any = None, + exc: Optional[BaseException] = None, + direct_path_attempted: Optional[bool] = None, + received_server_response: bool = False, +) -> str: + """Resolves per-call ``network.path`` (``'directpath'``, ``'cloudpath'``, or ``'unknown'``). + + Detection priority: + 1. GFE response metadata headers (``x-goog-gfe-service-time``, + ``x-guploader-uploadid``) on the Cython call or ``RpcError`` -> ``'cloudpath'``. + 2. Remote peer IP from ``call.peer()`` or C-core ``debug_error_string()`` + matched against DirectPath CIDRs ``34.126.0.0/18`` and + ``2001:4860:8040::/42`` -> ``'directpath'`` or ``'cloudpath'``. + 3. Fallback when Python ``grpc.aio`` hides ``peer()`` and no GFE header is + present: if the server responded, ``'directpath'`` when DirectPath was + attempted on the channel else ``'cloudpath'``; if no server response was + received (e.g. DNS or pre-connection failure), ``'unknown'``. + """ + has_server_metadata = False + detected_peer_ip: Optional[str] = None + + if call is not None: + for current in _iter_wrapped_calls(call): + if direct_path_attempted is None and isinstance(current, _TrackedCallBase): + direct_path_attempted = current._direct_path_attempted + + inst_dict = getattr(current, "__dict__", None) + cython_call = getattr(current, "_cython_call", None) + status_obj = ( + getattr(cython_call, "_status", None) + if cython_call is not None + else None + ) + status_trailing = None + if status_obj is not None: + trailing_fn = getattr(status_obj, "trailing_metadata", None) + status_trailing = ( + trailing_fn() + if callable(trailing_fn) + else getattr(status_obj, "_trailing_metadata", None) + ) + + md_candidates = [ + getattr(cython_call, "_initial_metadata", None) + if cython_call is not None + else None, + getattr(cython_call, "_trailing_metadata", None) + if cython_call is not None + else None, + status_trailing, + inst_dict.get("_initial_metadata") + if isinstance(inst_dict, dict) + else None, + inst_dict.get("_trailing_metadata") + if isinstance(inst_dict, dict) + else None, + ] + for md in md_candidates: + entries, is_gfe = _inspect_metadata(md) + if is_gfe: + return "cloudpath" + if entries: + has_server_metadata = True + + if detected_peer_ip is None: + peer_fn = getattr(current, "peer", None) + if callable(peer_fn): + try: + peer_val = peer_fn() + if isinstance(peer_val, bytes): + peer_val = peer_val.decode("utf-8", errors="replace") + if isinstance(peer_val, str): + detected_peer_ip = _extract_peer_ip(peer_val) + except Exception: # pragma: NO COVER + pass + + if detected_peer_ip is None and status_obj is not None: + dbg_fn = getattr(status_obj, "debug_error_string", None) + dbg_val = ( + dbg_fn() + if callable(dbg_fn) + else getattr(status_obj, "_debug_error_string", None) + ) + if isinstance(dbg_val, bytes): + dbg_val = dbg_val.decode("utf-8", errors="replace") + if isinstance(dbg_val, str): + detected_peer_ip = _extract_peer_ip(dbg_val) + + if exc is not None: + for item in _opentelemetry_metrics._iter_exception_chain(exc): + for md_attr in ("initial_metadata", "trailing_metadata"): + md_fn = getattr(item, md_attr, None) + md = md_fn() if callable(md_fn) else md_fn + entries, is_gfe = _inspect_metadata(md) + if is_gfe: + return "cloudpath" + if entries: + has_server_metadata = True + + if detected_peer_ip is None: + dbg_fn = getattr(item, "debug_error_string", None) + dbg_val = dbg_fn() if callable(dbg_fn) else dbg_fn + if isinstance(dbg_val, bytes): + dbg_val = dbg_val.decode("utf-8", errors="replace") + if isinstance(dbg_val, str): + detected_peer_ip = _extract_peer_ip(dbg_val) + + if detected_peer_ip is not None: + return _classify_ip(detected_peer_ip) + + server_responded = received_server_response or has_server_metadata + if direct_path_attempted is False: + return "cloudpath" if server_responded else "unknown" + if direct_path_attempted is True: + return "directpath" if server_responded else "unknown" + return "unknown" + + +def _mark_call_closing_cleanly(call: Any) -> None: + """Marks a wrapped gRPC call as closing cleanly before normal cancellation.""" + for current in _iter_wrapped_calls(call): + inst_dict = getattr(current, "__dict__", None) + if isinstance(current, _TrackedCallBase) or ( + isinstance(inst_dict, dict) and "_closing_cleanly" in inst_dict + ): + current._closing_cleanly = True + + +def _bind_call_to_operation( + call: Any, op_state: Optional[_opentelemetry_metrics._OperationState] +) -> None: + """Binds a long-lived gRPC stream call to the currently active operation.""" + if op_state is None: + return + for current in _iter_wrapped_calls(call): + if isinstance(current, _TrackedCallBase): + current._op_state = op_state + + +class _TrackedCallBase: + """Base proxy for ``grpc.aio.Call`` objects that records attempt metrics once.""" + + def __init__( + self, + call: Any, + recorder: _opentelemetry_metrics._MetricsRecorder, + rpc_method: str, + logical_operation: str, + start_time: float, + op_state: Optional[_opentelemetry_metrics._OperationState], + direct_path_attempted: bool = False, + ) -> None: + self._call = call + self._recorder = recorder + self._rpc_method = rpc_method + self._logical_operation = logical_operation + self._start_time = start_time + self._initial_op_state = op_state + self._op_state = op_state + self._started_after_failure = ( + op_state is not None and op_state.last_failure_error_type is not None + ) + self._direct_path_attempted = direct_path_attempted + self._received_response = False + self._ttfb_recorded = False + self._recorded = False + self._closing_cleanly = False + + def __getattr__(self, name: str) -> Any: + return getattr(self._call, name) + + def _record_ttfb_once( + self, + duration: Optional[float] = None, + network_path: Optional[str] = None, + ) -> None: + if not self._ttfb_recorded: + self._ttfb_recorded = True + ttfb_duration = ( + max(0.0, time.perf_counter() - self._start_time) + if duration is None + else duration + ) + resolved_path = ( + _resolve_network_path( + call=self._call, + direct_path_attempted=self._direct_path_attempted, + received_server_response=True, + ) + if network_path is None + else network_path + ) + self._recorder.record_ttfb( + duration=ttfb_duration, + operation=self._logical_operation, + network_path=resolved_path, + ) + + def _record_once( + self, + status_code: Optional[str], + error_type: str, + exc: Optional[BaseException] = None, + received_response: bool = False, + record_ttfb: bool = False, + ) -> None: + if not self._recorded: + self._recorded = True + current_op = _opentelemetry_metrics.get_current_operation() + fallback_op = ( + self._op_state + if (self._op_state is not None and not self._op_state.finished) + else None + ) + active_op = current_op or fallback_op + if ( + error_type != "OK" + and active_op is not None + and active_op is not self._initial_op_state + ): + effective_start = active_op.start_time + else: + effective_start = self._start_time + duration = max(0.0, time.perf_counter() - effective_start) + resolved_path = _resolve_network_path( + call=self._call, + exc=exc, + direct_path_attempted=self._direct_path_attempted, + received_server_response=received_response or self._received_response, + ) + if record_ttfb: + self._record_ttfb_once( + duration=duration, + network_path=resolved_path, + ) + is_stale_teardown = ( + active_op is not None + and active_op.last_failure_error_type is not None + and not self._started_after_failure + ) + saved_prior_error = ( + active_op.consume_retry_error() if is_stale_teardown else None + ) + self._recorder.record_rpc_attempt( + duration=duration, + operation=self._logical_operation, + status_code=status_code, + error_type=error_type, + rpc_method=self._rpc_method, + network_path=resolved_path, + op_state=active_op, + ) + if is_stale_teardown: + active_op.last_failure_error_type = saved_prior_error + + def _record_from_exception(self, exc: BaseException) -> None: + if isinstance(exc, asyncio.CancelledError) and self._closing_cleanly: + self._record_once( + status_code="OK", + error_type="OK", + received_response=self._received_response, + ) + else: + status_code, error_type = _resolve_status_and_error(exc) + self._record_once(status_code=status_code, error_type=error_type, exc=exc) + + async def wait_for_connection(self) -> None: + try: + await self._call.wait_for_connection() + except BaseException as exc: + self._record_from_exception(exc) + raise + + def cancel(self) -> bool: + result = self._call.cancel() + if self._closing_cleanly: + self._record_once( + status_code="OK", + error_type="OK", + received_response=self._received_response, + ) + else: + self._record_once(status_code="CANCELLED", error_type="CANCELLED") + return result + + def cancelled(self) -> bool: + return self._call.cancelled() + + def done(self) -> bool: + return self._call.done() + + def add_done_callback(self, callback: Any) -> None: + self._call.add_done_callback(callback) + + def time_remaining(self) -> Optional[float]: + return self._call.time_remaining() + + async def initial_metadata(self) -> Any: + return await self._call.initial_metadata() + + async def trailing_metadata(self) -> Any: + return await self._call.trailing_metadata() + + async def code(self) -> Any: + return await self._call.code() + + async def details(self) -> str: + return await self._call.details() + + async def debug_error_string(self) -> str: + return await self._call.debug_error_string() + + +class _TrackedUnaryResponseMixin: + """Mixin for unary-response gRPC calls.""" + + async def _await_impl(self) -> Any: + try: + response = await self._call + except BaseException as exc: + self._record_from_exception(exc) + raise + else: + self._received_response = True + self._record_once( + status_code="OK", + error_type="OK", + received_response=True, + record_ttfb=True, + ) + return response + + def __await__(self): + return self._await_impl().__await__() + + +class _TrackedStreamResponseMixin: + """Mixin for streaming-response gRPC calls.""" + + async def _aiter_impl(self): + completed_normally = False + try: + async for response in self._call: + self._received_response = True + self._record_ttfb_once() + yield response + completed_normally = True + except GeneratorExit: + if self._closing_cleanly: + self._record_once( + status_code="OK", + error_type="OK", + received_response=self._received_response, + ) + else: + self._record_once(status_code="CANCELLED", error_type="CANCELLED") + raise + except BaseException as exc: + self._record_from_exception(exc) + raise + finally: + if completed_normally: + self._record_once( + status_code="OK", error_type="OK", received_response=True + ) + + def __aiter__(self): + return self._aiter_impl() + + async def read(self) -> Any: + try: + response = await self._call.read() + except BaseException as exc: + self._record_from_exception(exc) + raise + else: + self._received_response = True + if response is grpc.aio.EOF or response == grpc.aio.EOF: + self._record_once( + status_code="OK", error_type="OK", received_response=True + ) + else: + self._record_ttfb_once() + return response + + +class _TrackedStreamRequestMixin: + """Mixin for streaming-request gRPC calls.""" + + async def write(self, request: Any) -> None: + try: + await self._call.write(request) + except BaseException as exc: + self._record_from_exception(exc) + raise + + async def done_writing(self) -> None: + try: + await self._call.done_writing() + except BaseException as exc: + self._record_from_exception(exc) + raise + + @property + def _done_writing_flag(self) -> bool: + return getattr(self._call, "_done_writing_flag", False) + + @_done_writing_flag.setter + def _done_writing_flag(self, value: bool) -> None: + setattr(self._call, "_done_writing_flag", value) + + +class _TrackedUnaryUnaryCall( + _TrackedUnaryResponseMixin, _TrackedCallBase, grpc.aio.UnaryUnaryCall +): + """Tracked ``grpc.aio.UnaryUnaryCall`` proxy.""" + + +class _TrackedUnaryStreamCall( + _TrackedStreamResponseMixin, _TrackedCallBase, grpc.aio.UnaryStreamCall +): + """Tracked ``grpc.aio.UnaryStreamCall`` proxy.""" + + +class _TrackedStreamUnaryCall( + _TrackedUnaryResponseMixin, + _TrackedStreamRequestMixin, + _TrackedCallBase, + grpc.aio.StreamUnaryCall, +): + """Tracked ``grpc.aio.StreamUnaryCall`` proxy.""" + + +class _TrackedStreamStreamCall( + _TrackedStreamResponseMixin, + _TrackedStreamRequestMixin, + _TrackedCallBase, + grpc.aio.StreamStreamCall, +): + """Tracked ``grpc.aio.StreamStreamCall`` proxy.""" + + +class _MetricsUnaryUnaryInterceptor(grpc.aio.UnaryUnaryClientInterceptor): + """Client interceptor recording metrics for unary-unary gRPC attempts.""" + + def __init__( + self, + recorder: _opentelemetry_metrics._MetricsRecorder, + direct_path_attempted: bool = False, + ) -> None: + self._recorder = recorder + self._direct_path_attempted = direct_path_attempted + + async def intercept_unary_unary(self, continuation, client_call_details, request): + rpc_method = _extract_rpc_method(getattr(client_call_details, "method", None)) + logical_operation = _extract_logical_operation(rpc_method) + op_state = _opentelemetry_metrics.get_current_operation() + start_time = time.perf_counter() + try: + call = await continuation(client_call_details, request) + except BaseException as exc: + duration = max(0.0, time.perf_counter() - start_time) + status_code, error_type = _resolve_status_and_error(exc) + resolved_path = _resolve_network_path( + exc=exc, + direct_path_attempted=self._direct_path_attempted, + received_server_response=False, + ) + self._recorder.record_rpc_attempt( + duration=duration, + operation=logical_operation, + status_code=status_code, + error_type=error_type, + rpc_method=rpc_method, + network_path=resolved_path, + op_state=op_state, + ) + raise + return _TrackedUnaryUnaryCall( + call, + self._recorder, + rpc_method, + logical_operation, + start_time, + op_state, + direct_path_attempted=self._direct_path_attempted, + ) + + +class _MetricsUnaryStreamInterceptor(grpc.aio.UnaryStreamClientInterceptor): + """Client interceptor recording metrics for unary-stream gRPC attempts.""" + + def __init__( + self, + recorder: _opentelemetry_metrics._MetricsRecorder, + direct_path_attempted: bool = False, + ) -> None: + self._recorder = recorder + self._direct_path_attempted = direct_path_attempted + + async def intercept_unary_stream(self, continuation, client_call_details, request): + rpc_method = _extract_rpc_method(getattr(client_call_details, "method", None)) + logical_operation = _extract_logical_operation(rpc_method) + op_state = _opentelemetry_metrics.get_current_operation() + start_time = time.perf_counter() + try: + call = await continuation(client_call_details, request) + except BaseException as exc: + duration = max(0.0, time.perf_counter() - start_time) + status_code, error_type = _resolve_status_and_error(exc) + resolved_path = _resolve_network_path( + exc=exc, + direct_path_attempted=self._direct_path_attempted, + received_server_response=False, + ) + self._recorder.record_rpc_attempt( + duration=duration, + operation=logical_operation, + status_code=status_code, + error_type=error_type, + rpc_method=rpc_method, + network_path=resolved_path, + op_state=op_state, + ) + raise + return _TrackedUnaryStreamCall( + call, + self._recorder, + rpc_method, + logical_operation, + start_time, + op_state, + direct_path_attempted=self._direct_path_attempted, + ) + + +class _MetricsStreamUnaryInterceptor(grpc.aio.StreamUnaryClientInterceptor): + """Client interceptor recording metrics for stream-unary gRPC attempts.""" + + def __init__( + self, + recorder: _opentelemetry_metrics._MetricsRecorder, + direct_path_attempted: bool = False, + ) -> None: + self._recorder = recorder + self._direct_path_attempted = direct_path_attempted + + async def intercept_stream_unary( + self, continuation, client_call_details, request_iterator + ): + rpc_method = _extract_rpc_method(getattr(client_call_details, "method", None)) + logical_operation = _extract_logical_operation(rpc_method) + op_state = _opentelemetry_metrics.get_current_operation() + start_time = time.perf_counter() + try: + call = await continuation(client_call_details, request_iterator) + except BaseException as exc: + duration = max(0.0, time.perf_counter() - start_time) + status_code, error_type = _resolve_status_and_error(exc) + resolved_path = _resolve_network_path( + exc=exc, + direct_path_attempted=self._direct_path_attempted, + received_server_response=False, + ) + self._recorder.record_rpc_attempt( + duration=duration, + operation=logical_operation, + status_code=status_code, + error_type=error_type, + rpc_method=rpc_method, + network_path=resolved_path, + op_state=op_state, + ) + raise + return _TrackedStreamUnaryCall( + call, + self._recorder, + rpc_method, + logical_operation, + start_time, + op_state, + direct_path_attempted=self._direct_path_attempted, + ) + + +class _MetricsStreamStreamInterceptor(grpc.aio.StreamStreamClientInterceptor): + """Client interceptor recording metrics for stream-stream gRPC attempts.""" + + def __init__( + self, + recorder: _opentelemetry_metrics._MetricsRecorder, + direct_path_attempted: bool = False, + ) -> None: + self._recorder = recorder + self._direct_path_attempted = direct_path_attempted + + async def intercept_stream_stream( + self, continuation, client_call_details, request_iterator + ): + rpc_method = _extract_rpc_method(getattr(client_call_details, "method", None)) + logical_operation = _extract_logical_operation(rpc_method) + op_state = _opentelemetry_metrics.get_current_operation() + start_time = time.perf_counter() + try: + call = await continuation(client_call_details, request_iterator) + except BaseException as exc: + duration = max(0.0, time.perf_counter() - start_time) + status_code, error_type = _resolve_status_and_error(exc) + resolved_path = _resolve_network_path( + exc=exc, + direct_path_attempted=self._direct_path_attempted, + received_server_response=False, + ) + self._recorder.record_rpc_attempt( + duration=duration, + operation=logical_operation, + status_code=status_code, + error_type=error_type, + rpc_method=rpc_method, + network_path=resolved_path, + op_state=op_state, + ) + raise + return _TrackedStreamStreamCall( + call, + self._recorder, + rpc_method, + logical_operation, + start_time, + op_state, + direct_path_attempted=self._direct_path_attempted, + ) + + +def _create_metrics_interceptors( + recorder: _opentelemetry_metrics._MetricsRecorder, + direct_path_attempted: bool = False, +) -> Tuple[grpc.aio.ClientInterceptor, ...]: + """Creates the 4 ``grpc.aio`` client interceptors for ``recorder``.""" + return ( + _MetricsUnaryUnaryInterceptor( + recorder, direct_path_attempted=direct_path_attempted + ), + _MetricsUnaryStreamInterceptor( + recorder, direct_path_attempted=direct_path_attempted + ), + _MetricsStreamUnaryInterceptor( + recorder, direct_path_attempted=direct_path_attempted + ), + _MetricsStreamStreamInterceptor( + recorder, direct_path_attempted=direct_path_attempted + ), + ) + def _validate_metadata(metadata): """Validates that metadata is a sequence of (key, value) pairs.""" @@ -86,6 +877,12 @@ class AsyncGrpcClient: metrics. If ``None``, falls back to the ``GCP_STORAGE_PYTHON_ENABLE_OTEL_DEBUG_METRICS`` environment variable, or ``False`` if unset. + + :type meter_provider: :class:`~opentelemetry.metrics.MeterProvider` or None + :param meter_provider: + (Optional, Experimental) A custom OpenTelemetry ``MeterProvider`` + instance used to create metric instruments. If ``None``, falls back to + the globally registered OpenTelemetry ``MeterProvider``. """ def __init__( @@ -98,9 +895,11 @@ def __init__( attempt_direct_path_xds_over_interconnect=False, enable_metrics=None, enable_debug_metrics=None, + meter_provider=None, ): self._enable_metrics = enable_metrics self._enable_debug_metrics = enable_debug_metrics + self._meter_provider = meter_provider self._metrics_enabled = _opentelemetry_metrics.is_metrics_enabled( enable_metrics ) @@ -108,6 +907,20 @@ def __init__( enable_debug_metrics ) + api_endpoint = None + if isinstance(client_options, dict): + api_endpoint = client_options.get("api_endpoint") + elif client_options is not None: + api_endpoint = getattr(client_options, "api_endpoint", None) + + self._metrics_recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=enable_metrics, + enable_debug_metrics=enable_debug_metrics, + meter_provider=meter_provider, + api_endpoint=api_endpoint, + rpc_system="grpc", + ) + if isinstance(credentials, auth_credentials.AnonymousCredentials): if client_options is None or client_options.api_endpoint is None: raise ValueError( @@ -116,6 +929,7 @@ def __init__( self._grpc_client = self._create_anonymous_client( client_options, credentials ) + self._grpc_client._metrics_recorder = self._metrics_recorder return if client_info is None: @@ -138,6 +952,7 @@ def __init__( attempt_direct_path=attempt_direct_path, attempt_direct_path_xds_over_interconnect=use_dp_interconnect, ) + self._grpc_client._metrics_recorder = self._metrics_recorder @property def metrics_enabled(self) -> bool: @@ -150,7 +965,15 @@ def debug_metrics_enabled(self) -> bool: return self._debug_metrics_enabled def _create_anonymous_client(self, client_options, credentials): - channel = grpc.aio.insecure_channel(client_options.api_endpoint) + if self._metrics_recorder.standard_enabled: + channel = grpc.aio.insecure_channel( + client_options.api_endpoint, + interceptors=_create_metrics_interceptors( + self._metrics_recorder, direct_path_attempted=False + ), + ) + else: + channel = grpc.aio.insecure_channel(client_options.api_endpoint) transport = storage_v2.services.storage.transports.StorageGrpcAsyncIOTransport( channel=channel, credentials=credentials ) @@ -187,6 +1010,16 @@ def _create_async_grpc_client( host = getattr(client_options, "api_endpoint", None) or _DEFAULT_HOST quota_project_id = getattr(client_options, "quota_project_id", None) + extra_channel_kwargs = {} + if self._metrics_recorder.standard_enabled: + direct_path_attempted = bool( + attempt_direct_path or attempt_direct_path_xds_over_interconnect + ) + extra_channel_kwargs["interceptors"] = _create_metrics_interceptors( + self._metrics_recorder, + direct_path_attempted=direct_path_attempted, + ) + if attempt_direct_path_xds_over_interconnect: host = _rewrite_host_for_interconnect(host) channel = transport_cls.create_channel( @@ -196,6 +1029,7 @@ def _create_async_grpc_client( attempt_direct_path_xds_over_interconnect=True, credentials=credentials, options=(("grpc.primary_user_agent", primary_user_agent),), + **extra_channel_kwargs, ) else: channel = transport_cls.create_channel( @@ -204,6 +1038,7 @@ def _create_async_grpc_client( attempt_direct_path=attempt_direct_path, credentials=credentials, options=(("grpc.primary_user_agent", primary_user_agent),), + **extra_channel_kwargs, ) transport = transport_cls(channel=channel) @@ -213,6 +1048,19 @@ def _create_async_grpc_client( client_options=client_options, ) + async def close(self) -> None: + """Closes the underlying gRPC transport and flushes pending metrics.""" + try: + await self._grpc_client.transport.close() + finally: + self._metrics_recorder.shutdown() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + @property def grpc_client(self): """The underlying gRPC client. @@ -276,25 +1124,32 @@ async def delete_object( :type retry: :class:`~google.api_core.retry_async.AsyncRetry` or None :param retry: (Optional) Designation of what errors, if any, should be retried. """ - _validate_metadata(metadata) - # The gRPC API requires the bucket name to be in the format "projects/_/buckets/bucket_name" - bucket_path = f"projects/_/buckets/{bucket_name}" - request = storage_v2.DeleteObjectRequest( - bucket=bucket_path, - object=object_name, - generation=generation, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - **kwargs, - ) - await self._grpc_client.delete_object( - request=request, - metadata=metadata, - timeout=timeout, - retry=retry, - ) + recorder = _opentelemetry_metrics.get_active_recorder(self) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncGrpcClient.delete_object", + operation="DeleteObject", + rpc_method="google.storage.v2.Storage/DeleteObject", + ): + _validate_metadata(metadata) + # The gRPC API requires the bucket name to be in the format "projects/_/buckets/bucket_name" + bucket_path = f"projects/_/buckets/{bucket_name}" + request = storage_v2.DeleteObjectRequest( + bucket=bucket_path, + object=object_name, + generation=generation, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + **kwargs, + ) + await self._grpc_client.delete_object( + request=request, + metadata=metadata, + timeout=timeout, + retry=retry, + ) async def get_object( self, @@ -356,25 +1211,32 @@ async def get_object( :rtype: :class:`google.cloud._storage_v2.types.Object` :returns: The object metadata resource. """ - _validate_metadata(metadata) - bucket_path = f"projects/_/buckets/{bucket_name}" - - request = storage_v2.GetObjectRequest( - bucket=bucket_path, - object=object_name, - generation=generation, - if_generation_match=if_generation_match, - if_generation_not_match=if_generation_not_match, - if_metageneration_match=if_metageneration_match, - if_metageneration_not_match=if_metageneration_not_match, - soft_deleted=soft_deleted or False, - **kwargs, - ) + recorder = _opentelemetry_metrics.get_active_recorder(self) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncGrpcClient.get_object", + operation="GetObject", + rpc_method="google.storage.v2.Storage/GetObject", + ): + _validate_metadata(metadata) + bucket_path = f"projects/_/buckets/{bucket_name}" - # Calls the underlying GAPIC StorageAsyncClient.get_object method - return await self._grpc_client.get_object( - request=request, - metadata=metadata, - timeout=timeout, - retry=retry, - ) + request = storage_v2.GetObjectRequest( + bucket=bucket_path, + object=object_name, + generation=generation, + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + soft_deleted=soft_deleted or False, + **kwargs, + ) + + # Calls the underlying GAPIC StorageAsyncClient.get_object method + return await self._grpc_client.get_object( + request=request, + metadata=metadata, + timeout=timeout, + retry=retry, + ) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py index 32757bf754b7..060fcf923435 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py @@ -23,6 +23,7 @@ from google.api_core.retry_async import AsyncRetry from google.cloud import _storage_v2 +from google.cloud.storage import _opentelemetry_metrics from google.cloud.storage._helpers import generate_random_56_bit_integer from google.cloud.storage.asyncio._stream_multiplexer import ( _StreamEnd, @@ -31,6 +32,8 @@ ) from google.cloud.storage.asyncio.async_grpc_client import ( AsyncGrpcClient, + _bind_call_to_operation, + _resolve_network_path, ) from google.cloud.storage.asyncio.async_read_object_stream import ( _AsyncReadObjectStream, @@ -48,6 +51,7 @@ from ._utils import raise_if_no_fast_crc32c _MAX_READ_RANGES_PER_BIDI_READ_REQUEST = 100 +_BIDI_READ_OBJECT_RPC = "google.storage.v2.Storage/BidiReadObject" _COMMON_READ_RETRYABLE_EXCEPTIONS = ( exceptions.InternalServerError, exceptions.ServiceUnavailable, @@ -151,16 +155,23 @@ async def create_mrd( :rtype: :class:`~google.cloud.storage.asyncio.async_multi_range_downloader.AsyncMultiRangeDownloader` :returns: An initialized AsyncMultiRangeDownloader instance for reading. """ - mrd = cls( - client, - bucket_name, - object_name, - generation=generation, - read_handle=read_handle, - **kwargs, - ) - await mrd.open(retry_policy=retry_policy, metadata=metadata) - return mrd + recorder = _opentelemetry_metrics.get_active_recorder(client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.create_mrd", + operation="ReadObject", + rpc_method=_BIDI_READ_OBJECT_RPC, + ): + mrd = cls( + client, + bucket_name, + object_name, + generation=generation, + read_handle=read_handle, + **kwargs, + ) + await mrd.open(retry_policy=retry_policy, metadata=metadata) + return mrd def __init__( self, @@ -226,6 +237,21 @@ async def __aexit__(self, exc_type, exc_val, exc_tb): if self.is_stream_open: await self.close() + def _current_network_path( + self, + exc: Optional[BaseException] = None, + received_server_response: bool = True, + ) -> str: + """Resolves ``network.path`` from the underlying Bidi read stream call.""" + call = getattr( + getattr(self.read_obj_str, "socket_like_rpc", None), "call", None + ) + return _resolve_network_path( + call=call, + exc=exc, + received_server_response=received_server_response, + ) + def _on_open_error(self, exc): """Extracts routing token and read handle on redirect error during open.""" logger.warning(f"Error occurred while opening MRD: {exc}") @@ -241,84 +267,122 @@ async def open( metadata: Optional[List[Tuple[str, str]]] = None, ) -> None: """Opens the bidi-gRPC connection to read from the object.""" - if self._is_stream_open: - raise ValueError("Underlying bidi-gRPC stream is already open") - - if retry_policy is None: - - def on_error_wrapper(exc): - self._open_retries += 1 - self._on_open_error(exc) - - retry_policy = AsyncRetry( - predicate=_is_open_retryable, on_error=on_error_wrapper - ) - else: - original_on_error = retry_policy._on_error - - def combined_on_error(exc): - self._open_retries += 1 - self._on_open_error(exc) - if original_on_error: - original_on_error(exc) - - retry_policy = AsyncRetry( - predicate=_is_open_retryable, - initial=retry_policy._initial, - maximum=retry_policy._maximum, - multiplier=retry_policy._multiplier, - deadline=retry_policy._deadline, - on_error=combined_on_error, - ) - - async def _do_open(): - current_metadata = list(metadata) if metadata else [] - - # Cleanup stream from previous failed attempt, if any. - if self.read_obj_str: - if self.read_obj_str.is_stream_open: - try: - await self.read_obj_str.close() - except exceptions.GoogleAPICallError as e: - logger.warning( - f"Failed to close existing stream during resumption: {e}" - ) - self.read_obj_str = None - self._is_stream_open = False + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.open", + operation="ReadObject", + rpc_method=_BIDI_READ_OBJECT_RPC, + ): + if self._is_stream_open: + raise ValueError("Underlying bidi-gRPC stream is already open") + + if retry_policy is None: + + def on_error_wrapper(exc): + self._open_retries += 1 + self._on_open_error(exc) + + retry_policy = AsyncRetry( + predicate=_is_open_retryable, on_error=on_error_wrapper + ) + else: + original_on_error = retry_policy._on_error + + def combined_on_error(exc): + self._open_retries += 1 + self._on_open_error(exc) + if original_on_error: + original_on_error(exc) + + retry_policy = AsyncRetry( + predicate=_is_open_retryable, + initial=retry_policy._initial, + maximum=retry_policy._maximum, + multiplier=retry_policy._multiplier, + deadline=retry_policy._deadline, + on_error=combined_on_error, + ) - self.read_obj_str = _AsyncReadObjectStream( - client=self.client.grpc_client, - bucket_name=self.bucket_name, - object_name=self.object_name, - generation_number=self.generation, - read_handle=self.read_handle, - ) + async def _do_open(): + current_op = _opentelemetry_metrics.get_current_operation() + attempts_before = ( + current_op.recorded_attempts if current_op is not None else 0 + ) + current_metadata = list(metadata) if metadata else [] - if self._routing_token: - current_metadata.append( - ("x-goog-request-params", f"routing_token={self._routing_token}") + # Cleanup stream from previous failed attempt, if any. + if self.read_obj_str: + if self.read_obj_str.is_stream_open: + try: + await self.read_obj_str.close() + except exceptions.GoogleAPICallError as e: + logger.warning( + f"Failed to close existing stream during resumption: {e}" + ) + self.read_obj_str = None + self._is_stream_open = False + + self.read_obj_str = _AsyncReadObjectStream( + client=self.client.grpc_client, + bucket_name=self.bucket_name, + object_name=self.object_name, + generation_number=self.generation, + read_handle=self.read_handle, ) - self._routing_token = None + if recorder is not None: + self.read_obj_str._metrics_recorder = recorder + + if self._routing_token: + current_metadata.append( + ( + "x-goog-request-params", + f"routing_token={self._routing_token}", + ) + ) + self._routing_token = None - await self.read_obj_str.open( - metadata=current_metadata if current_metadata else None - ) + try: + await self.read_obj_str.open( + metadata=current_metadata if current_metadata else None + ) + except Exception as exc: + if recorder is not None: + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + exc, + attempts_before=attempts_before, + will_retry=bool(retry_policy._predicate(exc)), + network_path=self._current_network_path( + exc=exc, received_server_response=False + ), + ) + raise + + if recorder is not None: + _opentelemetry_metrics.record_pending_retry( + recorder, + operation="ReadObject", + network_path=self._current_network_path( + received_server_response=True + ), + ) - if self.read_obj_str.generation_number: - self.generation = self.read_obj_str.generation_number - if self.read_obj_str.read_handle: - self.read_handle = self.read_obj_str.read_handle - if self.read_obj_str.persisted_size is not None: - self.persisted_size = self.read_obj_str.persisted_size - self.is_finalized = self.read_obj_str.is_finalized - self.full_obj_server_crc32c = self.read_obj_str.full_obj_server_crc32c + if self.read_obj_str.generation_number: + self.generation = self.read_obj_str.generation_number + if self.read_obj_str.read_handle: + self.read_handle = self.read_obj_str.read_handle + if self.read_obj_str.persisted_size is not None: + self.persisted_size = self.read_obj_str.persisted_size + self.is_finalized = self.read_obj_str.is_finalized + self.full_obj_server_crc32c = self.read_obj_str.full_obj_server_crc32c - self._is_stream_open = True + self._is_stream_open = True - await retry_policy(_do_open)() - self._multiplexer = _StreamMultiplexer(self.read_obj_str) + await retry_policy(_do_open)() + self._multiplexer = _StreamMultiplexer(self.read_obj_str) - def _create_stream_factory(self, state, metadata): + def _create_stream_factory(self, state, metadata, recorder=None): """Create a factory that opens a new stream with current routing state.""" async def factory(): @@ -332,6 +396,8 @@ async def factory(): generation_number=self.generation, read_handle=current_handle, ) + if recorder is not None: + stream._metrics_recorder = recorder current_metadata = list(metadata) if metadata else [] if current_token: @@ -401,147 +467,185 @@ async def download_ranges( """ - if len(read_ranges) > 1000: - raise ValueError( - "Invalid input - length of read_ranges cannot be more than 1000" - ) + recorder = _opentelemetry_metrics.get_active_recorder(self.client) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + operation="ReadObject", + rpc_method=_BIDI_READ_OBJECT_RPC, + ): + if len(read_ranges) > 1000: + raise ValueError( + "Invalid input - length of read_ranges cannot be more than 1000" + ) - if enable_checksum: - raise_if_no_fast_crc32c() + if enable_checksum: + raise_if_no_fast_crc32c() - if not self._is_stream_open: - raise ValueError("Underlying bidi-gRPC stream is not open") + if not self._is_stream_open: + raise ValueError("Underlying bidi-gRPC stream is not open") - if retry_policy is None: - retry_policy = AsyncRetry(predicate=_is_read_retryable) - - # Initialize Global State for Retry Strategy - download_states = {} - for read_range in read_ranges: - read_id = generate_random_56_bit_integer() - # Unpack tuple into self-documenting variable names to improve readability. - offset, length, user_buffer = read_range - - # Heuristic to detect full object reads: - # - Implicit full object read: start offset is 0 and length is 0 (read all). - # - Explicit full object read: start offset is 0 and length matches the exact persisted size. - is_full_object_read = (offset == 0 and length == 0) or ( - self.persisted_size is not None - and offset == 0 - and length == self.persisted_size - ) - download_states[read_id] = _DownloadState( - initial_offset=offset, - initial_length=length, - user_buffer=user_buffer, - is_full_object_read=is_full_object_read, + _bind_call_to_operation( + getattr( + getattr(self.read_obj_str, "socket_like_rpc", None), "call", None + ), + _opentelemetry_metrics.get_current_operation(), ) - initial_state = { - "download_states": download_states, - "read_handle": self.read_handle, - "routing_token": None, - "enable_checksum": enable_checksum, - "full_obj_server_crc32c": self.full_obj_server_crc32c, - } - - read_ids = set(download_states.keys()) - queue = self._multiplexer.register(read_ids) - - try: - attempt_count = 0 - last_broken_generation = None - - def send_and_recv_via_multiplexer( - requests: List[_storage_v2.ReadRange], - state: Dict[str, Any], - ): - async def generator(): - nonlocal attempt_count, last_broken_generation - attempt_count += 1 - - if attempt_count > 1: - logger.info( - f"Resuming download (attempt {attempt_count}) for {len(requests)} ranges." - ) + if retry_policy is None: + retry_policy = AsyncRetry(predicate=_is_read_retryable) + + # Initialize Global State for Retry Strategy + download_states = {} + for read_range in read_ranges: + read_id = generate_random_56_bit_integer() + # Unpack tuple into self-documenting variable names to improve readability. + offset, length, user_buffer = read_range + + # Heuristic to detect full object reads: + # - Implicit full object read: start offset is 0 and length is 0 (read all). + # - Explicit full object read: start offset is 0 and length matches the exact persisted size. + is_full_object_read = (offset == 0 and length == 0) or ( + self.persisted_size is not None + and offset == 0 + and length == self.persisted_size + ) + download_states[read_id] = _DownloadState( + initial_offset=offset, + initial_length=length, + user_buffer=user_buffer, + is_full_object_read=is_full_object_read, + ) - # Reopen stream if needed - should_reopen = ( - attempt_count > 1 and last_broken_generation is not None - ) or (attempt_count == 1 and metadata is not None) - if should_reopen: - broken_gen = ( - last_broken_generation - if attempt_count > 1 - else self._multiplexer.stream_generation - ) - stream_factory = self._create_stream_factory(state, metadata) - await self._multiplexer.reopen_stream( - broken_gen, stream_factory - ) + initial_state = { + "download_states": download_states, + "read_handle": self.read_handle, + "routing_token": None, + "enable_checksum": enable_checksum, + "full_obj_server_crc32c": self.full_obj_server_crc32c, + } - stream_generation = self._multiplexer.stream_generation + read_ids = set(download_states.keys()) + queue = self._multiplexer.register(read_ids) - # Send Requests - pending_read_ids = {r.read_id for r in requests} - for i in range( - 0, len(requests), _MAX_READ_RANGES_PER_BIDI_READ_REQUEST - ): - batch = requests[i : i + _MAX_READ_RANGES_PER_BIDI_READ_REQUEST] - try: - await self._multiplexer.send( - _storage_v2.BidiReadObjectRequest(read_ranges=batch) + try: + attempt_count = 0 + last_broken_generation = None + + def send_and_recv_via_multiplexer( + requests: List[_storage_v2.ReadRange], + state: Dict[str, Any], + ): + async def generator(): + nonlocal attempt_count, last_broken_generation + attempt_count += 1 + + if attempt_count > 1: + logger.info( + f"Resuming download (attempt {attempt_count}) for {len(requests)} ranges." ) - except Exception: - last_broken_generation = stream_generation - raise - # Receive Responses - while pending_read_ids: - item = await queue.get() + # Reopen stream if needed + should_reopen = ( + attempt_count > 1 and last_broken_generation is not None + ) or (attempt_count == 1 and metadata is not None) + if should_reopen: + if ( + attempt_count == 1 + and metadata is not None + and self.read_obj_str is not None + and hasattr(self.read_obj_str, "_mark_closing_cleanly") + ): + self.read_obj_str._mark_closing_cleanly() + broken_gen = ( + last_broken_generation + if attempt_count > 1 + else self._multiplexer.stream_generation + ) + stream_factory = self._create_stream_factory( + state, metadata, recorder=recorder + ) + await self._multiplexer.reopen_stream( + broken_gen, stream_factory + ) - if isinstance(item, _StreamEnd): - if pending_read_ids: - last_broken_generation = stream_generation - raise exceptions.ServiceUnavailable( - "Stream ended with pending read_ids" + stream_generation = self._multiplexer.stream_generation + + # Send Requests + pending_read_ids = {r.read_id for r in requests} + for i in range( + 0, len(requests), _MAX_READ_RANGES_PER_BIDI_READ_REQUEST + ): + batch = requests[ + i : i + _MAX_READ_RANGES_PER_BIDI_READ_REQUEST + ] + try: + await self._multiplexer.send( + _storage_v2.BidiReadObjectRequest(read_ranges=batch) ) - break - - if isinstance(item, _StreamError): - if item.generation < stream_generation: - continue # stale error, skip - last_broken_generation = item.generation - raise item.exception - - # Track completion - if item.object_data_ranges: - for data_range in item.object_data_ranges: - if data_range.range_end: - pending_read_ids.discard( - data_range.read_range.read_id + except Exception: + last_broken_generation = stream_generation + raise + + # Receive Responses + while pending_read_ids: + item = await queue.get() + + if isinstance(item, _StreamEnd): + if pending_read_ids: + last_broken_generation = stream_generation + raise exceptions.ServiceUnavailable( + "Stream ended with pending read_ids" ) - yield item + break + + if isinstance(item, _StreamError): + if item.generation < stream_generation: + continue # stale error, skip + last_broken_generation = item.generation + raise item.exception + + # Track completion + if item.object_data_ranges: + for data_range in item.object_data_ranges: + if data_range.range_end: + pending_read_ids.discard( + data_range.read_range.read_id + ) + yield item + + return generator() + + strategy = _ReadResumptionStrategy() + retry_manager = _BidiStreamRetryManager( + strategy, + send_and_recv_via_multiplexer, + recorder=recorder, + network_path_getter=lambda: self._current_network_path( + received_server_response=True + ), + ) - return generator() + try: + await retry_manager.execute(initial_state, retry_policy) + except DataCorruption: + if self.is_stream_open: + await self.close() + raise - strategy = _ReadResumptionStrategy() - retry_manager = _BidiStreamRetryManager( - strategy, send_and_recv_via_multiplexer - ) + if recorder is not None: + total_bytes = sum( + ds.bytes_written for ds in download_states.values() + ) + recorder.record_response_body_size( + total_bytes, operation="ReadObject" + ) - try: - await retry_manager.execute(initial_state, retry_policy) - except DataCorruption: - if self.is_stream_open: - await self.close() - raise - - if initial_state.get("read_handle"): - self.read_handle = initial_state["read_handle"] - finally: - if self._multiplexer is not None: - self._multiplexer.unregister(read_ids) + if initial_state.get("read_handle"): + self.read_handle = initial_state["read_handle"] + finally: + if self._multiplexer is not None: + self._multiplexer.unregister(read_ids) async def close(self): """ diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py index d3c0780673c7..2850dcf52236 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_read_object_stream.py @@ -14,15 +14,20 @@ import asyncio import logging -from typing import List, Optional, Tuple +from typing import Any, List, Optional, Tuple from google.api_core.bidi_async import AsyncBidiRpc from google.cloud import _storage_v2 +from google.cloud.storage import _opentelemetry_metrics from google.cloud.storage.asyncio.async_abstract_object_stream import ( _AsyncAbstractObjectStream, ) -from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_grpc_client import ( + AsyncGrpcClient, + _mark_call_closing_cleanly, + _resolve_network_path, +) logger = logging.getLogger(__name__) @@ -58,6 +63,7 @@ def __init__( object_name: str, generation_number: Optional[int] = None, read_handle: Optional[_storage_v2.BidiReadHandle] = None, + metrics_recorder: Optional[Any] = None, ) -> None: if client is None: raise ValueError("client must be provided") @@ -73,6 +79,11 @@ def __init__( ) self.client: AsyncGrpcClient.grpc_client = client self.read_handle: Optional[_storage_v2.BidiReadHandle] = read_handle + self._metrics_recorder = ( + metrics_recorder + if metrics_recorder is not None + else _opentelemetry_metrics.get_active_recorder(client) + ) self._full_bucket_name = f"projects/_/buckets/{self.bucket_name}" @@ -156,11 +167,29 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: if response and response.read_handle: self.read_handle = response.read_handle + if self._metrics_recorder is not None: + _opentelemetry_metrics.record_pending_retry( + self._metrics_recorder, + operation="ReadObject", + network_path=self._get_network_path(received_server_response=True), + ) + self._is_stream_open = True except (asyncio.CancelledError, Exception): await self._close_socket_like_rpc() raise + def _get_network_path(self, received_server_response: bool = True) -> str: + """Resolves ``network.path`` for the active gRPC stream call.""" + return _resolve_network_path( + call=getattr(self.socket_like_rpc, "call", None), + received_server_response=received_server_response, + ) + + def _mark_closing_cleanly(self) -> None: + """Marks the underlying gRPC call as closing cleanly for metrics.""" + _mark_call_closing_cleanly(getattr(self.socket_like_rpc, "call", None)) + async def _close_socket_like_rpc(self) -> None: try: await self.socket_like_rpc.close() @@ -174,6 +203,7 @@ async def close(self) -> None: try: if self.socket_like_rpc.is_active: await self.requests_done() + self._mark_closing_cleanly() finally: self._is_stream_open = False await self._close_socket_like_rpc() diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py index 2d78a8c20fd1..882c5df59a29 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_write_object_stream.py @@ -12,18 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional, Tuple +import asyncio +from typing import Any, List, Optional, Tuple import grpc from google.api_core.bidi_async import AsyncBidiRpc from google.cloud import _storage_v2 -from google.cloud.storage import Blob, _grpc_conversions +from google.cloud.storage import Blob, _grpc_conversions, _opentelemetry_metrics from google.cloud.storage.asyncio import _utils from google.cloud.storage.asyncio.async_abstract_object_stream import ( _AsyncAbstractObjectStream, ) -from google.cloud.storage.asyncio.async_grpc_client import AsyncGrpcClient +from google.cloud.storage.asyncio.async_grpc_client import ( + AsyncGrpcClient, + _mark_call_closing_cleanly, + _resolve_network_path, +) class _AsyncWriteObjectStream(_AsyncAbstractObjectStream): @@ -73,6 +78,7 @@ def __init__( routing_token: Optional[str] = None, blob: Optional[Blob] = None, storage_class: Optional[str] = None, + metrics_recorder: Optional[Any] = None, ) -> None: if client is None: raise ValueError("client must be provided") @@ -91,6 +97,11 @@ def __init__( self.routing_token: Optional[str] = routing_token self.blob: Optional[Blob] = blob self.storage_class: Optional[str] = storage_class + self._metrics_recorder = ( + metrics_recorder + if metrics_recorder is not None + else _opentelemetry_metrics.get_active_recorder(client) + ) self._full_bucket_name = f"projects/_/buckets/{self.bucket_name}" self.rpc = self.client._client._transport._wrapped_methods[ @@ -168,8 +179,19 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: metadata=final_metadata, ) - await self.socket_like_rpc.open() # this is actually 1 send - response = await self.socket_like_rpc.recv() + try: + await self.socket_like_rpc.open() # this is actually 1 send + response = await self.socket_like_rpc.recv() + except (asyncio.CancelledError, Exception): + await self._close_socket_like_rpc() + raise + if self._metrics_recorder is not None: + net_path = self._get_network_path(received_server_response=True) + _opentelemetry_metrics.record_pending_retry( + self._metrics_recorder, + operation="WriteObject", + network_path=net_path, + ) self._is_stream_open = True if response.persisted_size: @@ -187,13 +209,33 @@ async def open(self, metadata: Optional[List[Tuple[str, str]]] = None) -> None: if response.write_handle: self.write_handle = response.write_handle + async def _close_socket_like_rpc(self) -> None: + try: + await self.socket_like_rpc.close() + except Exception: + pass + + def _get_network_path(self, received_server_response: bool = True) -> str: + """Resolves ``network.path`` for the active gRPC stream call.""" + return _resolve_network_path( + call=getattr(self.socket_like_rpc, "call", None), + received_server_response=received_server_response, + ) + + def _mark_closing_cleanly(self) -> None: + """Marks the underlying gRPC call as closing cleanly for metrics.""" + _mark_call_closing_cleanly(getattr(self.socket_like_rpc, "call", None)) + async def close(self) -> None: """Closes the bidi-gRPC connection.""" if not self._is_stream_open: raise ValueError("Stream is not open") - await self.requests_done() - await self.socket_like_rpc.close() - self._is_stream_open = False + try: + await self.requests_done() + self._mark_closing_cleanly() + finally: + self._is_stream_open = False + await self.socket_like_rpc.close() async def requests_done(self): """Signals that all requests have been sent.""" diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/retry/bidi_stream_retry_manager.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/retry/bidi_stream_retry_manager.py index 947ee74c1183..9bee357674d7 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/retry/bidi_stream_retry_manager.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/retry/bidi_stream_retry_manager.py @@ -13,8 +13,9 @@ # limitations under the License. import logging -from typing import Any, AsyncIterator, Callable +from typing import Any, AsyncIterator, Callable, Optional +from google.cloud.storage import _opentelemetry_metrics from google.cloud.storage.asyncio.retry.base_strategy import ( _BaseResumptionStrategy, ) @@ -29,15 +30,21 @@ def __init__( self, strategy: _BaseResumptionStrategy, send_and_recv: Callable[..., AsyncIterator[Any]], + recorder: Optional[Any] = None, + network_path_getter: Optional[Callable[[], Optional[str]]] = None, ): """Initializes the retry manager. Args: strategy: The strategy for managing the state of a specific bidi operation (e.g., reads or writes). send_and_recv: An async callable that opens a new gRPC stream. + recorder: Optional `_MetricsRecorder` for retry metric accounting. + network_path_getter: Optional callable returning the current ``network.path``. """ self._strategy = strategy self._send_and_recv = send_and_recv + self._recorder = recorder + self._network_path_getter = network_path_getter async def execute(self, initial_state: Any, retry_policy): """ @@ -50,14 +57,38 @@ async def execute(self, initial_state: Any, retry_policy): state = initial_state async def attempt(): + current_op = _opentelemetry_metrics.get_current_operation() + attempts_before = current_op.recorded_attempts if current_op else 0 requests = self._strategy.generate_requests(state) stream = self._send_and_recv(requests, state) try: async for response in stream: self._strategy.update_state_from_response(response, state) + if self._recorder is not None: + _opentelemetry_metrics.record_pending_retry( + self._recorder, + network_path=( + self._network_path_getter() + if self._network_path_getter is not None + else None + ), + ) return except Exception as e: - if retry_policy._predicate(e): + will_retry = bool(retry_policy._predicate(e)) + if self._recorder is not None: + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + self._recorder, + e, + attempts_before=attempts_before, + will_retry=will_retry, + network_path=( + self._network_path_getter() + if self._network_path_getter is not None + else None + ), + ) + if will_retry: logger.warning( f"Bidi stream operation failed: {e}. Attempting state recovery and retry." ) diff --git a/packages/google-cloud-storage/tests/unit/asyncio/retry/test_bidi_stream_retry_manager.py b/packages/google-cloud-storage/tests/unit/asyncio/retry/test_bidi_stream_retry_manager.py index 71b60ed28ec7..175d9f4f1c6e 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/retry/test_bidi_stream_retry_manager.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/retry/test_bidi_stream_retry_manager.py @@ -154,3 +154,81 @@ async def mock_send_and_recv(*args, **kwargs): ) mock_strategy.recover_state_on_failure.assert_not_called() + + @pytest.mark.asyncio + async def test_execute_records_retry_metrics_with_and_without_network_path_getter( + self, monkeypatch + ): + from google.cloud.storage import _opentelemetry_metrics + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr( + _opentelemetry_metrics, "metrics", mock_metrics, raising=False + ) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + mock_strategy = mock.AsyncMock(spec=base_strategy._BaseResumptionStrategy) + attempt_count = 0 + + async def mock_send_and_recv(*args, **kwargs): + nonlocal attempt_count + attempt_count += 1 + if attempt_count == 1: + raise exceptions.ServiceUnavailable("503") + yield "ok" + + retry_manager = manager._BidiStreamRetryManager( + strategy=mock_strategy, + send_and_recv=mock_send_and_recv, + recorder=recorder, + network_path_getter=lambda: "directpath", + ) + retry_policy = AsyncRetry(predicate=_is_retriable, initial=0.001) + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ): + with mock.patch("asyncio.sleep", new_callable=mock.AsyncMock): + await retry_manager.execute(initial_state={}, retry_policy=retry_policy) + + recorder.retries_counter.add.assert_called_once() + _, retry_kw = recorder.retries_counter.add.call_args + assert retry_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert retry_kw["attributes"]["error.type"] == "UNAVAILABLE" + assert retry_kw["attributes"]["network.path"] == "directpath" + + # Without network_path_getter on non-retryable failure + recorder.retries_counter.add.reset_mock() + + async def fail_send_and_recv(*args, **kwargs): + if False: + yield + raise exceptions.PermissionDenied("403") + + retry_manager_no_getter = manager._BidiStreamRetryManager( + strategy=mock_strategy, + send_and_recv=fail_send_and_recv, + recorder=recorder, + network_path_getter=None, + ) + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ): + with pytest.raises(exceptions.PermissionDenied): + await retry_manager_no_getter.execute( + initial_state={}, retry_policy=DEFAULT_TEST_RETRY + ) + recorder.retries_counter.add.assert_not_called() diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py index e6fcd90f0b84..48a4e06163ff 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_appendable_object_writer.py @@ -870,3 +870,193 @@ async def test_close_retry_on_redirect_error(self, mock_appendable_writer): assert writer.write_handle.handle == b"h2" writer.open.assert_awaited_once() assert not writer._is_stream_open + + @pytest.mark.asyncio + async def test_writer_metrics_recording(self, mock_appendable_writer): + mock_recorder = mock.MagicMock() + mock_recorder.standard_enabled = True + mock_client = mock_appendable_writer["mock_client"] + mock_client._metrics_recorder = mock_recorder + mock_stream = mock_appendable_writer["mock_stream"] + + writer = self._make_one(mock_client) + await writer.open() + assert mock_stream._metrics_recorder is mock_recorder + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncAppendableObjectWriter.open" + ) + + # Direct append records request_body_size once + mock_recorder.record_logical_operation.reset_mock() + await writer.append(b"hello") + mock_recorder.record_request_body_size.assert_called_once_with( + 5, operation="WriteObject" + ) + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncAppendableObjectWriter.append" + ) + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["operation"] + == "WriteObject" + ) + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["rpc_method"] + == "google.storage.v2.Storage/BidiWriteObject" + ) + + # append_from_file suppresses per-chunk append metrics and records total size once + mock_recorder.record_logical_operation.reset_mock() + mock_recorder.record_request_body_size.reset_mock() + await writer.append_from_file(io.BytesIO(b"abcdefghij"), block_size=4) + mock_recorder.record_request_body_size.assert_called_once_with( + 10, operation="WriteObject" + ) + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncAppendableObjectWriter.append_from_file" + ) + + # state_lookup, simple_flush, flush, close(finalize_on_close=True) + for method_name, call_coro in ( + ("state_lookup", writer.state_lookup()), + ("simple_flush", writer.simple_flush()), + ("flush", writer.flush()), + ): + mock_recorder.record_logical_operation.reset_mock() + await call_coro + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == f"storage.asyncio.AsyncAppendableObjectWriter.{method_name}" + ) + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["operation"] + == "WriteObject" + ) + + mock_recorder.record_logical_operation.reset_mock() + mock_stream.recv.return_value = storage_type.BidiWriteObjectResponse( + resource=storage_type.Object(size=15) + ) + await writer.close(finalize_on_close=True) + # close(finalize_on_close=True) records outer close without duplicating finalize + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncAppendableObjectWriter.close" + ) + + @pytest.mark.asyncio + async def test_writer_retry_metrics_accounting( + self, mock_appendable_writer, monkeypatch + ): + from google.api_core.retry_async import AsyncRetry + + from google.cloud.storage import _opentelemetry_metrics + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr( + _opentelemetry_metrics, "metrics", mock_metrics, raising=False + ) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + mock_client = mock_appendable_writer["mock_client"] + mock_client._metrics_recorder = recorder + mock_stream = mock_appendable_writer["mock_stream"] + + fast_retry = AsyncRetry( + predicate=_is_write_retryable, + initial=0.001, + maximum=0.001, + on_error=mock.Mock(), + ) + writer = self._make_one(mock_client) + + # 1. open() retry + mock_stream.open = AsyncMock( + side_effect=[exceptions.ServiceUnavailable("503 open"), None] + ) + await writer.open(retry_policy=fast_retry) + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "UNAVAILABLE" + ) + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"][ + "gcp.storage.operation" + ] + == "WriteObject" + ) + + # 2. append() retry + recorder.retries_counter.add.reset_mock() + mock_stream.open = AsyncMock(return_value=None) + mock_stream.send = AsyncMock( + side_effect=[exceptions.ServiceUnavailable("503 append"), None] + ) + await writer.append(b"abc", retry_policy=fast_retry) + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "UNAVAILABLE" + ) + + # 3. close(finalize_on_close=False) retry + recorder.retries_counter.add.reset_mock() + writer.offset = 0 + mock_stream.persisted_size = 0 + redirect = BidiWriteObjectRedirectedError( + routing_token="rt_close", + write_handle=storage_type.BidiWriteHandle(handle=b"hc"), + ) + mock_stream.close = AsyncMock( + side_effect=[exceptions.Aborted("redirect", errors=[redirect]), None] + ) + await writer.close(finalize_on_close=False, retry_policy=fast_retry) + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "ABORTED" + ) + + # 4. finalize() retry + recorder.retries_counter.add.reset_mock() + writer._is_stream_open = True + writer.offset = 0 + mock_stream.close = AsyncMock(return_value=None) + mock_stream.send = AsyncMock(return_value=None) + mock_stream.recv = AsyncMock( + side_effect=[ + exceptions.InternalServerError("500 finalize"), + storage_type.BidiWriteObjectResponse( + resource=storage_type.Object(size=3) + ), + ] + ) + await writer.finalize(retry_policy=fast_retry) + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "INTERNAL" + ) + + # 5. open() failure when metrics are disabled (recorder is None) + mock_client._metrics_recorder = None + writer_disabled = self._make_one(mock_client) + mock_stream.open = AsyncMock(side_effect=exceptions.NotFound("404")) + with pytest.raises(exceptions.NotFound): + await writer_disabled.open() diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py index 0b96a92eb055..0b0a72a6f3b0 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py @@ -12,15 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio +import types from unittest import mock +import grpc import pytest from google.api_core import client_info as client_info_lib from google.api_core import client_options from google.auth import credentials as auth_credentials from google.auth.credentials import AnonymousCredentials -from google.cloud.storage import __version__ +from google.cloud.storage import __version__, _opentelemetry_metrics from google.cloud.storage.asyncio import async_grpc_client @@ -648,3 +651,816 @@ def test_async_grpc_client_interconnect_env_override( ): with pytest.raises(ValueError): async_grpc_client.AsyncGrpcClient(credentials=mock_creds) + + @mock.patch("google.cloud._storage_v2.StorageAsyncClient") + @pytest.mark.asyncio + async def test_metrics_interceptors_and_client_lifecycle( + self, mock_async_storage_client, monkeypatch + ): + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + + mock_transport_cls = mock.MagicMock() + mock_async_storage_client.get_transport_class.return_value = mock_transport_cls + mock_gapic_client = mock.AsyncMock() + mock_gapic_client.transport = mock.MagicMock() + mock_gapic_client.transport.close = mock.AsyncMock() + mock_async_storage_client.return_value = mock_gapic_client + + mock_provider = mock.MagicMock() + client = async_grpc_client.AsyncGrpcClient( + credentials=_make_credentials(), + client_options={"api_endpoint": "custom.storage.googleapis.com:443"}, + enable_metrics=True, + meter_provider=mock_provider, + ) + assert client.metrics_enabled is True + assert client._meter_provider is mock_provider + assert client._metrics_recorder._meter_provider is mock_provider + assert ( + client._metrics_recorder.server_address == "custom.storage.googleapis.com" + ) + assert client._grpc_client._metrics_recorder is client._metrics_recorder + + create_channel_kwargs = mock_transport_cls.create_channel.call_args.kwargs + assert "interceptors" in create_channel_kwargs + assert len(create_channel_kwargs["interceptors"]) == 4 + + mock_recorder = mock.MagicMock() + mock_recorder.standard_enabled = True + client._metrics_recorder = mock_recorder + + await client.delete_object("b", "o") + mock_recorder.record_logical_operation.assert_called_once() + _, del_kwargs = mock_recorder.record_logical_operation.call_args + assert ( + del_kwargs["client_method"] + == "storage.asyncio.AsyncGrpcClient.delete_object" + ) + assert del_kwargs["operation"] == "DeleteObject" + assert del_kwargs["rpc_method"] == "google.storage.v2.Storage/DeleteObject" + assert del_kwargs["error_type"] == "OK" + + mock_recorder.record_logical_operation.reset_mock() + await client.get_object("b", "o") + mock_recorder.record_logical_operation.assert_called_once() + _, get_kwargs = mock_recorder.record_logical_operation.call_args + assert ( + get_kwargs["client_method"] == "storage.asyncio.AsyncGrpcClient.get_object" + ) + assert get_kwargs["operation"] == "GetObject" + assert get_kwargs["rpc_method"] == "google.storage.v2.Storage/GetObject" + assert get_kwargs["error_type"] == "OK" + + async with client as entered: + assert entered is client + mock_gapic_client.transport.close.assert_awaited_once() + mock_recorder.shutdown.assert_called_once() + + @mock.patch( + "google.cloud._storage_v2.services.storage.transports.StorageGrpcAsyncIOTransport" + ) + @mock.patch("google.cloud._storage_v2.StorageAsyncClient") + @mock.patch("grpc.aio.insecure_channel") + def test_anonymous_client_with_metrics_enabled( + self, + mock_insecure_channel, + mock_async_storage_client, + mock_transport, + monkeypatch, + ): + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + opts = client_options.ClientOptions(api_endpoint="localhost:9000") + client = async_grpc_client.AsyncGrpcClient( + credentials=auth_credentials.AnonymousCredentials(), + client_options=opts, + enable_metrics=True, + ) + assert client.metrics_enabled is True + _, kwargs = mock_insecure_channel.call_args + assert "interceptors" in kwargs + assert len(kwargs["interceptors"]) == 4 + + @pytest.mark.asyncio + async def test_tracked_calls_and_interceptors(self): + assert ( + async_grpc_client._extract_rpc_operation( + b"/google.storage.v2.Storage/GetObject" + ) + == "google.storage.v2.Storage/GetObject" + ) + assert async_grpc_client._extract_rpc_operation(None) == "UNKNOWN" + assert async_grpc_client._extract_rpc_operation("/") == "UNKNOWN" + + status_code, err_type = async_grpc_client._resolve_status_and_error( + asyncio.CancelledError() + ) + assert (status_code, err_type) == ("CANCELLED", "CANCELLED") + + class _FakeRpcError(Exception): + def code(self): + return grpc.StatusCode.UNAVAILABLE + + assert async_grpc_client._resolve_status_and_error(_FakeRpcError()) == ( + "UNAVAILABLE", + "UNAVAILABLE", + ) + + from google.api_core import exceptions as api_exceptions + + assert async_grpc_client._resolve_status_and_error( + api_exceptions.NotFound("missing") + ) == ("NOT_FOUND", "NOT_FOUND") + assert async_grpc_client._resolve_status_and_error( + api_exceptions.Conflict("conflict") + ) == ("ABORTED", "ABORTED") + + recorder = mock.MagicMock() + recorder.standard_enabled = True + u_u, u_s, s_u, s_s = async_grpc_client._create_metrics_interceptors(recorder) + + details = mock.MagicMock(method="/google.storage.v2.Storage/GetObject") + + # 1. Unary-Unary success and methods delegation + class _AwaitableCall: + def __init__(self, result=None, exc=None): + self._result = result + self._exc = exc + self._done_writing_flag = False + self.wait_for_connection = mock.AsyncMock() + self.cancel = mock.MagicMock(return_value=True) + self.cancelled = mock.MagicMock(return_value=False) + self.done = mock.MagicMock(return_value=True) + self.add_done_callback = mock.MagicMock() + self.time_remaining = mock.MagicMock(return_value=5.0) + self.initial_metadata = mock.AsyncMock(return_value=(("k", "v"),)) + self.trailing_metadata = mock.AsyncMock(return_value=(("t", "v"),)) + self.code = mock.AsyncMock(return_value=grpc.StatusCode.OK) + self.details = mock.AsyncMock(return_value="ok") + self.debug_error_string = mock.AsyncMock(return_value="dbg") + self.write = mock.AsyncMock() + self.done_writing = mock.AsyncMock() + + async def _coro(self): + if self._exc: + raise self._exc + return self._result + + def __await__(self): + return self._coro().__await__() + + raw_uu = _AwaitableCall(result="resp") + raw_uu.custom_attr = "delegated_val" + tracked_uu = await u_u.intercept_unary_unary( + mock.AsyncMock(return_value=raw_uu), details, "req" + ) + assert tracked_uu.custom_attr == "delegated_val" + await tracked_uu.wait_for_connection() + assert tracked_uu.cancelled() is False + assert tracked_uu.done() is True + tracked_uu.add_done_callback(lambda c: None) + assert tracked_uu.time_remaining() == 5.0 + assert await tracked_uu.initial_metadata() == (("k", "v"),) + assert await tracked_uu.trailing_metadata() == (("t", "v"),) + assert await tracked_uu.code() == grpc.StatusCode.OK + assert await tracked_uu.details() == "ok" + assert await tracked_uu.debug_error_string() == "dbg" + assert await tracked_uu == "resp" + recorder.record_rpc_attempt.assert_called_once() + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + recorder.record_ttfb.assert_called_once() + assert recorder.record_ttfb.call_args.kwargs["operation"] == "GetObject" + + # 2. Stream-Unary error and continuation error + recorder.record_rpc_attempt.reset_mock() + recorder.record_ttfb.reset_mock() + raw_su = _AwaitableCall(exc=_FakeRpcError()) + tracked_su = await s_u.intercept_stream_unary( + mock.AsyncMock(return_value=raw_su), details, iter([]) + ) + tracked_su._done_writing_flag = True + assert tracked_su._done_writing_flag is True + await tracked_su.write("chunk") + await tracked_su.done_writing() + with pytest.raises(_FakeRpcError): + await tracked_su + assert ( + recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "UNAVAILABLE" + ) + recorder.record_ttfb.assert_not_called() + + for interceptor, method_name in ( + (u_u, "intercept_unary_unary"), + (u_s, "intercept_unary_stream"), + (s_u, "intercept_stream_unary"), + (s_s, "intercept_stream_stream"), + ): + recorder.record_rpc_attempt.reset_mock() + with pytest.raises(_FakeRpcError): + await getattr(interceptor, method_name)( + mock.AsyncMock(side_effect=_FakeRpcError()), details, "req" + ) + recorder.record_rpc_attempt.assert_called_once() + + # 3. Unary-Stream __aiter__ and Stream-Stream read() + clean close + class _StreamCall(_AwaitableCall): + def __init__(self, items): + super().__init__() + self._items = list(items) + + def __aiter__(self): + async def _gen(): + for item in self._items: + if isinstance(item, BaseException): + raise item + yield item + + return _gen() + + async def read(self): + if not self._items: + return grpc.aio.EOF + item = self._items.pop(0) + if isinstance(item, BaseException): + raise item + return item + + recorder.record_rpc_attempt.reset_mock() + recorder.record_ttfb.reset_mock() + tracked_us = await u_s.intercept_unary_stream( + mock.AsyncMock(return_value=_StreamCall(["a", "b"])), details, "req" + ) + collected = [x async for x in tracked_us] + assert collected == ["a", "b"] + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + recorder.record_ttfb.assert_called_once() + + recorder.record_rpc_attempt.reset_mock() + recorder.record_ttfb.reset_mock() + tracked_ss = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=_StreamCall(["x", "y"])), details, iter([]) + ) + assert await tracked_ss.read() == "x" + assert await tracked_ss.read() == "y" + assert await tracked_ss.read() == grpc.aio.EOF + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + recorder.record_ttfb.assert_called_once() + + # Immediate EOF without messages records attempt OK without TTFB + recorder.record_rpc_attempt.reset_mock() + recorder.record_ttfb.reset_mock() + tracked_empty = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=_StreamCall([])), details, iter([]) + ) + assert await tracked_empty.read() == grpc.aio.EOF + recorder.record_rpc_attempt.assert_called_once() + recorder.record_ttfb.assert_not_called() + + # Clean close vs abnormal cancel (including _interceptors_task, cycle, and non-dict object) + recorder.record_rpc_attempt.reset_mock() + tracked_ss_clean = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=_StreamCall([])), details, iter([]) + ) + done_fut = asyncio.get_running_loop().create_future() + done_fut.set_result(tracked_ss_clean) + wrapper = mock.MagicMock(_call=42, _interceptors_task=done_fut) + wrapper._wrapped = wrapper # cycle + async_grpc_client._mark_call_closing_cleanly(None) + async_grpc_client._mark_call_closing_cleanly(wrapper) + assert tracked_ss_clean._closing_cleanly is True + tracked_ss_clean.cancel() + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + + # Error paths in wait_for_connection, read, write, done_writing + err_call = _StreamCall([_FakeRpcError()]) + err_call.wait_for_connection = mock.AsyncMock(side_effect=_FakeRpcError()) + err_call.write = mock.AsyncMock(side_effect=_FakeRpcError()) + err_call.done_writing = mock.AsyncMock(side_effect=_FakeRpcError()) + + for op_fn in ( + lambda c: c.wait_for_connection(), + lambda c: c.read(), + lambda c: c.write("x"), + lambda c: c.done_writing(), + ): + recorder.record_rpc_attempt.reset_mock() + c = await s_s.intercept_stream_stream( + mock.AsyncMock( + return_value=_StreamCall([_FakeRpcError()]) if op_fn else err_call + ), + details, + iter([]), + ) + c._call = err_call + err_call._items = [_FakeRpcError()] + with pytest.raises(_FakeRpcError): + await op_fn(c) + recorder.record_rpc_attempt.assert_called_once() + + # CancelledError when _closing_cleanly is True records OK + recorder.record_rpc_attempt.reset_mock() + clean_cancel_call = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=_StreamCall([asyncio.CancelledError()])), + details, + iter([]), + ) + clean_cancel_call._closing_cleanly = True + with pytest.raises(asyncio.CancelledError): + await clean_cancel_call.read() + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + + recorder.record_rpc_attempt.reset_mock() + tracked_ss_cancel = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=_StreamCall([])), details, iter([]) + ) + tracked_ss_cancel.cancel() + tracked_ss_cancel.cancel() # second call is a no-op + assert ( + recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "CANCELLED" + ) + assert ( + async_grpc_client._resolve_network_path( + mock.MagicMock(peer=lambda: 123), + direct_path_attempted=False, + received_server_response=True, + ) + == "cloudpath" + ) + assert ( + async_grpc_client._resolve_network_path( + mock.MagicMock(peer=lambda: "unix:/tmp/sock"), + direct_path_attempted=False, + received_server_response=False, + ) + == "unknown" + ) + + class _NoneCodeRpcErr(Exception): + def code(self): + return None + + assert async_grpc_client._resolve_status_and_error(_NoneCodeRpcErr()) == ( + "UNKNOWN", + "UNKNOWN", + ) + + # Early break from __aiter__ (clean vs cancelled) and peer() network_path detection + recorder.record_rpc_attempt.reset_mock() + stream_with_peer = _StreamCall(["a", "b", "c"]) + stream_with_peer.peer = lambda: b"ipv6:[2001:4860:8040::1]:443" + tracked_early = await u_s.intercept_unary_stream( + mock.AsyncMock(return_value=stream_with_peer), details, "req" + ) + ait = tracked_early.__aiter__() + assert await ait.__anext__() == "a" + await ait.aclose() + assert ( + recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "CANCELLED" + ) + assert ( + recorder.record_rpc_attempt.call_args.kwargs["network_path"] == "directpath" + ) + + recorder.record_rpc_attempt.reset_mock() + stream_cloud = _StreamCall(["a", "b"]) + stream_cloud.peer = lambda: "ipv4:142.250.1.1:443" + tracked_early_clean = await u_s.intercept_unary_stream( + mock.AsyncMock(return_value=stream_cloud), details, "req" + ) + tracked_early_clean._closing_cleanly = True + ait_clean = tracked_early_clean.__aiter__() + assert await ait_clean.__anext__() == "a" + await ait_clean.aclose() + assert recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + assert ( + recorder.record_rpc_attempt.call_args.kwargs["network_path"] == "cloudpath" + ) + + # Mid-iteration exception in __aiter__ + recorder.record_rpc_attempt.reset_mock() + tracked_err_iter = await u_s.intercept_unary_stream( + mock.AsyncMock(return_value=_StreamCall(["a", _FakeRpcError()])), + details, + "req", + ) + with pytest.raises(_FakeRpcError): + _ = [x async for x in tracked_err_iter] + assert ( + recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "UNAVAILABLE" + ) + + def test_resolve_network_path_comprehensive(self): + # 1. _inspect_metadata edge cases + assert async_grpc_client._inspect_metadata(123) == (False, False) + assert async_grpc_client._inspect_metadata(()) == (False, False) + assert async_grpc_client._inspect_metadata([object()]) == (True, False) + assert async_grpc_client._inspect_metadata([()]) == (True, False) + + # 2. GFE headers override direct_path_attempted=True -> "cloudpath" + call_cy_init = types.SimpleNamespace( + _cython_call=types.SimpleNamespace( + _initial_metadata=( + types.SimpleNamespace(key="x-goog-gfe-service-time", value="10"), + ), + _trailing_metadata=None, + _status=None, + ) + ) + assert ( + async_grpc_client._resolve_network_path( + call=call_cy_init, direct_path_attempted=True + ) + == "cloudpath" + ) + + call_cy_status_fn = types.SimpleNamespace( + _cython_call=types.SimpleNamespace( + _initial_metadata=(), + _trailing_metadata=None, + _status=types.SimpleNamespace( + trailing_metadata=lambda: ((b"X-GUploader-UploadID", b"up1"),) + ), + ) + ) + assert ( + async_grpc_client._resolve_network_path( + call=call_cy_status_fn, direct_path_attempted=True + ) + == "cloudpath" + ) + + call_cy_status_attr = types.SimpleNamespace( + _cython_call=types.SimpleNamespace( + _initial_metadata=None, + _trailing_metadata=None, + _status=types.SimpleNamespace( + _trailing_metadata=(("x-goog-gfe-service-time", "1"),), + _debug_error_string="no ip here", + ), + ) + ) + assert ( + async_grpc_client._resolve_network_path( + call=call_cy_status_attr, direct_path_attempted=True + ) + == "cloudpath" + ) + + # Non-GFE server metadata sets has_server_metadata=True + call_non_gfe = types.SimpleNamespace( + _initial_metadata=[("x-custom-server", "ok")] + ) + assert ( + async_grpc_client._resolve_network_path( + call=call_non_gfe, + direct_path_attempted=True, + received_server_response=False, + ) + == "directpath" + ) + + # GFE metadata on RpcError (grpc.aio.Metadata) overrides direct_path_attempted=True + class _GfeRpcErr(Exception): + def initial_metadata(self): + return grpc.aio.Metadata() + + def trailing_metadata(self): + return grpc.aio.Metadata(("x-goog-gfe-service-time", "5")) + + assert ( + async_grpc_client._resolve_network_path( + exc=_GfeRpcErr(), direct_path_attempted=True + ) + == "cloudpath" + ) + + class _NonGfeAttrErr(Exception): + initial_metadata = None + trailing_metadata = [("x-server", "1")] + debug_error_string = "ipv4:142.250.10.1:443" + + assert ( + async_grpc_client._resolve_network_path( + exc=_NonGfeAttrErr(), direct_path_attempted=True + ) + == "cloudpath" + ) + + # Wrapped call + chained exception when peer IP already detected on outer layer + inner_call = types.SimpleNamespace(peer=lambda: "ipv4:142.250.1.1:443") + outer_call_with_peer = types.SimpleNamespace( + peer=lambda: "ipv4:34.126.1.1:443", _call=inner_call + ) + chained_exc = _NonGfeAttrErr("outer") + chained_exc.__cause__ = RuntimeError("inner") + assert ( + async_grpc_client._resolve_network_path( + call=outer_call_with_peer, exc=chained_exc + ) + == "directpath" + ) + + class _NonGfeNoIpErr(Exception): + initial_metadata = None + trailing_metadata = [("x-server", "1")] + debug_error_string = None + + assert ( + async_grpc_client._resolve_network_path( + exc=_NonGfeNoIpErr(), direct_path_attempted=True + ) + == "directpath" + ) + + # 3. DirectPath IPv4 (/18) and IPv6 (/42) CIDR boundaries + IPv4-mapped IPv6 + for peer_uri, expected in ( + ("ipv4:34.126.0.1:443", "directpath"), + ("ipv4:34.126.63.255:443", "directpath"), + ("ipv4:34.126.64.1:443", "cloudpath"), + ("ipv6:[2001:4860:8040::1]:443", "directpath"), + ("ipv6:%5B2001:4860:8040:ffff::1%5D:443", "directpath"), + ("ipv6:2001:4860:8040::2", "directpath"), + ("ipv6:[2001:4860:8080::1]:443", "cloudpath"), + ("ipv6:[::ffff:34.126.1.2]:443", "directpath"), + ("ipv6:[::ffff:142.250.1.2]:443", "cloudpath"), + ): + c = types.SimpleNamespace(peer=lambda p=peer_uri: p) + assert async_grpc_client._resolve_network_path(call=c) == expected + + # Invalid IP ignored -> falls back + invalid_ip_call = types.SimpleNamespace(peer=lambda: "ipv4:999.999.999.999:443") + assert ( + async_grpc_client._resolve_network_path( + call=invalid_ip_call, + direct_path_attempted=False, + received_server_response=False, + ) + == "unknown" + ) + + # 4. C-core debug_error_string (bytes & str, status & exc) + call_dbg_bytes = types.SimpleNamespace( + _cython_call=types.SimpleNamespace( + _initial_metadata=(), + _trailing_metadata=None, + _status=types.SimpleNamespace( + trailing_metadata=lambda: (), + debug_error_string=lambda: b'{"peer":"ipv4:34.126.5.5:443"}', + ), + ) + ) + assert ( + async_grpc_client._resolve_network_path(call=call_dbg_bytes) == "directpath" + ) + + class _BytesDbgErr(Exception): + def debug_error_string(self): + return b"ipv6:%5b2001:4860:8040::9%5d:443" + + assert ( + async_grpc_client._resolve_network_path(exc=_BytesDbgErr()) == "directpath" + ) + + # 5. Fallback matrix & wrapped _TrackedCallBase discovery + >16 chain cap + assert ( + async_grpc_client._resolve_network_path( + direct_path_attempted=True, received_server_response=True + ) + == "directpath" + ) + assert ( + async_grpc_client._resolve_network_path( + direct_path_attempted=True, received_server_response=False + ) + == "unknown" + ) + assert ( + async_grpc_client._resolve_network_path( + direct_path_attempted=None, received_server_response=True + ) + == "unknown" + ) + + tracked = async_grpc_client._TrackedUnaryUnaryCall( + types.SimpleNamespace(), + mock.MagicMock(), + "google.storage.v2.Storage/GetObject", + "GetObject", + 0.0, + None, + direct_path_attempted=True, + ) + outer = types.SimpleNamespace(_call=tracked) + assert ( + async_grpc_client._resolve_network_path( + call=outer, + direct_path_attempted=None, + received_server_response=True, + ) + == "directpath" + ) + + # Deep chain >16 steps terminates cleanly + head = types.SimpleNamespace() + cur = head + for _ in range(20): + nxt = types.SimpleNamespace() + cur._call = nxt + cur = nxt + assert len(list(async_grpc_client._iter_wrapped_calls(head))) == 16 + + @pytest.mark.asyncio + async def test_unary_retry_and_bidi_active_operation_metrics(self, monkeypatch): + from google.api_core import exceptions as api_exceptions + from google.api_core.retry_async import AsyncRetry + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr( + _opentelemetry_metrics, "metrics", mock_metrics, raising=False + ) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + u_u, _, _, s_s = async_grpc_client._create_metrics_interceptors( + recorder, direct_path_attempted=True + ) + details = mock.MagicMock(method="/google.storage.v2.Storage/GetObject") + + # 1. End-to-end Unary-Unary retry (UNAVAILABLE -> OK) + calls_made = 0 + + class _SimpleAwaitable: + def __init__(self, val=None, exc=None): + self._val = val + self._exc = exc + + async def _run(self): + if self._exc: + raise self._exc + return self._val + + def __await__(self): + return self._run().__await__() + + async def unary_continuation(_details, _req): + nonlocal calls_made + calls_made += 1 + if calls_made == 1: + return _SimpleAwaitable(exc=api_exceptions.ServiceUnavailable("503")) + return _SimpleAwaitable(val="object_meta") + + retry = AsyncRetry( + predicate=lambda e: isinstance(e, api_exceptions.ServiceUnavailable), + initial=0.001, + maximum=0.001, + ) + + async def invoke_rpc(): + call = await u_u.intercept_unary_unary(unary_continuation, details, "req") + return await call + + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncGrpcClient.get_object", + operation="GetObject", + rpc_method="google.storage.v2.Storage/GetObject", + ): + res = await retry(invoke_rpc)() + + assert res == "object_meta" + assert recorder.attempts_counter.add.call_count == 2 + assert recorder.ttfb.record.call_count == 1 + assert recorder.errors_counter.add.call_count == 1 + assert recorder.retries_counter.add.call_count == 1 + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "UNAVAILABLE" + ) + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"][ + "gcp.storage.operation" + ] + == "GetObject" + ) + assert recorder.operations_counter.add.call_count == 1 + assert ( + recorder.operations_counter.add.call_args.kwargs["attributes"]["error.type"] + == "OK" + ) + assert ( + recorder.client_request_duration.record.call_args.kwargs["attributes"][ + "rpc.method" + ] + == "google.storage.v2.Storage/GetObject" + ) + + # 2. Bidi stream opened in open() op and failing mid-stream in download_ranges() op + bidi_details = mock.MagicMock( + method="/google.storage.v2.Storage/BidiReadObject" + ) + raw_bidi = mock.MagicMock() + raw_bidi.read = mock.AsyncMock( + side_effect=api_exceptions.Aborted("Idle stream closed") + ) + + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.open", + operation="ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) as open_op: + tracked_bidi = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=raw_bidi), bidi_details, iter([]) + ) + # Simulate stream opened 100s ago + tracked_bidi._start_time -= 100.0 + + assert open_op.recorded_attempts == 0 + assert open_op.finished is True + async_grpc_client._bind_call_to_operation(tracked_bidi, None) + + recorder.rpc_call_duration.record.reset_mock() + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + operation="ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) as dl_op: + with pytest.raises(api_exceptions.Aborted): + await tracked_bidi.read() + assert dl_op.recorded_attempts == 1 + assert dl_op.last_failure_error_type == "ABORTED" + recorded_dur = recorder.rpc_call_duration.record.call_args.args[0] + assert recorded_dur < 10.0 + + # 3. _bind_call_to_operation preserves _start_time: + # - background recv failure uses bound op start_time + # - clean stream close records full physical stream lifetime from _start_time + raw_bg_fail = mock.MagicMock() + raw_bg_fail.read = mock.AsyncMock( + side_effect=api_exceptions.Aborted("Background recv failed") + ) + tracked_bg = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=raw_bg_fail), bidi_details, iter([]) + ) + tracked_bg._start_time -= 100.0 + original_bg_start = tracked_bg._start_time + wrapped_bg = types.SimpleNamespace(_call=tracked_bg) + bg_op = _opentelemetry_metrics._OperationState( + client_method="storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + operation="ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) + async_grpc_client._bind_call_to_operation(wrapped_bg, bg_op) + assert tracked_bg._op_state is bg_op + assert tracked_bg._start_time == original_bg_start + + recorder.rpc_call_duration.record.reset_mock() + with pytest.raises(api_exceptions.Aborted): + await tracked_bg.read() + assert recorder.rpc_call_duration.record.call_args.args[0] < 10.0 + + # 4. Healthy Bidi stream closed after _bind_call_to_operation and inside finalize() + # records full physical stream lifetime (>= 100.0s) + raw_healthy = mock.MagicMock() + raw_healthy.cancel = mock.MagicMock(return_value=True) + raw_healthy.read = mock.AsyncMock(return_value=grpc.aio.EOF) + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.open", + operation="WriteObject", + rpc_method="google.storage.v2.Storage/BidiWriteObject", + ): + tracked_healthy = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=raw_healthy), + mock.MagicMock(method="/google.storage.v2.Storage/BidiWriteObject"), + iter([]), + ) + tracked_healthy._start_time -= 100.0 + + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.append", + operation="WriteObject", + rpc_method="google.storage.v2.Storage/BidiWriteObject", + ) as append_op: + async_grpc_client._bind_call_to_operation(tracked_healthy, append_op) + + recorder.rpc_call_duration.record.reset_mock() + with _opentelemetry_metrics.start_operation( + recorder, + client_method="storage.asyncio.AsyncAppendableObjectWriter.finalize", + operation="WriteObject", + rpc_method="google.storage.v2.Storage/BidiWriteObject", + ): + async_grpc_client._mark_call_closing_cleanly(tracked_healthy) + tracked_healthy.cancel() + assert recorder.rpc_call_duration.record.call_args.args[0] >= 100.0 diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py index afe1dfd221d9..411cef7cab45 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py @@ -865,3 +865,431 @@ def test_is_read_retryable_predicate(self): assert _is_read_retryable(exceptions.NotFound("not found")) is False assert _is_read_retryable(exceptions.PermissionDenied("denied")) is False assert _is_read_retryable(exceptions.InvalidArgument("invalid")) is False + + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader.generate_random_56_bit_integer" + ) + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader._AsyncReadObjectStream" + ) + @pytest.mark.asyncio + async def test_mrd_metrics_recording( + self, mock_cls_async_read_object_stream, mock_random_int + ): + mock_recorder = mock.MagicMock() + mock_recorder.standard_enabled = True + mock_client = mock.MagicMock() + mock_client._metrics_recorder = mock_recorder + mock_client.grpc_client = mock.AsyncMock() + + stream = mock.MagicMock() + stream.open = AsyncMock() + stream.close = AsyncMock() + stream.send = AsyncMock() + stream._mark_closing_cleanly = mock.MagicMock() + stream.generation_number = _TEST_GENERATION_NUMBER + stream.persisted_size = _TEST_OBJECT_SIZE + stream.read_handle = _TEST_READ_HANDLE + stream.is_finalized = True + stream.full_obj_server_crc32c = None + stream.is_stream_open = True + mock_cls_async_read_object_stream.return_value = stream + + response = _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=b"data", crc32c=google_crc32c.value(b"data") + ), + range_end=True, + read_range=_storage_v2.ReadRange( + read_offset=0, read_length=4, read_id=123 + ), + ) + ] + ) + stream.recv = AsyncMock(side_effect=[response]) + mock_random_int.return_value = 123 + + mrd = await AsyncMultiRangeDownloader.create_mrd( + mock_client, + _TEST_BUCKET_NAME, + _TEST_OBJECT_NAME, + ) + # create_mrd suppresses nested open() so only 1 logical operation is recorded + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncMultiRangeDownloader.create_mrd" + ) + + mock_recorder.record_logical_operation.reset_mock() + buf = BytesIO() + await mrd.download_ranges([(0, 4, buf)]) + mock_recorder.record_response_body_size.assert_called_once_with( + 4, operation="ReadObject" + ) + mock_recorder.record_logical_operation.assert_called_once() + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["client_method"] + == "storage.asyncio.AsyncMultiRangeDownloader.download_ranges" + ) + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["operation"] + == "ReadObject" + ) + assert ( + mock_recorder.record_logical_operation.call_args.kwargs["rpc_method"] + == "google.storage.v2.Storage/BidiReadObject" + ) + + # Verify download_ranges with custom metadata marks stream clean before reopen + chunk1 = _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=b"ab", crc32c=google_crc32c.value(b"ab") + ), + range_end=False, + read_range=_storage_v2.ReadRange( + read_offset=0, read_length=2, read_id=123 + ), + ) + ] + ) + chunk2 = _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=b"cd", crc32c=google_crc32c.value(b"cd") + ), + range_end=True, + read_range=_storage_v2.ReadRange( + read_offset=2, read_length=2, read_id=123 + ), + ) + ] + ) + stream.recv = AsyncMock(side_effect=[chunk1, chunk2]) + mock_recorder.record_response_body_size.reset_mock() + stream._mark_closing_cleanly.reset_mock() + buf2 = BytesIO() + await mrd.download_ranges([(0, 4, buf2)], metadata=[("x-custom", "val")]) + stream._mark_closing_cleanly.assert_called() + mock_recorder.record_response_body_size.assert_called_once_with( + 4, operation="ReadObject" + ) + + await mrd.close() + stream._mark_closing_cleanly.assert_called() + + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader.generate_random_56_bit_integer" + ) + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader._AsyncReadObjectStream" + ) + @pytest.mark.asyncio + async def test_mrd_retry_metrics_accounting( + self, mock_cls_async_read_object_stream, mock_random_int, monkeypatch + ): + from google.api_core.retry_async import AsyncRetry + + from google.cloud.storage import _opentelemetry_metrics + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr( + _opentelemetry_metrics, "metrics", mock_metrics, raising=False + ) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + mock_client = mock.MagicMock() + mock_client._metrics_recorder = recorder + mock_client.grpc_client = mock.AsyncMock() + + # 1. MRD.open() retry (attempt 1 fails with 503, attempt 2 succeeds) + failed_open_stream = mock.MagicMock() + failed_open_stream.open = AsyncMock( + side_effect=exceptions.ServiceUnavailable("503 open failed") + ) + failed_open_stream.is_stream_open = False + + open_stream = mock.MagicMock() + open_stream.open = AsyncMock() + open_stream.close = AsyncMock() + open_stream.send = AsyncMock() + open_stream.generation_number = _TEST_GENERATION_NUMBER + open_stream.persisted_size = _TEST_OBJECT_SIZE + open_stream.read_handle = _TEST_READ_HANDLE + open_stream.is_finalized = True + open_stream.full_obj_server_crc32c = None + open_stream.is_stream_open = True + + # 2. MRD.download_ranges() mid-stream retry (attempt 1 Aborted, attempt 2 succeeds) + open_stream.recv = AsyncMock( + side_effect=exceptions.Aborted("Idle stream closed") + ) + recovered_stream = mock.MagicMock() + recovered_stream.open = AsyncMock() + recovered_stream.close = AsyncMock() + recovered_stream.send = AsyncMock() + recovered_stream.generation_number = _TEST_GENERATION_NUMBER + recovered_stream.persisted_size = _TEST_OBJECT_SIZE + recovered_stream.read_handle = _TEST_READ_HANDLE + recovered_stream.is_finalized = True + recovered_stream.full_obj_server_crc32c = None + recovered_stream.is_stream_open = True + recovered_stream.recv = AsyncMock( + side_effect=[ + _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=b"data", crc32c=google_crc32c.value(b"data") + ), + range_end=True, + read_range=_storage_v2.ReadRange( + read_offset=0, read_length=4, read_id=123 + ), + ) + ] + ) + ] + ) + + mock_cls_async_read_object_stream.side_effect = [ + failed_open_stream, + open_stream, + recovered_stream, + ] + mock_random_int.return_value = 123 + + user_on_error = mock.Mock() + fast_retry = AsyncRetry(initial=0.001, maximum=0.001, on_error=user_on_error) + mrd = AsyncMultiRangeDownloader( + mock_client, _TEST_BUCKET_NAME, _TEST_OBJECT_NAME + ) + await mrd.open(retry_policy=fast_retry) + + user_on_error.assert_called_once() + recorder.retries_counter.add.assert_called_once() + _, open_retry_kw = recorder.retries_counter.add.call_args + assert open_retry_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert open_retry_kw["attributes"]["error.type"] == "UNAVAILABLE" + + recorder.retries_counter.add.reset_mock() + buf = BytesIO() + await mrd.download_ranges( + [(0, 4, buf)], + retry_policy=AsyncRetry( + predicate=lambda e: True, initial=0.001, maximum=0.001 + ), + ) + assert buf.getvalue() == b"data" + recorder.retries_counter.add.assert_called_once() + _, dl_retry_kw = recorder.retries_counter.add.call_args + assert dl_retry_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert dl_retry_kw["attributes"]["error.type"] == "ABORTED" + + # 3. MRD.open() failure when metrics are disabled (recorder is None) + disabled_client = mock.MagicMock() + disabled_client._metrics_recorder = None + disabled_client.grpc_client = mock.AsyncMock() + fail_stream = mock.MagicMock() + fail_stream.open = AsyncMock(side_effect=exceptions.NotFound("404")) + mock_cls_async_read_object_stream.side_effect = [fail_stream] + mrd_disabled = AsyncMultiRangeDownloader( + disabled_client, _TEST_BUCKET_NAME, _TEST_OBJECT_NAME + ) + with pytest.raises(exceptions.NotFound): + await mrd_disabled.open() + + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader.generate_random_56_bit_integer" + ) + @pytest.mark.asyncio + async def test_mrd_bidi_stream_interceptor_ttfb_duration_and_reopen_metrics( + self, mock_random_int, monkeypatch + ): + import grpc + from google.api_core.retry_async import AsyncRetry + + from google.cloud.storage import _opentelemetry_metrics + from google.cloud.storage.asyncio import async_grpc_client + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr( + _opentelemetry_metrics, "metrics", mock_metrics, raising=False + ) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + _, _, _, s_s = async_grpc_client._create_metrics_interceptors( + recorder, direct_path_attempted=False + ) + details = mock.MagicMock(method="/google.storage.v2.Storage/BidiReadObject") + + class _FakeBidiCall: + def __init__(self): + self._queue = asyncio.Queue() + self._done = False + self._pump_task = None + + def done(self): + return self._done + + def add_done_callback(self, callback): + pass + + def cancel(self): + self._done = True + self._pump_task.cancel() + self._queue.put_nowait(grpc.aio.EOF) + return True + + async def write(self, request): + if request.read_object_spec.bucket: + self._queue.put_nowait( + _storage_v2.BidiReadObjectResponse( + metadata=_storage_v2.Object( + generation=_TEST_GENERATION_NUMBER, + size=_TEST_OBJECT_SIZE, + ), + read_handle=_storage_v2.BidiReadHandle( + handle=_TEST_READ_HANDLE + ), + ) + ) + if request.read_ranges: + for r in request.read_ranges: + self._queue.put_nowait( + _storage_v2.BidiReadObjectResponse( + object_data_ranges=[ + _storage_v2.ObjectRangeData( + checksummed_data=_storage_v2.ChecksummedData( + content=b"data", + crc32c=google_crc32c.value(b"data"), + ), + range_end=True, + read_range=_storage_v2.ReadRange( + read_offset=r.read_offset, + read_length=4, + read_id=r.read_id, + ), + ) + ] + ) + ) + + async def done_writing(self): + self._queue.put_nowait(grpc.aio.EOF) + + async def read(self): + return await self._queue.get() + + created_tracked_calls = [] + raw_calls = [ + _FakeBidiCall(), + _FakeBidiCall(), + _FakeBidiCall(), + ] + + async def start_bidi_rpc(request_iterator=None, metadata=None): + raw = raw_calls.pop(0) + tracked = await s_s.intercept_stream_stream( + mock.AsyncMock(return_value=raw), details, request_iterator + ) + + async def _pump_requests(): + async for req in request_iterator: + await tracked.write(req) + await tracked.done_writing() + + raw._pump_task = asyncio.create_task(_pump_requests()) + created_tracked_calls.append(tracked) + return tracked + + mock_client = mock.MagicMock() + mock_client._metrics_recorder = recorder + bidi_key = object() + mock_client.grpc_client._client._transport.bidi_read_object = bidi_key + mock_client.grpc_client._client._transport._wrapped_methods = { + bidi_key: start_bidi_rpc + } + mock_random_int.side_effect = [101, 102, 201, 202] + + # Scenario A: Healthy MRD lifecycle across 2 download_ranges() calls + close() + mrd = await AsyncMultiRangeDownloader.create_mrd( + mock_client, _TEST_BUCKET_NAME, _TEST_OBJECT_NAME + ) + created_tracked_calls[0]._start_time -= 50.0 + assert recorder.ttfb.record.call_count == 1 + + b1, b2 = BytesIO(), BytesIO() + await mrd.download_ranges([(0, 4, b1)]) + await mrd.download_ranges([(0, 4, b2)]) + # Same physical stream: TTFB is still recorded only once + assert recorder.ttfb.record.call_count == 1 + assert b1.getvalue() == b"data" + assert b2.getvalue() == b"data" + + await mrd.close() + assert recorder.attempts_counter.add.call_count == 1 + assert ( + recorder.attempts_counter.add.call_args.kwargs["attributes"]["error.type"] + == "OK" + ) + assert recorder.rpc_call_duration.record.call_args.args[0] >= 50.0 + + # Scenario B: Broken stream during download_ranges() triggers reopen_stream() + recorder.ttfb.record.reset_mock() + recorder.attempts_counter.add.reset_mock() + recorder.retries_counter.add.reset_mock() + recorder.rpc_call_duration.record.reset_mock() + + mrd2 = await AsyncMultiRangeDownloader.create_mrd( + mock_client, _TEST_BUCKET_NAME, _TEST_OBJECT_NAME + ) + assert recorder.ttfb.record.call_count == 1 + mrd2.read_obj_str.send = AsyncMock( + side_effect=exceptions.ServiceUnavailable("send broke") + ) + + b3 = BytesIO() + await mrd2.download_ranges( + [(0, 4, b3)], + retry_policy=AsyncRetry( + predicate=lambda e: True, initial=0.001, maximum=0.001 + ), + ) + assert b3.getvalue() == b"data" + # Stream 1 + reopened Stream 2 => 2 TTFB points and 1 retry + assert recorder.ttfb.record.call_count == 2 + assert recorder.retries_counter.add.call_count == 1 + # Broken stream 1 was NOT falsely marked OK by reopen_stream() + stream1_err = recorder.attempts_counter.add.call_args_list[0].kwargs[ + "attributes" + ]["error.type"] + assert stream1_err == "CANCELLED" + + await mrd2.close() + assert recorder.attempts_counter.add.call_count == 2 + stream2_err = recorder.attempts_counter.add.call_args_list[1].kwargs[ + "attributes" + ]["error.type"] + assert stream2_err == "OK" diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py index 393fa7b2b709..3c5687581c1c 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_read_object_stream.py @@ -537,3 +537,62 @@ async def test_open_unfinalized_object_skips_checksum( assert read_obj_stream.is_finalized is False assert read_obj_stream.full_obj_server_crc32c is None + + +@mock.patch("google.cloud.storage.asyncio.async_read_object_stream.AsyncBidiRpc") +@mock.patch( + "google.cloud.storage.asyncio.async_grpc_client.AsyncGrpcClient.grpc_client" +) +@pytest.mark.asyncio +async def test_open_records_pending_retry_and_close_marks_clean( + mock_client, mock_cls_async_bidi_rpc, monkeypatch +): + from google.cloud.storage import _opentelemetry_metrics + + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + socket_like_rpc = AsyncMock() + mock_call = mock.MagicMock(_closing_cleanly=False) + mock_call.peer = lambda: "ipv4:34.126.1.1:443" + socket_like_rpc.call = mock_call + mock_cls_async_bidi_rpc.return_value = socket_like_rpc + socket_like_rpc.open = AsyncMock() + recv_response = mock.MagicMock(spec=_storage_v2.BidiReadObjectResponse) + recv_response.metadata = None + recv_response.read_handle = _TEST_READ_HANDLE + socket_like_rpc.recv = AsyncMock(return_value=recv_response) + + read_obj_stream = _AsyncReadObjectStream( + client=mock_client, + bucket_name=_TEST_BUCKET_NAME, + object_name=_TEST_OBJECT_NAME, + metrics_recorder=recorder, + ) + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.open", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) as op_state: + op_state.mark_attempt_failed("UNAVAILABLE") + await read_obj_stream.open() + + recorder.retries_counter.add.assert_called_once() + _, retry_kw = recorder.retries_counter.add.call_args + assert retry_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert retry_kw["attributes"]["error.type"] == "UNAVAILABLE" + assert retry_kw["attributes"]["network.path"] == "directpath" + + read_obj_stream.requests_done = AsyncMock() + await read_obj_stream.close() + assert mock_call._closing_cleanly is True diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py index 049c9f6608a9..120e9fa23ea7 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_write_object_stream.py @@ -393,3 +393,69 @@ async def test_methods_require_open_raises(self, mock_client): await stream.recv() with pytest.raises(ValueError, match="Stream is not open"): await stream.close() + + @mock.patch("google.cloud.storage.asyncio.async_write_object_stream.AsyncBidiRpc") + @pytest.mark.asyncio + async def test_open_records_write_ttfb_and_close_marks_clean( + self, mock_rpc_cls, mock_client + ): + from google.cloud.storage.asyncio import async_grpc_client + + mock_recorder = MagicMock() + mock_recorder.standard_enabled = True + _, _, _, s_s = async_grpc_client._create_metrics_interceptors( + mock_recorder, direct_path_attempted=False + ) + details = MagicMock(method="/google.storage.v2.Storage/BidiWriteObject") + raw_call = MagicMock() + raw_call.read = AsyncMock( + side_effect=[ + MagicMock(persisted_size=0, resource=None, write_handle=b"h1"), + MagicMock(persisted_size=4, resource=None, write_handle=b"h1"), + grpc.aio.EOF, + ] + ) + tracked_call = await s_s.intercept_stream_stream( + AsyncMock(return_value=raw_call), details, iter([]) + ) + + mock_rpc = mock_rpc_cls.return_value + mock_rpc.open = AsyncMock() + mock_rpc.send = AsyncMock() + mock_rpc.recv = AsyncMock(side_effect=tracked_call.read) + mock_rpc.close = AsyncMock() + mock_rpc.call = tracked_call + + stream = _AsyncWriteObjectStream( + mock_client, BUCKET, OBJECT, metrics_recorder=mock_recorder + ) + await stream.open() + mock_recorder.record_ttfb.assert_called_once() + assert mock_recorder.record_ttfb.call_args.kwargs["operation"] == "WriteObject" + assert "network_path" in mock_recorder.record_ttfb.call_args.kwargs + + # Subsequent recv on the same physical stream must not emit duplicate TTFB + await stream.recv() + mock_recorder.record_ttfb.assert_called_once() + + await stream.close() + assert tracked_call._closing_cleanly is True + mock_recorder.record_rpc_attempt.assert_called_once() + assert mock_recorder.record_rpc_attempt.call_args.kwargs["status_code"] == "OK" + + @mock.patch("google.cloud.storage.asyncio.async_write_object_stream.AsyncBidiRpc") + @pytest.mark.asyncio + async def test_open_closes_rpc_when_initial_recv_fails( + self, mock_rpc_cls, mock_client + ): + mock_rpc = mock_rpc_cls.return_value + mock_rpc.open = AsyncMock() + mock_rpc.recv = AsyncMock(side_effect=RuntimeError("initial recv failed")) + mock_rpc.close = AsyncMock() + + stream = _AsyncWriteObjectStream(mock_client, BUCKET, OBJECT) + with pytest.raises(RuntimeError, match="initial recv failed"): + await stream.open() + + mock_rpc.close.assert_awaited_once() + assert not stream.is_stream_open diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py b/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py index 9c75b9e8e1e1..2c54d985535e 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py @@ -13,7 +13,7 @@ # limitations under the License. import asyncio -from unittest.mock import AsyncMock +from unittest.mock import AsyncMock, MagicMock import grpc import pytest @@ -399,6 +399,7 @@ class TestReopenStream: async def test_reopen_bumps_generation_and_replaces_stream(self): # Given a multiplexer with a registered queue old_stream = AsyncMock() + old_stream._mark_closing_cleanly = MagicMock() old_stream.recv = AsyncMock(side_effect=asyncio.Event().wait) mux = _StreamMultiplexer(old_stream) mux.register({1}) @@ -410,7 +411,8 @@ async def test_reopen_bumps_generation_and_replaces_stream(self): # When the stream is reopened with the correct generation await mux.reopen_stream(0, factory) - # Then the generation is bumped and the stream is replaced + # Then the generation is bumped, broken stream is not marked clean, and stream is replaced + old_stream._mark_closing_cleanly.assert_not_called() assert mux.stream_generation == 1 assert mux._stream is new_stream factory.assert_called_once() @@ -560,6 +562,7 @@ class TestClose: async def test_close_cancels_recv_loop(self): # Given a multiplexer with an active receive loop mock_stream = AsyncMock() + mock_stream._mark_closing_cleanly = MagicMock() mock_stream.recv = AsyncMock(side_effect=asyncio.Event().wait) mux = _StreamMultiplexer(mock_stream) mux.register({1}) @@ -569,7 +572,8 @@ async def test_close_cancels_recv_loop(self): # When closing the multiplexer await mux.close() - # Then the receive loop task is cancelled + # Then the stream is marked clean and the receive loop task is cancelled + mock_stream._mark_closing_cleanly.assert_called_once() assert recv_task.cancelled() @pytest.mark.asyncio diff --git a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py index 966cb8a1c78d..2e10d433cbe5 100644 --- a/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py +++ b/packages/google-cloud-storage/tests/unit/test__opentelemetry_metrics.py @@ -12,13 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio +import concurrent.futures +import http.client import importlib +import socket +import ssl import sys +import types from unittest import mock import pytest +import requests.exceptions +import urllib3.exceptions +from google.api_core import exceptions as api_exceptions +from google.auth import exceptions as auth_exceptions from google.cloud.storage import _opentelemetry_metrics +from google.cloud.storage import exceptions as storage_exceptions from google.cloud.storage.version import __version__ @@ -46,6 +57,8 @@ def test_opentelemetry_import_error_on_load(monkeypatch): is False ) assert _opentelemetry_metrics.get_meter(enable_metrics=True) is None + m.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + assert _opentelemetry_metrics.get_meter(enable_metrics=True) is None finally: importlib.reload(_opentelemetry_metrics) @@ -144,7 +157,6 @@ def test_client_setting_overrides_env_var(monkeypatch): def test_non_boolean_client_setting_raises_type_error(): """Passing a non-boolean setting such as 'false' must raise TypeError.""" - # Argument validation executes before the _ENABLE_METRICS_DEV_GATE check. with pytest.raises(TypeError, match="enable_metrics must be a boolean or None"): _opentelemetry_metrics.is_metrics_enabled(enable_metrics="false") # type: ignore[arg-type] @@ -218,9 +230,14 @@ def test_otel_missing_disables_metrics(monkeypatch): def test_get_common_attributes(): """Verify common attributes conform to GCS OTel specification.""" attrs = _opentelemetry_metrics.get_common_attributes() + assert set(attrs.keys()) == { + "gcp.client.service", + "gcp.client.version", + "gcp.client.artifact", + } + assert "gcp.client.repo" not in attrs assert attrs["gcp.client.service"] == "storage" assert attrs["gcp.client.version"] == __version__ - assert attrs["gcp.client.repo"] == "googleapis/google-cloud-python" assert attrs["gcp.client.artifact"] == "google-cloud-storage" attrs["gcp.client.service"] = "mutated" @@ -287,3 +304,1302 @@ def test_get_meter(monkeypatch): __version__, meter_provider=mock_provider, ) + + +# --------------------------------------------------------------------------- +# Histogram Bucket Boundaries & Views Tests +# --------------------------------------------------------------------------- +def test_histogram_bucket_boundaries(): + """Verify custom latency and byte-size bucket boundaries match specification.""" + latency = _opentelemetry_metrics.LATENCY_HISTOGRAM_BOUNDARIES + assert len(latency) == 165 + assert latency[0] == 0.0 + assert latency[1] == 0.002 + assert latency[49] == 0.098 + assert latency[50] == 0.1 + assert latency[60] == 0.2 + assert latency[-1] < 300.0 + assert list(latency) == sorted(latency) + + sizes = _opentelemetry_metrics.SIZE_HISTOGRAM_BOUNDARIES + assert len(sizes) == 48 + assert sizes[0] == 0 + assert sizes[1] == 128 * 1024 + assert sizes[32] == 4 * 1024 * 1024 + assert sizes[33] == int(4.25 * 1024 * 1024) + assert sizes[34] == int(4.75 * 1024 * 1024) + assert sizes[-1] == 8593866752 + assert list(sizes) == sorted(sizes) + + +def test_get_histogram_views_with_and_without_sdk(monkeypatch): + """Verify get_histogram_views builds 6 Views when SDK is installed and [] otherwise.""" + fake_view_mod = types.ModuleType("opentelemetry.sdk.metrics.view") + + class FakeAggregation: + def __init__(self, boundaries): + self.boundaries = boundaries + + class FakeView: + def __init__(self, instrument_name, aggregation, meter_name=None): + self.instrument_name = instrument_name + self.aggregation = aggregation + self.meter_name = meter_name + + fake_view_mod.ExplicitBucketHistogramAggregation = FakeAggregation + fake_view_mod.View = FakeView + + with monkeypatch.context() as m: + m.setitem(sys.modules, "opentelemetry", types.ModuleType("opentelemetry")) + m.setitem( + sys.modules, "opentelemetry.sdk", types.ModuleType("opentelemetry.sdk") + ) + m.setitem( + sys.modules, + "opentelemetry.sdk.metrics", + types.ModuleType("opentelemetry.sdk.metrics"), + ) + m.setitem(sys.modules, "opentelemetry.sdk.metrics.view", fake_view_mod) + + views = _opentelemetry_metrics.get_histogram_views() + assert len(views) == 6 + assert all(v.meter_name == "google.cloud.storage" for v in views) + latency_views = [ + v + for v in views + if v.instrument_name in _opentelemetry_metrics._LATENCY_METRIC_NAMES + ] + size_views = [ + v + for v in views + if v.instrument_name in _opentelemetry_metrics._SIZE_METRIC_NAMES + ] + assert len(latency_views) == 4 + assert len(size_views) == 2 + assert ( + latency_views[0].aggregation.boundaries + == _opentelemetry_metrics.LATENCY_HISTOGRAM_BOUNDARIES + ) + assert ( + size_views[0].aggregation.boundaries + == _opentelemetry_metrics.SIZE_HISTOGRAM_BOUNDARIES + ) + + # Without opentelemetry-sdk installed + with monkeypatch.context() as m: + m.setitem(sys.modules, "opentelemetry.sdk.metrics.view", None) + assert _opentelemetry_metrics.get_histogram_views() == [] + + +# --------------------------------------------------------------------------- +# Error Classification (`classify_error_type`) Tests +# --------------------------------------------------------------------------- +def test_classify_error_type_none_and_status_codes(): + assert _opentelemetry_metrics.classify_error_type() == "OK" + assert _opentelemetry_metrics.classify_error_type(None, status_code=200) == "OK" + assert _opentelemetry_metrics.classify_error_type(None, status_code=206) == "OK" + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=True) == "OK" # type: ignore[arg-type] + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=400) + == "INVALID_ARGUMENT" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=401) + == "UNAUTHENTICATED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=403) + == "PERMISSION_DENIED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=404) == "NOT_FOUND" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=408) + == "DEADLINE_EXCEEDED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=409) == "ABORTED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=412) + == "FAILED_PRECONDITION" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=416) + == "OUT_OF_RANGE" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=429) + == "RESOURCE_EXHAUSTED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=499) == "CANCELLED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=500) == "INTERNAL" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=501) + == "UNIMPLEMENTED" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=503) + == "UNAVAILABLE" + ) + assert ( + _opentelemetry_metrics.classify_error_type(None, status_code=504) + == "DEADLINE_EXCEEDED" + ) + assert _opentelemetry_metrics.classify_error_type(None, status_code=502) == "502" + assert _opentelemetry_metrics.classify_error_type(None, status_code=99999) == "OK" + + +def test_classify_error_type_exceptions(): + # 1. Checksum Mismatch (including subclasses of DataCorruption) + resp_mock = mock.Mock(status_code=200) + assert ( + _opentelemetry_metrics.classify_error_type( + storage_exceptions.DataCorruption(resp_mock, "crc mismatch") + ) + == "CHECKSUM_MISMATCH" + ) + + class CustomDataCorruption(storage_exceptions.DataCorruption): + pass + + assert ( + _opentelemetry_metrics.classify_error_type( + CustomDataCorruption(resp_mock, "subclass mismatch") + ) + == "CHECKSUM_MISMATCH" + ) + + # 2. Authentication Errors + assert ( + _opentelemetry_metrics.classify_error_type( + auth_exceptions.RefreshError("token expired") + ) + == "AUTHENTICATION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + auth_exceptions.TransportError("auth transport error") + ) + == "AUTHENTICATION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + auth_exceptions.DefaultCredentialsError("missing adc") + ) + == "AUTHENTICATION_ERROR" + ) + + # 3. Cancellation + assert ( + _opentelemetry_metrics.classify_error_type(asyncio.CancelledError()) + == "CANCELLED" + ) + + # 4. HTTP Conflict (409) consistency with status_code=409 + conflict_exc = api_exceptions.Conflict("conflict") + assert _opentelemetry_metrics.classify_error_type(conflict_exc) == "ABORTED" + + # 5. gRPC / GoogleAPICallError status codes + not_found_exc = api_exceptions.NotFound("not found") + assert _opentelemetry_metrics.classify_error_type(not_found_exc) == "NOT_FOUND" + + custom_api_exc = api_exceptions.GoogleAPICallError("custom") + custom_api_exc.grpc_status_code = "RAW_GRPC_STATUS" # type: ignore[assignment] + assert ( + _opentelemetry_metrics.classify_error_type(custom_api_exc) == "RAW_GRPC_STATUS" + ) + + # Direct grpc.RpcError-like exception with .code() method + class FakeStatusCode: + name = "DEADLINE_EXCEEDED" + + class FakeRpcError(Exception): + def code(self): + return FakeStatusCode() + + assert ( + _opentelemetry_metrics.classify_error_type(FakeRpcError()) + == "DEADLINE_EXCEEDED" + ) + + class FakeRpcErrorNoName(Exception): + def code(self): + return object() + + assert _opentelemetry_metrics.classify_error_type(FakeRpcErrorNoName()) == "UNKNOWN" + + # 6. HTTP status code on exception + http_api_exc = api_exceptions.GoogleAPICallError("bad gateway") + http_api_exc.code = 502 + http_api_exc.grpc_status_code = None + assert _opentelemetry_metrics.classify_error_type(http_api_exc) == "502" + + invalid_resp_with_code = storage_exceptions.InvalidResponse( + mock.Mock(status_code=429), "rate limited" + ) + assert ( + _opentelemetry_metrics.classify_error_type(invalid_resp_with_code) + == "RESOURCE_EXHAUSTED" + ) + + req_http_err = requests.exceptions.HTTPError(response=mock.Mock(status_code=503)) + assert _opentelemetry_metrics.classify_error_type(req_http_err) == "UNAVAILABLE" + + # 7. Timeouts + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ConnectTimeout("connect timeout") + ) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ReadTimeout("read timeout") + ) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type(socket.timeout("timed out")) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + urllib3.exceptions.ReadTimeoutError(None, "/", "read timed out") # type: ignore[arg-type] + ) + == "TIMEOUT" + ) + assert ( + _opentelemetry_metrics.classify_error_type(concurrent.futures.TimeoutError()) + == "TIMEOUT" + ) + retry_err = api_exceptions.RetryError( + "deadline exceeded", cause=socket.timeout("timed out") + ) + assert _opentelemetry_metrics.classify_error_type(retry_err) == "TIMEOUT" + + # 8. TLS / SSL Failure + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.SSLError("cert verify failed") + ) + == "TLS_FAILURE" + ) + assert ( + _opentelemetry_metrics.classify_error_type(ssl.SSLError("ssl handshake error")) + == "TLS_FAILURE" + ) + + # 9. DNS, Connection Refused, Connection Reset (direct and wrapped in requests/urllib3) + gai_err = socket.gaierror(-2, "Name or service not known") + new_conn_err = urllib3.exceptions.NewConnectionError(None, "dns failed") # type: ignore[arg-type] + new_conn_err.__cause__ = gai_err + wrapped_dns = requests.exceptions.ConnectionError( + urllib3.exceptions.MaxRetryError( + None, # type: ignore[arg-type] + "/", + reason=new_conn_err, + ) + ) + assert _opentelemetry_metrics.classify_error_type(wrapped_dns) == "DNS_FAILURE" + + refused_err = ConnectionRefusedError(111, "Connection refused") + wrapped_refused = requests.exceptions.ConnectionError(refused_err) + assert ( + _opentelemetry_metrics.classify_error_type(wrapped_refused) + == "CONNECTION_ERROR" + ) + + reset_err = ConnectionResetError(104, "Connection reset by peer") + wrapped_reset = requests.exceptions.ConnectionError(reset_err) + assert ( + _opentelemetry_metrics.classify_error_type(wrapped_reset) == "CONNECTION_ERROR" + ) + + # 10. Generic Connection Errors + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ConnectionError("connection aborted") + ) + == "CONNECTION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + requests.exceptions.ChunkedEncodingError("incomplete read") + ) + == "CONNECTION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type(OSError("network unreachable")) + == "CONNECTION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type( + http.client.IncompleteRead(b"partial", 10) + ) + == "CONNECTION_ERROR" + ) + + # 11. Explicit status_code fallback when exception is generic + assert ( + _opentelemetry_metrics.classify_error_type( + RuntimeError("unexpected"), status_code=503 + ) + == "UNAVAILABLE" + ) + + # 12. Client Validation Errors + assert ( + _opentelemetry_metrics.classify_error_type(ValueError("bad arg")) + == "CLIENT_VALIDATION_ERROR" + ) + assert ( + _opentelemetry_metrics.classify_error_type(TypeError("bad type")) + == "CLIENT_VALIDATION_ERROR" + ) + invalid_resp_no_code = storage_exceptions.InvalidResponse( + mock.Mock(status_code=None), "missing header" + ) + assert ( + _opentelemetry_metrics.classify_error_type(invalid_resp_no_code) + == "CLIENT_VALIDATION_ERROR" + ) + + # 13. Cyclic exception chain and unknown exception + e1 = RuntimeError("cycle 1") + e2 = RuntimeError("cycle 2", e1) + e1.__cause__ = e2 + assert _opentelemetry_metrics.classify_error_type(e1) == "UNKNOWN" + + +# --------------------------------------------------------------------------- +# Route Normalization (`normalize_url_template`) Tests +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "raw_url,expected", + [ + (None, "UNKNOWN"), + ("", "UNKNOWN"), + (123, "UNKNOWN"), + ("https://storage.googleapis.com", "UNKNOWN"), + ("/", "UNKNOWN"), + ("///", "UNKNOWN"), + ( + "https://storage.googleapis.com//storage/v1/b/my-bucket//", + "/storage/v1/b/{bucket}", + ), + ( + "https://storage.googleapis.com/upload/storage/v1/b/my-bucket/o/my-obj?uploadType=resumable", + "/upload/storage/v1/b/{bucket}/o", + ), + ( + "https://storage.googleapis.com/custom/path?tag=my_uploads", + "UNKNOWN", + ), + ( + "https://storage.googleapis.com/custom/path?my_uploadId=123", + "UNKNOWN", + ), + ( + "https://storage.googleapis.com/storage/v1/b?project=my-proj", + "/storage/v1/b", + ), + ("/b?project=my-proj", "/storage/v1/b"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket?projection=full", + "/storage/v1/b/{bucket}", + ), + ("/b/my-bucket/", "/storage/v1/b/{bucket}"), + ("b/my-bucket", "/storage/v1/b/{bucket}"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o?prefix=foo", + "/storage/v1/b/{bucket}/o", + ), + ("/b/my-bucket/o", "/storage/v1/b/{bucket}/o"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/folder%2Ffile.txt?generation=1", + "/storage/v1/b/{bucket}/o/{object}", + ), + ("/b/my-bucket/o/file.txt", "/storage/v1/b/{bucket}/o/{object}"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/dest.txt/compose", + "/storage/v1/b/{bucket}/o/{object}/compose", + ), + ( + "/b/my-bucket/o/dest.txt/compose", + "/storage/v1/b/{bucket}/o/{object}/compose", + ), + ( + "https://storage.googleapis.com/storage/v1/b/src-b/o/src-o/rewriteTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}", + ), + ( + "/b/src-b/o/src-o/rewriteTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/rewriteTo/b/{bucket}/o/{object}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/src-b/o/src-o/copyTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/copyTo/b/{bucket}/o/{object}", + ), + ( + "/b/src-b/o/src-o/copyTo/b/dst-b/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/copyTo/b/{bucket}/o/{object}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/src-o/moveTo/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}", + ), + ( + "/b/my-bucket/o/src-o/moveTo/o/dst-o", + "/storage/v1/b/{bucket}/o/{object}/moveTo/o/{object}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/o/my-o/restore", + "/storage/v1/b/{bucket}/o/{object}/restore", + ), + ( + "/b/my-bucket/o/my-o/restore", + "/storage/v1/b/{bucket}/o/{object}/restore", + ), + ( + "/b/my-bucket/o/my-o/acl", + "/storage/v1/b/{bucket}/o/{object}/acl", + ), + ( + "/b/my-bucket/o/my-o/acl/allUsers", + "/storage/v1/b/{bucket}/o/{object}/acl/{entity}", + ), + ( + "/b/my-bucket/o/my-o/iam", + "/storage/v1/b/{bucket}/o/{object}/iam", + ), + ( + "/b/my-bucket/o/my-o/iam/testPermissions", + "/storage/v1/b/{bucket}/o/{object}/iam", + ), + ( + "/b/my-bucket/acl", + "/storage/v1/b/{bucket}/acl", + ), + ( + "/b/my-bucket/acl/allUsers", + "/storage/v1/b/{bucket}/acl/{entity}", + ), + ( + "/b/my-bucket/defaultObjectAcl", + "/storage/v1/b/{bucket}/defaultObjectAcl", + ), + ( + "/b/my-bucket/defaultObjectAcl/allUsers", + "/storage/v1/b/{bucket}/defaultObjectAcl/{entity}", + ), + ( + "/b/my-bucket/restore", + "/storage/v1/b/{bucket}/restore", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/iam/testPermissions", + "/storage/v1/b/{bucket}/iam", + ), + ("/b/my-bucket/iam", "/storage/v1/b/{bucket}/iam"), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/notificationConfigs", + "/storage/v1/b/{bucket}/notificationConfigs", + ), + ( + "/b/my-bucket/notificationConfigs", + "/storage/v1/b/{bucket}/notificationConfigs", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/notificationConfigs/42", + "/storage/v1/b/{bucket}/notificationConfigs/{notification}", + ), + ( + "/b/my-bucket/notificationConfigs/42", + "/storage/v1/b/{bucket}/notificationConfigs/{notification}", + ), + ( + "https://storage.googleapis.com/storage/v1/b/my-bucket/lockRetentionPolicy", + "/storage/v1/b/{bucket}/lockRetentionPolicy", + ), + ( + "/b/my-bucket/lockRetentionPolicy", + "/storage/v1/b/{bucket}/lockRetentionPolicy", + ), + ( + "/storage/v1/projects/my-proj/serviceAccount", + "/storage/v1/projects/{project}/serviceAccount", + ), + ( + "/projects/my-proj/serviceAccount", + "/storage/v1/projects/{project}/serviceAccount", + ), + ( + "/storage/v1/projects/my-proj/hmacKeys", + "/storage/v1/projects/{project}/hmacKeys", + ), + ("/projects/my-proj/hmacKeys", "/storage/v1/projects/{project}/hmacKeys"), + ( + "/storage/v1/projects/my-proj/hmacKeys/GOOG123", + "/storage/v1/projects/{project}/hmacKeys/{access_id}", + ), + ( + "/projects/my-proj/hmacKeys/GOOG123", + "/storage/v1/projects/{project}/hmacKeys/{access_id}", + ), + ( + "https://storage.googleapis.com/upload/storage/v1/b/my-bucket/o?uploadType=resumable", + "/upload/storage/v1/b/{bucket}/o", + ), + ( + "https://storage.googleapis.com/download/storage/v1/b/my-bucket/o/my-obj?alt=media", + "/download/storage/v1/b/{bucket}/o/{object}", + ), + ("https://storage.googleapis.com/batch/storage/v1", "/batch/storage/v1"), + ( + "https://storage.googleapis.com/my-bucket/my-object?uploads", + "/{bucket}/{object}", + ), + ( + "https://storage.googleapis.com/my-bucket/nested/dir/object.bin?uploadId=xyz&partNumber=2", + "/{bucket}/{object}", + ), + ( + "https://storage.googleapis.com/unrecognized/two-segment", + "UNKNOWN", + ), + ("https://storage.googleapis.com/storage/v2", "UNKNOWN"), + ("https://storage.googleapis.com/storage/v2/other?uploadId=xyz", "UNKNOWN"), + ("https://storage.googleapis.com/custom/unrecognized/path/here", "UNKNOWN"), + ("https://[::1/storage/v1/b", "UNKNOWN"), + ], +) +def test_normalize_url_template(raw_url, expected): + assert _opentelemetry_metrics.normalize_url_template(raw_url) == expected + + +# --------------------------------------------------------------------------- +# Logical Operation Context (`start_operation`) Tests +# --------------------------------------------------------------------------- +def test_start_operation_disabled_and_nesting(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + # Disabled recorder -> yields None and records nothing + disabled_recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=False) + with _opentelemetry_metrics.start_operation( + disabled_recorder, "storage.Blob.reload", "storage.objects.get" + ) as op: + assert op is None + assert _opentelemetry_metrics.get_current_operation() is None + + # Enabled recorder -> outermost call records, nested call is suppressed + enabled_recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + with _opentelemetry_metrics.start_operation( + enabled_recorder, + "storage.Blob.download_to_filename", + "storage.objects.get", + ) as outer_op: + assert outer_op is not None + assert outer_op.client_method == "storage.Blob.download_to_filename" + assert outer_op.rpc_method == "storage.objects.get" + assert _opentelemetry_metrics.get_current_operation() == outer_op + + with _opentelemetry_metrics.start_operation( + enabled_recorder, + "storage.Blob.download_to_file", + "storage.objects.get", + ) as inner_op: + assert inner_op is outer_op + + assert _opentelemetry_metrics.get_current_operation() is None + enabled_recorder.client_request_duration.record.assert_called_once() + enabled_recorder.operations_counter.add.assert_called_once() + _, kwargs = enabled_recorder.operations_counter.add.call_args + assert ( + kwargs["attributes"]["gcp.client.method"] == "storage.Blob.download_to_filename" + ) + assert kwargs["attributes"]["gcp.storage.operation"] == "storage.objects.get" + assert kwargs["attributes"]["error.type"] == "OK" + + +def test_start_operation_exception_and_suppression(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + + with pytest.raises(api_exceptions.NotFound): + with _opentelemetry_metrics.start_operation( + recorder, "storage.Bucket.get_blob", "storage.objects.get" + ): + raise api_exceptions.NotFound("missing") + + assert _opentelemetry_metrics.get_current_operation() is None + _, kwargs = recorder.operations_counter.add.call_args + assert kwargs["attributes"]["error.type"] == "NOT_FOUND" + + # Suppressed outer operation (e.g. transfer_manager worker) suppresses both + # logical operation metrics and body size recording + recorder.operations_counter.add.reset_mock() + recorder.client_request_duration.record.reset_mock() + with _opentelemetry_metrics.start_operation( + None, + "storage.transfer_manager.download_many_to_path", + "storage.objects.get", + suppress=True, + ) as suppressed_op: + assert suppressed_op is not None + assert suppressed_op.suppressed is True + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.download_to_filename", + "storage.objects.get", + ): + recorder.record_request_body_size(2048) + recorder.record_response_body_size(4096) + + assert _opentelemetry_metrics.get_current_operation() is None + recorder.operations_counter.add.assert_not_called() + recorder.client_request_duration.record.assert_not_called() + recorder.request_body_size.record.assert_not_called() + recorder.response_body_size.record.assert_not_called() + + # Nested suppress=True inside an active unsuppressed operation (e.g. BlobReader prefetch) + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.open", + "storage.objects.get", + ) as active_op: + assert active_op is not None and active_op.suppressed is False + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.download_as_bytes", + "storage.objects.get", + suppress=True, + ) as child_suppressed: + assert child_suppressed is not None + assert child_suppressed.suppressed is True + assert child_suppressed.client_method == "storage.Blob.open" + # Prefetch body size and TTFB are suppressed inside child_suppressed + recorder.record_response_body_size(40 * 1024 * 1024) + recorder.record_ttfb(0.05) + # Outside the suppressed prefetch block, actual user bytes are recorded + assert _opentelemetry_metrics.get_current_operation() == active_op + recorder.record_response_body_size(1024) + recorder.record_response_body_size(True) # type: ignore[arg-type] + recorder.record_request_body_size(False) # type: ignore[arg-type] + + recorder.response_body_size.record.assert_called_once() + assert recorder.response_body_size.record.call_args[0][0] == 1024 + recorder.operation_ttfb.record.assert_not_called() + recorder.operations_counter.add.assert_called_once() + + +def test_start_operation_asyncio_concurrency_isolation(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + + async def worker(method_name: str, rpc_name: str): + with _opentelemetry_metrics.start_operation(recorder, method_name, rpc_name): + await asyncio.sleep(0.005) + current = _opentelemetry_metrics.get_current_operation() + assert current is not None + assert current.client_method == method_name + assert current.rpc_method == rpc_name + + async def run_all(): + await asyncio.gather( + worker( + "storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + "google.storage.v2.Storage/BidiReadObject", + ), + worker( + "storage.asyncio.AsyncAppendableObjectWriter.append", + "google.storage.v2.Storage/BidiWriteObject", + ), + ) + + asyncio.run(run_all()) + assert recorder.operations_counter.add.call_count == 2 + + +# --------------------------------------------------------------------------- +# `_extract_server_address` & `_MetricsRecorder` Tests +# --------------------------------------------------------------------------- +@pytest.mark.parametrize( + "endpoint,expected", + [ + (None, "storage.googleapis.com"), + ("", "storage.googleapis.com"), + (" ", "storage.googleapis.com"), + ("https://storage.googleapis.com:443", "storage.googleapis.com"), + ("storage.googleapis.com:443", "storage.googleapis.com"), + ("//storage.googleapis.com:443", "storage.googleapis.com"), + ("https://[::1]:8080", "::1"), + ("http://localhost:9023/storage/v1", "localhost"), + ("https://", "storage.googleapis.com"), + ("https://[::1", "storage.googleapis.com"), + ("[::1", "storage.googleapis.com"), + ("dns://8.8.8.8/custom.endpoint.com:443", "custom.endpoint.com"), + ("dns:custom.endpoint.com:443", "custom.endpoint.com"), + ("dns://custom.endpoint.com:443", "custom.endpoint.com"), + ("dns://8.8.8.8/", "storage.googleapis.com"), + ("ipv4:127.0.0.1:443", "127.0.0.1"), + ("ipv6:[::1]:443", "::1"), + ], +) +def test_extract_server_address(endpoint, expected): + assert _opentelemetry_metrics._extract_server_address(endpoint) == expected + + +def test_metrics_recorder_recording_and_shutdown(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + mock_provider = mock.Mock() + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, + meter_provider=mock_provider, + api_endpoint="https://custom.storage.googleapis.com:443", + rpc_system="http", + ) + assert recorder.standard_enabled is True + assert recorder.enabled is True + assert recorder.server_address == "custom.storage.googleapis.com" + + base_attrs = recorder.get_base_attributes() + assert base_attrs["server.address"] == "custom.storage.googleapis.com" + assert base_attrs["rpc.system.name"] == "http" + + # Verify exact instrument names, units, and descriptions + histogram_specs = { + call.args[0]: (call.kwargs["unit"], call.kwargs["description"]) + for call in mock_meter.create_histogram.call_args_list + } + assert histogram_specs == { + "rpc.client.call.duration": ( + "s", + "Duration of an individual gRPC attempt in seconds (excluding retries).", + ), + "http.client.request.duration": ( + "s", + "Duration of an individual HTTP request attempt in seconds (excluding retries).", + ), + "gcp.client.request.duration": ( + "s", + "End-to-end latency of a logical user-facing Veneer operation in seconds.", + ), + "gcp.storage.client.request.body.size": ( + "By", + "Application payload bytes submitted by the caller for an upload or write operation.", + ), + "gcp.storage.client.response.body.size": ( + "By", + "Application payload bytes delivered to the caller.", + ), + "gcp.storage.client.ttfb": ( + "s", + "Time elapsed from attempt initiation to receipt of the first response byte.", + ), + } + counter_specs = { + call.args[0]: (call.kwargs["unit"], call.kwargs["description"]) + for call in mock_meter.create_counter.call_args_list + } + assert counter_specs == { + "gcp.storage.client.operations": ( + "{operation}", + "Number of logical GCS client operations.", + ), + "gcp.storage.client.attempts": ( + "{attempt}", + "Number of physical network transport attempts executed.", + ), + "gcp.storage.client.retries": ( + "{retry}", + "Number of transport retries executed after a failed attempt.", + ), + "gcp.storage.client.errors": ( + "{error}", + "Number of physical network transport attempts terminating with a non-OK status.", + ), + } + + # Record within a logical operation context + with _opentelemetry_metrics.start_operation( + recorder, + "storage.Blob.upload_from_string", + "WriteObject", + rpc_method="google.storage.v2.Storage/WriteObject", + ): + recorder.record_http_attempt( + duration=0.05, + http_method="post", + url_template="/upload/storage/v1/b/{bucket}/o", + status_code=503, + error_type="UNAVAILABLE", + network_path="cloudpath", + ) + recorder.record_http_attempt( + duration=0.04, + http_method="POST", + url_template="/upload/storage/v1/b/{bucket}/o", + status_code=200, + error_type="OK", + network_path="cloudpath", + ) + recorder.record_request_body_size(1024) + recorder.record_response_body_size(512) + recorder.record_ttfb(0.01, network_path="cloudpath") + + # Logical operation metrics: duration has rpc.method; operations counter does not + recorder.client_request_duration.record.assert_called_once() + _, req_dur_kwargs = recorder.client_request_duration.record.call_args + assert req_dur_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert ( + req_dur_kwargs["attributes"]["rpc.method"] + == "google.storage.v2.Storage/WriteObject" + ) + assert ( + req_dur_kwargs["attributes"]["gcp.client.method"] + == "storage.Blob.upload_from_string" + ) + assert req_dur_kwargs["attributes"]["error.type"] == "OK" + + recorder.operations_counter.add.assert_called_once() + _, op_cnt_kwargs = recorder.operations_counter.add.call_args + assert op_cnt_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert "rpc.method" not in op_cnt_kwargs["attributes"] + + assert recorder.http_request_duration.record.call_count == 2 + _, http_dur_kwargs = recorder.http_request_duration.record.call_args_list[0] + assert http_dur_kwargs["attributes"]["http.request.method"] == "POST" + assert ( + http_dur_kwargs["attributes"]["url.template"] + == "/upload/storage/v1/b/{bucket}/o" + ) + assert http_dur_kwargs["attributes"]["http.response.status_code"] == 503 + assert http_dur_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert http_dur_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.storage.operation" not in http_dur_kwargs["attributes"] + assert "gcp.client.method" not in http_dur_kwargs["attributes"] + + assert recorder.attempts_counter.add.call_count == 2 + _, attempt_kwargs = recorder.attempts_counter.add.call_args_list[0] + assert attempt_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert attempt_kwargs["attributes"]["http.response.status_code"] == 503 + assert attempt_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert attempt_kwargs["attributes"]["network.path"] == "cloudpath" + assert "url.template" not in attempt_kwargs["attributes"] + assert "http.request.method" not in attempt_kwargs["attributes"] + assert "gcp.client.method" not in attempt_kwargs["attributes"] + + # Retry counter incremented once for the second attempt after 503 UNAVAILABLE + recorder.retries_counter.add.assert_called_once() + _, http_retry_kwargs = recorder.retries_counter.add.call_args + assert http_retry_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert http_retry_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert http_retry_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.client.method" not in http_retry_kwargs["attributes"] + + # Only the 503 attempt records an error (no gcp.client.method on errors) + recorder.errors_counter.add.assert_called_once() + _, err_kwargs = recorder.errors_counter.add.call_args + assert err_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert err_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert err_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.client.method" not in err_kwargs["attributes"] + assert "url.template" not in err_kwargs["attributes"] + assert "http.response.status_code" not in err_kwargs["attributes"] + + recorder.request_body_size.record.assert_called_once() + recorder.response_body_size.record.assert_called_once() + recorder.ttfb.record.assert_called_once() + _, ttfb_kwargs = recorder.ttfb.record.call_args + assert ttfb_kwargs["attributes"]["gcp.storage.operation"] == "WriteObject" + assert ttfb_kwargs["attributes"]["network.path"] == "cloudpath" + assert "gcp.client.method" not in ttfb_kwargs["attributes"] + + # gRPC attempt recording (UNAVAILABLE -> OK retry sequence) + recorder.errors_counter.add.reset_mock() + recorder.attempts_counter.add.reset_mock() + recorder.retries_counter.add.reset_mock() + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.download_ranges", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ): + recorder.record_rpc_attempt( + duration=0.03, + operation="ReadObject", + status_code="UNAVAILABLE", + error_type="UNAVAILABLE", + rpc_method="google.storage.v2.Storage/BidiReadObject", + network_path="directpath", + ) + recorder.record_rpc_attempt( + duration=0.02, + operation="ReadObject", + status_code="OK", + error_type="OK", + rpc_method="google.storage.v2.Storage/BidiReadObject", + network_path="directpath", + ) + assert recorder.rpc_call_duration.record.call_count == 2 + _, rpc_dur_kwargs = recorder.rpc_call_duration.record.call_args_list[1] + assert ( + rpc_dur_kwargs["attributes"]["rpc.method"] + == "google.storage.v2.Storage/BidiReadObject" + ) + assert rpc_dur_kwargs["attributes"]["rpc.response.status_code"] == "OK" + assert rpc_dur_kwargs["attributes"]["error.type"] == "OK" + assert rpc_dur_kwargs["attributes"]["network.path"] == "directpath" + assert "gcp.storage.operation" not in rpc_dur_kwargs["attributes"] + assert "gcp.client.method" not in rpc_dur_kwargs["attributes"] + + assert recorder.attempts_counter.add.call_count == 2 + _, rpc_attempt_kwargs = recorder.attempts_counter.add.call_args_list[1] + assert rpc_attempt_kwargs["attributes"]["gcp.storage.operation"] == "ReadObject" + assert rpc_attempt_kwargs["attributes"]["rpc.response.status_code"] == "OK" + assert rpc_attempt_kwargs["attributes"]["error.type"] == "OK" + assert rpc_attempt_kwargs["attributes"]["network.path"] == "directpath" + assert "gcp.client.method" not in rpc_attempt_kwargs["attributes"] + assert "rpc.method" not in rpc_attempt_kwargs["attributes"] + + recorder.errors_counter.add.assert_called_once() + _, rpc_err_kwargs = recorder.errors_counter.add.call_args + assert rpc_err_kwargs["attributes"]["gcp.storage.operation"] == "ReadObject" + assert rpc_err_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert rpc_err_kwargs["attributes"]["network.path"] == "directpath" + assert "gcp.client.method" not in rpc_err_kwargs["attributes"] + assert "rpc.response.status_code" not in rpc_err_kwargs["attributes"] + + recorder.retries_counter.add.assert_called_once() + _, rpc_retry_kwargs = recorder.retries_counter.add.call_args + assert rpc_retry_kwargs["attributes"]["gcp.storage.operation"] == "ReadObject" + assert rpc_retry_kwargs["attributes"]["error.type"] == "UNAVAILABLE" + assert rpc_retry_kwargs["attributes"]["network.path"] == "directpath" + + # Record outside of start_operation with status_code=None and callable .code() returning None + recorder.record_http_attempt( + duration=0.01, + http_method="GET", + url_template="/storage/v1/b", + status_code=None, + error_type="UNKNOWN", + ) + recorder.record_rpc_attempt( + duration=0.01, + operation="google.storage.v2.Storage/GetObject", + status_code=None, + error_type="UNKNOWN", + ) + recorder.record_request_body_size(64) + + class _NoneCodeErr(Exception): + def code(self): + return None + + class _NonTupleArgsErr(Exception): + args = 123 # type: ignore[assignment] + + assert _opentelemetry_metrics.classify_error_type(_NoneCodeErr()) == "UNKNOWN" + assert _opentelemetry_metrics.classify_error_type(_NonTupleArgsErr()) == "UNKNOWN" + assert ( + _opentelemetry_metrics.classify_error_type(concurrent.futures.CancelledError()) + == "CANCELLED" + ) + assert ( + _opentelemetry_metrics._is_subclass_by_module( + mock.MagicMock(__mro__=123), "google.cloud.storage", {"DataCorruption"} + ) + is False + ) + assert ( + _opentelemetry_metrics.classify_error_type( + urllib3.exceptions.NewConnectionError(None, "conn err") + ) + == "CONNECTION_ERROR" + ) + + # Shutdown flushes custom meter_provider and swallows force_flush errors gracefully + recorder.shutdown() + mock_provider.force_flush.assert_called_once() + + mock_provider.force_flush.side_effect = RuntimeError("flush failed") + recorder.shutdown() # Must not raise + + +def test_get_active_recorder_and_grpc_uri_schemes(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "HAS_OPENTELEMETRY_METRICS", True) + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + + assert ( + _opentelemetry_metrics._extract_server_address("dns:///custom.endpoint.com:443") + == "custom.endpoint.com" + ) + assert ( + _opentelemetry_metrics._extract_server_address( + "google-c2p-experimental:///storage.googleapis.com" + ) + == "storage.googleapis.com" + ) + + # Unconfigured MagicMock should not be treated as an active recorder + unconfigured_mock = mock.MagicMock() + assert _opentelemetry_metrics.get_active_recorder(unconfigured_mock) is None + assert _opentelemetry_metrics.get_active_recorder(None) is None + + active_rec = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + holder = mock.MagicMock() + holder._metrics_recorder = active_rec + assert _opentelemetry_metrics.get_active_recorder(holder) is active_rec + + disabled_rec = _opentelemetry_metrics._MetricsRecorder(enable_metrics=False) + holder._metrics_recorder = disabled_rec + assert _opentelemetry_metrics.get_active_recorder(holder) is None + + +def test_metrics_recorder_disabled_no_ops(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", False) + recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=True) + assert recorder.standard_enabled is False + assert recorder.enabled is False + + # All recording methods are safe no-ops when disabled + recorder.record_logical_operation(0.1, "m", "op") + recorder.record_retry("ReadObject", "UNAVAILABLE", "cloudpath") + recorder.record_http_attempt(0.1, "GET", "/storage/v1/b") + recorder.record_rpc_attempt(0.1, "google.storage.v2.Storage/ReadObject") + recorder.record_request_body_size(100) + recorder.record_response_body_size(100) + recorder.record_ttfb(0.01) + recorder.shutdown() + + +def test_normalize_logical_operation_and_resolve_rpc_method(): + assert _opentelemetry_metrics.normalize_logical_operation(None) == "UNKNOWN" + assert _opentelemetry_metrics.normalize_logical_operation("") == "UNKNOWN" + assert _opentelemetry_metrics.normalize_logical_operation(123) == "UNKNOWN" # type: ignore[arg-type] + assert _opentelemetry_metrics.normalize_logical_operation("///") == "UNKNOWN" + assert ( + _opentelemetry_metrics.normalize_logical_operation( + "google.storage.v2.Storage/BidiReadObject" + ) + == "ReadObject" + ) + assert ( + _opentelemetry_metrics.normalize_logical_operation( + "/google.storage.v2.Storage/BidiWriteObject" + ) + == "WriteObject" + ) + assert ( + _opentelemetry_metrics.normalize_logical_operation( + "google.storage.v2.Storage/GetObject" + ) + == "GetObject" + ) + assert ( + _opentelemetry_metrics.normalize_logical_operation("GetNotificationConfig") + == "GetNotificationConfig" + ) + + assert _opentelemetry_metrics.resolve_rpc_method(None, None) == "UNKNOWN" + assert _opentelemetry_metrics.resolve_rpc_method("", "") == "UNKNOWN" + assert _opentelemetry_metrics.resolve_rpc_method("///", None) == "UNKNOWN" + assert _opentelemetry_metrics.resolve_rpc_method("UNKNOWN", None) == "UNKNOWN" + assert ( + _opentelemetry_metrics.resolve_rpc_method( + "ReadObject", "google.storage.v2.Storage/BidiReadObject" + ) + == "google.storage.v2.Storage/BidiReadObject" + ) + assert ( + _opentelemetry_metrics.resolve_rpc_method("ReadObject") + == "google.storage.v2.Storage/ReadObject" + ) + assert ( + _opentelemetry_metrics.resolve_rpc_method("GetNotificationConfig") + == "google.storage.v1.Storage/GetNotificationConfig" + ) + assert ( + _opentelemetry_metrics.resolve_rpc_method("storage.objects.get") + == "storage.objects.get" + ) + + +def test_iter_exception_chain_traverses_errors_list(): + wrapped = api_exceptions.Aborted( + "aborted", + errors=[socket.timeout("nested timeout in errors")], + ) + chain = list(_opentelemetry_metrics._iter_exception_chain(wrapped)) + assert any(isinstance(item, socket.timeout) for item in chain) + + class _NonSeqErrorsErr(Exception): + errors = 123 + + assert _opentelemetry_metrics.classify_error_type(_NonSeqErrorsErr()) == "UNKNOWN" + + +def test_operation_state_and_retry_helpers(monkeypatch): + monkeypatch.setattr(_opentelemetry_metrics, "_ENABLE_METRICS_DEV_GATE", True) + mock_metrics = mock.Mock() + mock_meter = mock.Mock() + mock_meter.create_histogram.side_effect = lambda *a, **kw: mock.Mock() + mock_meter.create_counter.side_effect = lambda *a, **kw: mock.Mock() + mock_metrics.get_meter.return_value = mock_meter + monkeypatch.setattr(_opentelemetry_metrics, "metrics", mock_metrics, raising=False) + + recorder = _opentelemetry_metrics._MetricsRecorder( + enable_metrics=True, rpc_system="grpc" + ) + disabled_recorder = _opentelemetry_metrics._MetricsRecorder(enable_metrics=False) + + # No-ops when recorder is None, disabled, or no active operation + _opentelemetry_metrics.record_pending_retry(None) + _opentelemetry_metrics.record_pending_retry(disabled_recorder) + _opentelemetry_metrics.record_pending_retry(recorder) + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + None, RuntimeError("e"), attempts_before=0, will_retry=True + ) + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + disabled_recorder, RuntimeError("e"), attempts_before=0, will_retry=True + ) + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, RuntimeError("e"), attempts_before=0, will_retry=True + ) + + # Standalone record_retry and record_ttfb outside start_operation without optional attrs + recorder.record_retry(operation=None, error_type="UNAVAILABLE", network_path=None) + _, retry_no_op_kw = recorder.retries_counter.add.call_args + assert "gcp.storage.operation" not in retry_no_op_kw["attributes"] + assert "network.path" not in retry_no_op_kw["attributes"] + + recorder.record_ttfb(0.02, operation=None, network_path=None) + _, ttfb_no_op_kw = recorder.ttfb.record.call_args + assert "gcp.storage.operation" not in ttfb_no_op_kw["attributes"] + assert "network.path" not in ttfb_no_op_kw["attributes"] + + recorder.retries_counter.add.reset_mock() + with _opentelemetry_metrics.start_operation( + recorder, + "storage.asyncio.AsyncMultiRangeDownloader.open", + "ReadObject", + rpc_method="google.storage.v2.Storage/BidiReadObject", + ) as op_state: + assert op_state is not None + op_state.mark_attempt_failed("OK") + op_state.mark_attempt_failed("") + assert op_state.last_failure_error_type is None + + # Pending retry when no failure occurred is a no-op + _opentelemetry_metrics.record_pending_retry(recorder) + recorder.retries_counter.add.assert_not_called() + + # When transport already recorded the attempt, handle_unrecorded_retryable_failure skips + op_state.recorded_attempts = 1 + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.ServiceUnavailable("503"), + attempts_before=0, + will_retry=True, + network_path="directpath", + ) + assert op_state.recorded_attempts == 1 + assert op_state.last_failure_error_type is None + + # Unrecorded failure 1 (will_retry=True) -> marks failure, no retry yet + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.ServiceUnavailable("503"), + attempts_before=1, + will_retry=True, + network_path="unknown", + ) + assert op_state.recorded_attempts == 2 + assert op_state.last_failure_error_type == "UNAVAILABLE" + recorder.retries_counter.add.assert_not_called() + + # Consecutive unrecorded failure 2 (will_retry=True) -> emits retry for failure 1, marks failure 2 + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.DeadlineExceeded("timeout"), + attempts_before=2, + will_retry=True, + network_path="cloudpath", + ) + assert op_state.recorded_attempts == 3 + assert op_state.last_failure_error_type == "DEADLINE_EXCEEDED" + recorder.retries_counter.add.assert_called_once() + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "UNAVAILABLE" + ) + + # Unrecorded terminal failure 3 (will_retry=False) -> emits retry for failure 2, leaves error cleared + _opentelemetry_metrics.handle_unrecorded_retryable_failure( + recorder, + api_exceptions.PermissionDenied("403"), + attempts_before=3, + will_retry=False, + network_path="cloudpath", + ) + assert op_state.recorded_attempts == 4 + assert op_state.last_failure_error_type is None + assert recorder.retries_counter.add.call_count == 2 + assert ( + recorder.retries_counter.add.call_args.kwargs["attributes"]["error.type"] + == "DEADLINE_EXCEEDED" + ) + + # Mark failure and consume via record_pending_retry + op_state.mark_attempt_failed("ABORTED") + _opentelemetry_metrics.record_pending_retry(recorder, network_path="directpath") + assert recorder.retries_counter.add.call_count == 3 + _, pending_kw = recorder.retries_counter.add.call_args + assert pending_kw["attributes"]["gcp.storage.operation"] == "ReadObject" + assert pending_kw["attributes"]["error.type"] == "ABORTED" + assert pending_kw["attributes"]["network.path"] == "directpath" + + import contextvars + + copied_ctx = contextvars.copy_context() + + # Once start_operation exits, state.finished is True even in copied contexts + assert op_state.finished is True + assert copied_ctx.run(_opentelemetry_metrics.get_current_operation) is None