feat(gax): implement operation-level client request spans (T3)

sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/ApiTracerContext.java

@@ -268,6 +268,18 @@ Map<String, Object> getMetricsAttributes() {
     return attributes;
   }
 
+  /**
+   * @return a map of attributes to be included in operation-level spans
+   */
+  Map<String, Object> getOperationAttributes() {
+    Map<String, Object> attributes = getMetricsAttributes();
+    if (!Strings.isNullOrEmpty(destinationResourceId())) {
+      attributes.put(
+          ObservabilityAttributes.DESTINATION_RESOURCE_ID_ATTRIBUTE, destinationResourceId());
+    }
+    return attributes;
+  }
+
   /**
    * Merges this context with another context. The values in the other context take precedence.
    *

sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java

@@ -35,7 +35,10 @@
 import io.opentelemetry.api.trace.Span;
 import io.opentelemetry.api.trace.SpanBuilder;
 import io.opentelemetry.api.trace.SpanKind;
+import io.opentelemetry.api.trace.StatusCode;
 import io.opentelemetry.api.trace.Tracer;
+import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
+import io.opentelemetry.context.Context;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.CancellationException;
@@ -51,15 +54,24 @@
   private final Tracer tracer;
   private final Map<String, Object> attemptAttributes;
   private final String attemptSpanName;
+  private final String operationSpanName;
   private final ApiTracerContext apiTracerContext;
+  // Captures the active trace context from the calling thread at RPC initiation.
+  // This allows the operation span and attempt spans to link back to the caller's trace.
+  private final Context parentContext;
+  // Trace context containing the operationSpan, serving as the parent for attempt spans.
+  private final Context operationContext;
+  private @Nullable Span operationSpan;
   private @Nullable Span attemptSpan;
 
   @Override
-  public void injectTraceContext(java.util.Map<String, String> carrier) {
-    if (attemptSpan != null) {
-      io.opentelemetry.context.Context context =
-          io.opentelemetry.context.Context.current().with(attemptSpan);
-      io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator.getInstance()
+  public void injectTraceContext(Map<String, String> carrier) {
+    // Prefer the active attempt span so outgoing RPC wire context reflects the specific attempt;
+    // fall back to the overall operation span if no attempt is currently in-flight.
+    Span currentSpan = attemptSpan != null ? attemptSpan : operationSpan;
+    if (currentSpan != null) {
+      Context context = Context.current().with(currentSpan);
+      W3CTraceContextPropagator.getInstance()
           .inject(
               context,
               carrier,
@@ -78,11 +90,7 @@
    * @param apiTracerContext the {@link ApiTracerContext} to use for recording spans
    */
   OpenTelemetryTracingTracer(Tracer tracer, ApiTracerContext apiTracerContext) {
-    this.tracer = tracer;
-    this.apiTracerContext = apiTracerContext;
-    this.attemptSpanName = resolveAttemptSpanName(apiTracerContext);
-    this.attemptAttributes = new HashMap<>();
-    buildAttributes();
+    this(tracer, apiTracerContext, resolveAttemptSpanName(apiTracerContext));
   }
 
   /**
@@ -97,12 +105,53 @@
   OpenTelemetryTracingTracer(
       Tracer tracer, ApiTracerContext apiTracerContext, String attemptSpanName) {
     this.tracer = tracer;
-    this.attemptSpanName = attemptSpanName;
     this.apiTracerContext = apiTracerContext;
+    this.attemptSpanName = attemptSpanName;
+    this.operationSpanName = resolveOperationSpanName(attemptSpanName);
     this.attemptAttributes = new HashMap<>();
+    this.parentContext = Context.current();
     buildAttributes();
+    this.operationSpan = startOperationSpan();
+    this.operationContext = parentContext.with(this.operationSpan);
+  }
+
+  /**
+   * Starts and initializes the operation-level client request span.
+   *
+   * @return the newly started {@link Span} for the overall operation
+   */
+  private Span startOperationSpan() {
+    SpanBuilder operationSpanBuilder = tracer.spanBuilder(operationSpanName);
+    operationSpanBuilder.setSpanKind(SpanKind.INTERNAL);
+    operationSpanBuilder.setParent(parentContext);
+    operationSpanBuilder.setAllAttributes(
+        ObservabilityUtils.toOtelAttributes(this.apiTracerContext.getOperationAttributes()));
+    return operationSpanBuilder.startSpan();
+  }
+
+  /**
+   * Derives the operation-level span name from the attempt span name.
+   *
+   * @param attemptSpanName the attempt span name
+   * @return the operation span name
+   */
+  private static String resolveOperationSpanName(String attemptSpanName) {
+    if (!Strings.isNullOrEmpty(attemptSpanName)) {
+      if (attemptSpanName.endsWith("/attempt")) {
+        String name = attemptSpanName.substring(0, attemptSpanName.length() - "/attempt".length());
+        return name.isEmpty() ? "operation" : name;
+      }
+      return "attempt".equals(attemptSpanName) ? "operation" : attemptSpanName;
+    }
+    return "operation";
   }
 
+  /**
+   * Resolves the canonical attempt-level span name based on transport and context.
+   *
+   * @param apiTracerContext the tracer context containing transport and method metadata
+   * @return the attempt span name
+   */
   private static String resolveAttemptSpanName(ApiTracerContext apiTracerContext) {
     if (apiTracerContext.transport() == ApiTracerContext.Transport.GRPC) {
       // gRPC Uses the full method name as span name.
@@ -118,6 +167,7 @@
     }
   }
 
+  /** Copies attempt-level attributes from the tracer context into the local attribute cache. */
   private void buildAttributes() {
     this.attemptAttributes.putAll(this.apiTracerContext.getAttemptAttributes());
   }
@@ -142,19 +192,72 @@
     // Attempt spans are of the CLIENT kind
     spanBuilder.setSpanKind(SpanKind.CLIENT);
 
+    // Link attempt span to operation context (parent operation span)
+    spanBuilder.setParent(operationContext);
+
     // Pass the combined attributes to the new SpanBuilder method
     spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes));
 
     this.attemptSpan = spanBuilder.startSpan();
   }
 
+  /**
+   * Signals that the overall logical operation succeeded.
+   *
+   * <p>Closes any remaining in-flight attempt span and ends the operation span.
+   */
+  @Override
+  public void operationSucceeded() {
+    recordErrorAndEndOperation(null);
+  }
+
+  /**
+   * Signals that the overall logical operation was cancelled.
+   *
+   * <p>Closes any remaining in-flight attempt span with a {@link CancellationException} and ends
+   * the operation span with an ERROR status.
+   */
+  @Override
+  public void operationCancelled() {
+    recordErrorAndEndOperation(new CancellationException());
+  }
+
+  /**
+   * Signals that the overall logical operation failed permanently.
+   *
+   * <p>Closes any remaining in-flight attempt span with the provided error details and ends the
+   * operation span with an ERROR status.
+   *
+   * @param error the cause of the operation failure
+   */
+  @Override
+  public void operationFailed(Throwable error) {
+    recordErrorAndEndOperation(error);
+  }
+
+  /**
+   * Records error details and ends both the active attempt span and the operation span.
+   *
+   * @param error the exception associated with the operation failure, or {@code null} if successful
+   */
+  private void recordErrorAndEndOperation(@Nullable Throwable error) {
+    if (attemptSpan != null) {
+      endSpan(attemptSpan, error);
+      attemptSpan = null;
+    }
+    if (operationSpan != null) {
+      endSpan(operationSpan, error);
+      operationSpan = null;
+    }
+  }
+
   @Override
   public void attemptSucceeded() {
     recordErrorAndEndAttempt(null);
   }
 
   @Override
-  public void responseHeadersReceived(java.util.Map<String, Object> headers) {
+  public void responseHeadersReceived(Map<String, Object> headers) {
     if (attemptSpan == null) {
       return;
     }
@@ -174,7 +277,7 @@
    * @param headers the map of response headers.
    * @return the content length in bytes, or -1 if the header is missing or malformed.
    */
-  private long extractContentLength(java.util.Map<String, Object> headers) {
+  private long extractContentLength(Map<String, Object> headers) {
     try {
       if (headers == null || headers.isEmpty()) return -1;
       // google-http-client HttpHeaders uses a case-insensitive map but we copy it for safety
@@ -215,31 +318,40 @@
     recordErrorAndEndAttempt(error);
   }
 
+  /**
+   * Records error details and ends the current attempt span.
+   *
+   * @param error the exception associated with the attempt failure, or {@code null} if successful
+   */
   private void recordErrorAndEndAttempt(@Nullable Throwable error) {
     if (attemptSpan == null) {
       return;
     }
+    endSpan(attemptSpan, error);
+    attemptSpan = null;
+  }
+
+  /**
+   * Attaches response status attributes and error messages to the span and ends it.
+   *
+   * @param span the span to finish
+   * @param error the exception that caused the span to end, or {@code null} if successful
+   */
+  private void endSpan(Span span, @Nullable Throwable error) {
     Map<String, Object> responseAttributes =
         ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport());
     if (!responseAttributes.isEmpty()) {
-      attemptSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes));
+      span.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes));
     }
 
-    if (error != null && !Strings.isNullOrEmpty(error.getMessage())) {
-      attemptSpan.setAttribute(
-          ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage());
-    }
-
-    endAttempt();
-  }
-
-  private void endAttempt() {
-    if (attemptSpan == null) {
-      return;
+    if (error != null) {
+      span.setStatus(StatusCode.ERROR);
+      if (!Strings.isNullOrEmpty(error.getMessage())) {
+        span.setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage());
+      }
     }
 
-    attemptSpan.end();
-    attemptSpan = null;
+    span.end();
   }
 
   @Override

sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/ApiTracerContextTest.java

@@ -529,4 +529,43 @@ void testDestinationResourceId_evaluatedEveryTime() {
     assertThat(result.destinationResourceId()).isEqualTo("extracted-id-2");
     assertThat(counter[0]).isEqualTo(2);
   }
+
+  @Test
+  void testGetOperationAttributes() {
+    ApiTracerContext context =
+        ApiTracerContext.newBuilder()
+            .setLibraryMetadata(LibraryMetadata.newBuilder().setRepository("test-repo").build())
+            .setServerAddress("test-address")
+            .setServerPort(443)
+            .setServiceName("test-service")
+            .setTransport(ApiTracerContext.Transport.HTTP)
+            .setHttpMethod("POST")
+            .setHttpPathTemplate("/v1/{name}:access")
+            .setFullMethodName("google.test.v1.TestService/Access")
+            .setUrlDomain("test.googleapis.com")
+            .setDestinationResourceIdSupplier(() -> "projects/p/secrets/s")
+            .build();
+
+    Map<String, Object> attributes = context.getOperationAttributes();
+
+    assertThat(attributes)
+        .containsEntry(ObservabilityAttributes.SERVER_ADDRESS_ATTRIBUTE, "test-address");
+    assertThat(attributes).containsEntry(ObservabilityAttributes.SERVER_PORT_ATTRIBUTE, 443);
+    assertThat(attributes).containsEntry(ObservabilityAttributes.REPO_ATTRIBUTE, "test-repo");
+    assertThat(attributes)
+        .containsEntry(ObservabilityAttributes.GCP_CLIENT_SERVICE_ATTRIBUTE, "test-service");
+    assertThat(attributes).containsEntry(ObservabilityAttributes.RPC_SYSTEM_NAME_ATTRIBUTE, "http");
+    assertThat(attributes)
+        .containsEntry(
+            ObservabilityAttributes.GRPC_RPC_METHOD_ATTRIBUTE, "google.test.v1.TestService/Access");
+    assertThat(attributes)
+        .containsEntry(ObservabilityAttributes.URL_DOMAIN_ATTRIBUTE, "test.googleapis.com");
+    assertThat(attributes)
+        .containsEntry(ObservabilityAttributes.URL_TEMPLATE_ATTRIBUTE, "/v1/{name}:access");
+    assertThat(attributes)
+        .containsEntry(
+            ObservabilityAttributes.DESTINATION_RESOURCE_ID_ATTRIBUTE,
+            "//test.googleapis.com/projects/p/secrets/s");
+    assertThat(attributes).doesNotContainKey(ObservabilityAttributes.HTTP_METHOD_ATTRIBUTE);
+  }
 }

sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java

@@ -75,6 +75,7 @@ void setUp() {
     when(openTelemetry.getTracer(anyString())).thenReturn(tracer);
     when(tracer.spanBuilder(anyString())).thenReturn(spanBuilder);
     when(spanBuilder.setSpanKind(any())).thenReturn(spanBuilder);
+    when(spanBuilder.setParent(any())).thenReturn(spanBuilder);
     when(spanBuilder.setAllAttributes(any(Attributes.class))).thenReturn(spanBuilder);
     when(spanBuilder.startSpan()).thenReturn(span);
 
@@ -228,7 +229,7 @@ void testNewTracer_withContext_grpc_usesFullMethodName() {
 
     tracerInstance.attemptStarted(null, 1);
 
-    verify(tracer).spanBuilder("google.cloud.v1.Service/Method");
+    verify(tracer, atLeastOnce()).spanBuilder("google.cloud.v1.Service/Method");
   }
 
   @ParameterizedTest
@@ -255,7 +256,7 @@ void testNewTracer_withContext_http_usesHttpMethodAndPathTemplate(
 
     tracerInstance.attemptStarted(null, 1);
 
-    verify(tracer).spanBuilder(expectedSpanName);
+    verify(tracer, atLeastOnce()).spanBuilder(expectedSpanName);
   }
 
   @Test
@@ -273,7 +274,7 @@ void testNewTracer_withContext_http_noHttpMethodOrPathTemplate_usesFullMethodNam
 
     tracerInstance.attemptStarted(null, 1);
 
-    verify(tracer).spanBuilder("google.cloud.v1.Service.Method");
+    verify(tracer, atLeastOnce()).spanBuilder("google.cloud.v1.Service.Method");
   }
 
   @Test
@@ -309,7 +310,7 @@ void testNewTracer_mergesFactoryContext() {
     tracerInstance.attemptStarted(null, 1);
 
     ArgumentCaptor<Attributes> attributesCaptor = ArgumentCaptor.forClass(Attributes.class);
-    verify(spanBuilder).setAllAttributes(attributesCaptor.capture());
+    verify(spanBuilder, atLeastOnce()).setAllAttributes(attributesCaptor.capture());
 
     Attributes attributes = attributesCaptor.getValue();
     assertThat(attributes.asMap())