From ca57ee496a45030cbdb962b2b6664ceeeba98dea Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 25 Sep 2026 13:09:38 -0400 Subject: [PATCH 01/22] feat(gax): link T3 attempt spans to parent context and clean up on operation completion --- .../tracing/OpenTelemetryTracingTracer.java | 114 ++++++++++++------ ...OpenTelemetryTracingTracerFactoryTest.java | 1 + .../OpenTelemetryTracingTracerTest.java | 68 ++++++++++- 3 files changed, 143 insertions(+), 40 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index cb41da9ccbeb..dd6e5144c918 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -52,7 +52,10 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final Map attemptAttributes; private final String attemptSpanName; private final ApiTracerContext apiTracerContext; - private @Nullable Span attemptSpan; + private final io.opentelemetry.context.Context parentContext; + private final java.util.concurrent.locks.ReentrantLock lock = + new java.util.concurrent.locks.ReentrantLock(); + private volatile @Nullable Span attemptSpan; @Override public void injectTraceContext(java.util.Map carrier) { @@ -82,6 +85,7 @@ public void injectTraceContext(java.util.Map carrier) { this.apiTracerContext = apiTracerContext; this.attemptSpanName = resolveAttemptSpanName(apiTracerContext); this.attemptAttributes = new HashMap<>(); + this.parentContext = io.opentelemetry.context.Context.current(); buildAttributes(); } @@ -100,6 +104,7 @@ public void injectTraceContext(java.util.Map carrier) { this.attemptSpanName = attemptSpanName; this.apiTracerContext = apiTracerContext; this.attemptAttributes = new HashMap<>(); + this.parentContext = io.opentelemetry.context.Context.current(); buildAttributes(); } @@ -124,28 +129,59 @@ private void buildAttributes() { @Override public void attemptStarted(Object request, int attemptNumber) { - Map currentAttemptAttributes = new HashMap<>(this.attemptAttributes); - - if (attemptNumber > 0) { - ApiTracerContext.Transport transport = apiTracerContext.transport(); - if (transport == ApiTracerContext.Transport.GRPC) { - currentAttemptAttributes.put( - ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber); - } else if (transport == ApiTracerContext.Transport.HTTP) { - currentAttemptAttributes.put( - ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber); + Span oldSpan = null; + lock.lock(); + try { + if (attemptSpan != null) { + oldSpan = attemptSpan; + attemptSpan = null; + } + Map currentAttemptAttributes = new HashMap<>(this.attemptAttributes); + + if (attemptNumber > 0) { + ApiTracerContext.Transport transport = apiTracerContext.transport(); + if (transport == ApiTracerContext.Transport.GRPC) { + currentAttemptAttributes.put( + ObservabilityAttributes.GRPC_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber); + } else if (transport == ApiTracerContext.Transport.HTTP) { + currentAttemptAttributes.put( + ObservabilityAttributes.HTTP_RESEND_COUNT_ATTRIBUTE, (long) attemptNumber); + } } - } - SpanBuilder spanBuilder = tracer.spanBuilder(attemptSpanName); + SpanBuilder spanBuilder = tracer.spanBuilder(attemptSpanName); - // Attempt spans are of the CLIENT kind - spanBuilder.setSpanKind(SpanKind.CLIENT); + // Attempt spans are of the CLIENT kind + spanBuilder.setSpanKind(SpanKind.CLIENT); - // Pass the combined attributes to the new SpanBuilder method - spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes)); + // Link attempt span to parent context + spanBuilder.setParent(parentContext); + + // Pass the combined attributes to the new SpanBuilder method + spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes)); + + this.attemptSpan = spanBuilder.startSpan(); + } finally { + lock.unlock(); + } + if (oldSpan != null) { + endAttemptSpan(oldSpan, null); + } + } + + @Override + public void operationSucceeded() { + recordErrorAndEndAttempt(null); + } + + @Override + public void operationCancelled() { + recordErrorAndEndAttempt(new CancellationException()); + } - this.attemptSpan = spanBuilder.startSpan(); + @Override + public void operationFailed(Throwable error) { + recordErrorAndEndAttempt(error); } @Override @@ -155,12 +191,13 @@ public void attemptSucceeded() { @Override public void responseHeadersReceived(java.util.Map headers) { - if (attemptSpan == null) { + Span currentSpan = attemptSpan; + if (currentSpan == null) { return; } long contentLength = extractContentLength(headers); if (contentLength >= 0) { - attemptSpan.setAttribute(ObservabilityAttributes.HTTP_RESPONSE_BODY_SIZE, contentLength); + currentSpan.setAttribute(ObservabilityAttributes.HTTP_RESPONSE_BODY_SIZE, contentLength); } } @@ -216,41 +253,46 @@ public void attemptPermanentFailure(Throwable error) { } private void recordErrorAndEndAttempt(@Nullable Throwable error) { - if (attemptSpan == null) { - return; + Span localAttemptSpan; + lock.lock(); + try { + localAttemptSpan = attemptSpan; + if (localAttemptSpan == null) { + return; + } + attemptSpan = null; + } finally { + lock.unlock(); } + + endAttemptSpan(localAttemptSpan, error); + } + + private void endAttemptSpan(Span localAttemptSpan, @Nullable Throwable error) { Map responseAttributes = ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport()); if (!responseAttributes.isEmpty()) { - attemptSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); + localAttemptSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); } if (error != null && !Strings.isNullOrEmpty(error.getMessage())) { - attemptSpan.setAttribute( + localAttemptSpan.setAttribute( ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage()); } - endAttempt(); - } - - private void endAttempt() { - if (attemptSpan == null) { - return; - } - - attemptSpan.end(); - attemptSpan = null; + localAttemptSpan.end(); } @Override public void requestUrlResolved(String url) { - if (attemptSpan == null) { + Span currentSpan = attemptSpan; + if (currentSpan == null) { return; } String sanitizedUrlString = ObservabilityUtils.sanitizeUrlFull(url); if (sanitizedUrlString.isEmpty()) { return; } - attemptSpan.setAttribute(ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE, sanitizedUrlString); + currentSpan.setAttribute(ObservabilityAttributes.HTTP_URL_FULL_ATTRIBUTE, sanitizedUrlString); } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java index 3c78cef6dbd7..de7941522587 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java @@ -75,6 +75,7 @@ void setUp() { when(openTelemetry.getTracer(anyString())).thenReturn(tracer); when(tracer.spanBuilder(anyString())).thenReturn(spanBuilder); when(spanBuilder.setSpanKind(any())).thenReturn(spanBuilder); + when(spanBuilder.setParent(any())).thenReturn(spanBuilder); when(spanBuilder.setAllAttributes(any(Attributes.class))).thenReturn(spanBuilder); when(spanBuilder.startSpan()).thenReturn(span); diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 33fa2efcc0da..41c7627cebe6 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -33,6 +33,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; +import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -69,10 +71,11 @@ class OpenTelemetryTracingTracerTest { @BeforeEach void setUp() { - when(tracer.spanBuilder(anyString())).thenReturn(spanBuilder); - when(spanBuilder.setSpanKind(any(SpanKind.class))).thenReturn(spanBuilder); - when(spanBuilder.setAllAttributes(any(Attributes.class))).thenReturn(spanBuilder); - when(spanBuilder.startSpan()).thenReturn(span); + lenient().when(tracer.spanBuilder(anyString())).thenReturn(spanBuilder); + lenient().when(spanBuilder.setSpanKind(any(SpanKind.class))).thenReturn(spanBuilder); + lenient().when(spanBuilder.setParent(any())).thenReturn(spanBuilder); + lenient().when(spanBuilder.setAllAttributes(any(Attributes.class))).thenReturn(spanBuilder); + lenient().when(spanBuilder.startSpan()).thenReturn(span); openTelemetryTracingTracer = new OpenTelemetryTracingTracer(tracer, ApiTracerContext.empty(), ATTEMPT_SPAN_NAME); } @@ -680,4 +683,61 @@ void testInjectTraceContext_addsHeaders() { assertThat(carrier.get("traceparent")).contains("00000000000000000000000000000001"); assertThat(carrier.get("traceparent")).contains("0000000000000002"); } + + @Test + void testAttemptStarted_setsParentToParentContext() { + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + verify(spanBuilder).setParent(any(io.opentelemetry.context.Context.class)); + } + + @Test + void testOperationSucceeded_endsActiveAttemptSpan() { + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + openTelemetryTracingTracer.operationSucceeded(); + + verify(span).end(); + } + + @Test + void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + openTelemetryTracingTracer.operationFailed(new RuntimeException("operation failed")); + + verify(span).setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, "operation failed"); + verify(span).end(); + } + + @Test + void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + openTelemetryTracingTracer.operationCancelled(); + + ArgumentCaptor attrsCaptor = ArgumentCaptor.forClass(Attributes.class); + verify(span).setAllAttributes(attrsCaptor.capture()); + verify(span).end(); + + assertThat(attrsCaptor.getValue().asMap()) + .containsEntry( + AttributeKey.stringKey(ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE), + "CANCELLED"); + } + + @Test + void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { + Span span1 = mock(Span.class); + Span span2 = mock(Span.class); + + when(spanBuilder.startSpan()).thenReturn(span1, span2); + + openTelemetryTracingTracer.attemptStarted(new Object(), 0); + + // Start a second attempt before the first attempt was ended + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + verify(span1).end(); + verify(span2, never()).end(); + + // Now complete the second attempt + openTelemetryTracingTracer.attemptSucceeded(); + verify(span2).end(); + } } From d5997216d2a979ae9117a6449fd130e362542b9c Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 25 Sep 2026 13:15:34 -0400 Subject: [PATCH 02/22] fix(gax): prevent new attempt spans after operation completion --- .../gax/tracing/OpenTelemetryTracingTracer.java | 17 ++++++++++++++--- .../tracing/OpenTelemetryTracingTracerTest.java | 9 +++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index dd6e5144c918..3d4f9322da1c 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -55,6 +55,7 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final io.opentelemetry.context.Context parentContext; private final java.util.concurrent.locks.ReentrantLock lock = new java.util.concurrent.locks.ReentrantLock(); + private boolean operationCompleted; private volatile @Nullable Span attemptSpan; @Override @@ -132,6 +133,9 @@ public void attemptStarted(Object request, int attemptNumber) { Span oldSpan = null; lock.lock(); try { + if (operationCompleted) { + return; + } if (attemptSpan != null) { oldSpan = attemptSpan; attemptSpan = null; @@ -171,17 +175,17 @@ public void attemptStarted(Object request, int attemptNumber) { @Override public void operationSucceeded() { - recordErrorAndEndAttempt(null); + recordErrorAndEndAttempt(null, true); } @Override public void operationCancelled() { - recordErrorAndEndAttempt(new CancellationException()); + recordErrorAndEndAttempt(new CancellationException(), true); } @Override public void operationFailed(Throwable error) { - recordErrorAndEndAttempt(error); + recordErrorAndEndAttempt(error, true); } @Override @@ -253,9 +257,16 @@ public void attemptPermanentFailure(Throwable error) { } private void recordErrorAndEndAttempt(@Nullable Throwable error) { + recordErrorAndEndAttempt(error, false); + } + + private void recordErrorAndEndAttempt(@Nullable Throwable error, boolean isOperationComplete) { Span localAttemptSpan; lock.lock(); try { + if (isOperationComplete) { + operationCompleted = true; + } localAttemptSpan = attemptSpan; if (localAttemptSpan == null) { return; diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 41c7627cebe6..2bd2dd320ce4 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -740,4 +740,13 @@ void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { openTelemetryTracingTracer.attemptSucceeded(); verify(span2).end(); } + + @Test + void testAttemptStarted_afterOperationCompleted_doesNotStartNewSpan() { + openTelemetryTracingTracer.operationSucceeded(); + + // Attempting to start a new attempt after operation completion should be a no-op + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + verify(spanBuilder, never()).startSpan(); + } } From cda9ac6a292c7a60c15274c225b67aa18a3c0e1d Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 25 Sep 2026 13:19:22 -0400 Subject: [PATCH 03/22] fix(gax): use imported ReentrantLock --- .../google/api/gax/tracing/OpenTelemetryTracingTracer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 3d4f9322da1c..03a0b8c2a67d 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -39,6 +39,7 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.CancellationException; +import java.util.concurrent.locks.ReentrantLock; import org.jspecify.annotations.NullMarked; import org.jspecify.annotations.Nullable; @@ -53,8 +54,7 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final String attemptSpanName; private final ApiTracerContext apiTracerContext; private final io.opentelemetry.context.Context parentContext; - private final java.util.concurrent.locks.ReentrantLock lock = - new java.util.concurrent.locks.ReentrantLock(); + private final ReentrantLock lock = new ReentrantLock(); private boolean operationCompleted; private volatile @Nullable Span attemptSpan; From 18a5cfff2fad42c38c91aad13f281ce0d1908f5b Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 28 Sep 2026 13:10:50 -0400 Subject: [PATCH 04/22] chore(gax): add integration test for OpenTelemetry attempt span tracing --- ...TelemetryTracingTracerIntegrationTest.java | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java new file mode 100644 index 000000000000..31832fbc3d71 --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -0,0 +1,223 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.tracing; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.api.gax.rpc.LibraryMetadata; +import com.google.api.gax.tracing.ApiTracerContext.Transport; +import com.google.api.gax.tracing.ApiTracerFactory.OperationType; +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.context.Scope; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.testing.exporter.InMemorySpanExporter; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.util.List; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class OpenTelemetryTracingTracerIntegrationTest { + + private static final String FULL_METHOD_NAME = "google.fake.v1.FakeService/FakeMethod"; + private static final LibraryMetadata LIBRARY_METADATA = + LibraryMetadata.newBuilder() + .setRepository("googleapis/google-cloud-java") + .setArtifactName("google-cloud-fake") + .setVersion("1.0.0") + .build(); + private static final ApiTracerContext TRACER_CONTEXT = + ApiTracerContext.newBuilder() + .setFullMethodName(FULL_METHOD_NAME) + .setTransport(Transport.GRPC) + .setLibraryMetadata(LIBRARY_METADATA) + .setOperationType(OperationType.Unary) + .build(); + + private InMemorySpanExporter spanExporter; + private SdkTracerProvider tracerProvider; + private OpenTelemetrySdk openTelemetrySdk; + private Tracer tracer; + private ApiTracerFactory tracingFactory; + + @BeforeEach + void setUp() { + spanExporter = InMemorySpanExporter.create(); + tracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor(SimpleSpanProcessor.create(spanExporter)) + .build(); + openTelemetrySdk = OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).build(); + tracer = openTelemetrySdk.getTracer("test-tracer"); + tracingFactory = new OpenTelemetryTracingFactory(openTelemetrySdk).withContext(TRACER_CONTEXT); + } + + @AfterEach + void tearDown() { + tracerProvider.close(); + } + + @Test + void testAttemptSpan_linkedToParentContextFromCallingThread() { + Span parentSpan = tracer.spanBuilder("application-parent-operation").startSpan(); + ApiTracer apiTracer; + try (Scope scope = parentSpan.makeCurrent()) { + apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + } + + apiTracer.attemptStarted(new Object(), 0); + apiTracer.attemptSucceeded(); + apiTracer.operationSucceeded(); + parentSpan.end(); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(2); + + SpanData attemptSpan = + finishedSpans.stream() + .filter(s -> s.getName().equals(FULL_METHOD_NAME)) + .findFirst() + .orElseThrow(() -> new AssertionError("Attempt span not found")); + SpanData rootSpan = + finishedSpans.stream() + .filter(s -> s.getName().equals("application-parent-operation")) + .findFirst() + .orElseThrow(() -> new AssertionError("Parent span not found")); + + assertThat(attemptSpan.getKind()).isEqualTo(SpanKind.CLIENT); + assertThat(attemptSpan.getParentSpanId()).isEqualTo(rootSpan.getSpanContext().getSpanId()); + assertThat(attemptSpan.getSpanContext().getTraceId()) + .isEqualTo(rootSpan.getSpanContext().getTraceId()); + } + + @Test + void testAttemptSpan_withoutParentContext_hasNoParent() { + ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + + apiTracer.attemptStarted(new Object(), 0); + apiTracer.attemptSucceeded(); + apiTracer.operationSucceeded(); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(1); + + SpanData attemptSpan = finishedSpans.get(0); + assertThat(attemptSpan.getName()).isEqualTo(FULL_METHOD_NAME); + assertThat(attemptSpan.getParentSpanContext().isValid()).isFalse(); + } + + @Test + void testSequentialAttempts_closesPreviousAttemptSpanAndLinksAllToParent() { + Span parentSpan = tracer.spanBuilder("application-parent-operation").startSpan(); + ApiTracer apiTracer; + try (Scope scope = parentSpan.makeCurrent()) { + apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + } + + // Start attempt 0 (e.g. transient failure without explicit endAttempt before retry) + apiTracer.attemptStarted(new Object(), 0); + + // Start attempt 1 - should automatically end attempt 0 + apiTracer.attemptStarted(new Object(), 1); + assertThat(spanExporter.getFinishedSpanItems()).hasSize(1); + SpanData attempt0Span = spanExporter.getFinishedSpanItems().get(0); + assertThat(attempt0Span.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); + + // Complete attempt 1 and operation + apiTracer.attemptSucceeded(); + apiTracer.operationSucceeded(); + parentSpan.end(); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, parent + + SpanData attempt1Span = + finishedSpans.stream() + .filter( + s -> + s.getName().equals(FULL_METHOD_NAME) + && !s.getSpanContext() + .getSpanId() + .equals(attempt0Span.getSpanContext().getSpanId())) + .findFirst() + .orElseThrow(() -> new AssertionError("Attempt 1 span not found")); + + assertThat(attempt1Span.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); + assertThat(attempt1Span.getSpanContext().getTraceId()) + .isEqualTo(parentSpan.getSpanContext().getTraceId()); + } + + @Test + void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { + Span parentSpan = tracer.spanBuilder("application-parent-operation").startSpan(); + ApiTracer apiTracer; + try (Scope scope = parentSpan.makeCurrent()) { + apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + } + + apiTracer.attemptStarted(new Object(), 0); + // Operation fails while attempt was still in flight + apiTracer.operationFailed(new RuntimeException("network timeout")); + parentSpan.end(); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(2); + + SpanData attemptSpan = + finishedSpans.stream() + .filter(s -> s.getName().equals(FULL_METHOD_NAME)) + .findFirst() + .orElseThrow(() -> new AssertionError("Attempt span not found")); + + assertThat(attemptSpan.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); + assertThat( + attemptSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isEqualTo("network timeout"); + } + + @Test + void testAttemptStarted_afterOperationCompleted_doesNotEmitNewSpan() { + ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + + apiTracer.operationSucceeded(); + + // Any attempts started after operation completed must be ignored + apiTracer.attemptStarted(new Object(), 0); + + assertThat(spanExporter.getFinishedSpanItems()).isEmpty(); + } +} From 2d5db0732c47d54d31911276a7c5c489be7c994b Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 28 Sep 2026 15:24:52 -0400 Subject: [PATCH 05/22] chore(gax): add explanatory comments to OpenTelemetryTracingTracer --- .../api/gax/tracing/OpenTelemetryTracingTracer.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 03a0b8c2a67d..b7f5fff79938 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -53,9 +53,14 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final Map attemptAttributes; private final String attemptSpanName; private final ApiTracerContext apiTracerContext; + // Captures the active trace context from the calling thread at RPC initiation. + // This allows attempt spans—including retries dispatched on background threads—to + // link back to the original parent trace. private final io.opentelemetry.context.Context parentContext; + // Lock coordinates attempt transitions and operation completion across threads. private final ReentrantLock lock = new ReentrantLock(); private boolean operationCompleted; + // attemptSpan is volatile to ensure fresh reads for thread-safe snapshotting. private volatile @Nullable Span attemptSpan; @Override @@ -133,9 +138,12 @@ public void attemptStarted(Object request, int attemptNumber) { Span oldSpan = null; lock.lock(); try { + // Prevent creating new attempt spans if the overall operation has already concluded. if (operationCompleted) { return; } + // If a previous attempt was not explicitly closed before a retry started, + // capture it so it can be ended cleanly outside the lock without blocking. if (attemptSpan != null) { oldSpan = attemptSpan; attemptSpan = null; @@ -168,6 +176,7 @@ public void attemptStarted(Object request, int attemptNumber) { } finally { lock.unlock(); } + // End lingering previous attempt outside the lock to avoid holding the lock during callbacks. if (oldSpan != null) { endAttemptSpan(oldSpan, null); } @@ -195,6 +204,8 @@ public void attemptSucceeded() { @Override public void responseHeadersReceived(java.util.Map headers) { + // Snapshot to a local variable to prevent race conditions if another thread + // clears attemptSpan concurrently. Span currentSpan = attemptSpan; if (currentSpan == null) { return; @@ -296,6 +307,8 @@ private void endAttemptSpan(Span localAttemptSpan, @Nullable Throwable error) { @Override public void requestUrlResolved(String url) { + // Snapshot to a local variable to prevent race conditions if another thread + // clears attemptSpan concurrently. Span currentSpan = attemptSpan; if (currentSpan == null) { return; From ed0e4432fdd962b714a472f1c95181731e55761d Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 28 Sep 2026 15:54:51 -0400 Subject: [PATCH 06/22] chore(gax): document new and overridden methods in OpenTelemetryTracingTracer --- .../tracing/OpenTelemetryTracingTracer.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index b7f5fff79938..e2019e90aed8 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -182,16 +182,35 @@ public void attemptStarted(Object request, int attemptNumber) { } } + /** + * Signals that the overall logical operation succeeded. + * + *

Marks the operation as complete and closes any remaining in-flight attempt span. + */ @Override public void operationSucceeded() { recordErrorAndEndAttempt(null, true); } + /** + * Signals that the overall logical operation was cancelled. + * + *

Marks the operation as complete and closes any remaining in-flight attempt span with a + * {@link CancellationException}. + */ @Override public void operationCancelled() { recordErrorAndEndAttempt(new CancellationException(), true); } + /** + * Signals that the overall logical operation failed permanently. + * + *

Marks the operation as complete and closes any remaining in-flight attempt span with the + * provided error details. + * + * @param error the cause of the operation failure + */ @Override public void operationFailed(Throwable error) { recordErrorAndEndAttempt(error, true); @@ -271,6 +290,13 @@ private void recordErrorAndEndAttempt(@Nullable Throwable error) { recordErrorAndEndAttempt(error, false); } + /** + * Records error details and ends the current attempt span in a thread-safe manner. + * + * @param error the exception associated with the attempt failure, or {@code null} if successful + * @param isOperationComplete {@code true} if this call marks the end of the entire logical + * operation, preventing subsequent retry attempts from starting + */ private void recordErrorAndEndAttempt(@Nullable Throwable error, boolean isOperationComplete) { Span localAttemptSpan; lock.lock(); @@ -290,6 +316,15 @@ private void recordErrorAndEndAttempt(@Nullable Throwable error, boolean isOpera endAttemptSpan(localAttemptSpan, error); } + /** + * Attaches response status attributes and error messages to the attempt span and ends it. + * + *

This method runs outside of synchronization locks to avoid blocking threads during + * OpenTelemetry span completion callbacks. + * + * @param localAttemptSpan the attempt span to finish + * @param error the exception that caused the attempt to end, or {@code null} if successful + */ private void endAttemptSpan(Span localAttemptSpan, @Nullable Throwable error) { Map responseAttributes = ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport()); From e15ae2dd954eaa971f696bb104638838192f128e Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 28 Sep 2026 16:37:36 -0400 Subject: [PATCH 07/22] fix(gax): add opentelemetry-sdk-trace test dependency in maven and bazel --- sdk-platform-java/gax-java/dependencies.properties | 1 + sdk-platform-java/gax-java/gax/BUILD.bazel | 1 + sdk-platform-java/gax-java/gax/pom.xml | 5 +++++ 3 files changed, 7 insertions(+) diff --git a/sdk-platform-java/gax-java/dependencies.properties b/sdk-platform-java/gax-java/dependencies.properties index 6682d09eff89..135749d324d9 100644 --- a/sdk-platform-java/gax-java/dependencies.properties +++ b/sdk-platform-java/gax-java/dependencies.properties @@ -97,6 +97,7 @@ maven.io_opentelemetry_opentelemetry_sdk_testing=io.opentelemetry:opentelemetry- maven.io_opentelemetry_opentelemetry_sdk=io.opentelemetry:opentelemetry-sdk:1.57.0 maven.io_opentelemetry_opentelemetry_sdk_common=io.opentelemetry:opentelemetry-sdk-common:1.57.0 maven.io_opentelemetry_opentelemetry_sdk_metrics=io.opentelemetry:opentelemetry-sdk-metrics:1.57.0 +maven.io_opentelemetry_opentelemetry_sdk_trace=io.opentelemetry:opentelemetry-sdk-trace:1.57.0 maven.com_google_guava_guava_testlib=com.google.guava:guava-testlib:32.1.3-jre maven.org_awaitility_awaitility=org.awaitility:awaitility:4.3.0 diff --git a/sdk-platform-java/gax-java/gax/BUILD.bazel b/sdk-platform-java/gax-java/gax/BUILD.bazel index c98939ba7e18..a77329ede135 100644 --- a/sdk-platform-java/gax-java/gax/BUILD.bazel +++ b/sdk-platform-java/gax-java/gax/BUILD.bazel @@ -49,6 +49,7 @@ _TEST_COMPILE_DEPS = [ "@io_opentelemetry_opentelemetry_sdk//jar", "@io_opentelemetry_opentelemetry_sdk_metrics//jar", "@io_opentelemetry_opentelemetry_sdk_common//jar", + "@io_opentelemetry_opentelemetry_sdk_trace//jar", "@com_google_guava_guava_testlib//jar", "@org_awaitility_awaitility//jar", ] diff --git a/sdk-platform-java/gax-java/gax/pom.xml b/sdk-platform-java/gax-java/gax/pom.xml index 02c64480defb..8db559aabf12 100644 --- a/sdk-platform-java/gax-java/gax/pom.xml +++ b/sdk-platform-java/gax-java/gax/pom.xml @@ -104,6 +104,11 @@ opentelemetry-sdk-common test + + io.opentelemetry + opentelemetry-sdk-trace + test + org.junit.jupiter junit-jupiter-api From 7ba64dc2f0e56bf57b125bb2cfc9e45110469ba6 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Tue, 29 Sep 2026 17:10:59 -0400 Subject: [PATCH 08/22] feat(gax): implement operation-level client request spans (T3) --- .../tracing/OpenTelemetryTracingTracer.java | 165 ++++++++++++++---- ...OpenTelemetryTracingTracerFactoryTest.java | 8 +- ...TelemetryTracingTracerIntegrationTest.java | 67 +++++-- .../OpenTelemetryTracingTracerTest.java | 66 +++++++ 4 files changed, 254 insertions(+), 52 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index e2019e90aed8..70f3f7298869 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -35,6 +35,7 @@ import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; import java.util.HashMap; import java.util.Map; @@ -52,22 +53,27 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final Tracer tracer; private final Map attemptAttributes; private final String attemptSpanName; + private final String operationSpanName; private final ApiTracerContext apiTracerContext; // Captures the active trace context from the calling thread at RPC initiation. - // This allows attempt spans—including retries dispatched on background threads—to - // link back to the original parent trace. + // This allows the operation span and attempt spans to link back to the caller's trace. private final io.opentelemetry.context.Context parentContext; + // Trace context containing the operationSpan, serving as the parent for attempt spans. + private final io.opentelemetry.context.Context operationContext; // Lock coordinates attempt transitions and operation completion across threads. private final ReentrantLock lock = new ReentrantLock(); private boolean operationCompleted; - // attemptSpan is volatile to ensure fresh reads for thread-safe snapshotting. + // operationSpan and attemptSpan are volatile to ensure fresh reads for thread-safe snapshotting. + private volatile @Nullable Span operationSpan; private volatile @Nullable Span attemptSpan; @Override public void injectTraceContext(java.util.Map carrier) { - if (attemptSpan != null) { + Span currentAttempt = attemptSpan; + Span spanToInject = currentAttempt != null ? currentAttempt : operationSpan; + if (spanToInject != null) { io.opentelemetry.context.Context context = - io.opentelemetry.context.Context.current().with(attemptSpan); + io.opentelemetry.context.Context.current().with(spanToInject); io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator.getInstance() .inject( context, @@ -80,6 +86,17 @@ public void injectTraceContext(java.util.Map carrier) { } } + @Override + public Scope inScope() { + Span currentAttempt = attemptSpan; + Span currentSpan = currentAttempt != null ? currentAttempt : operationSpan; + if (currentSpan == null) { + return () -> {}; + } + io.opentelemetry.context.Scope otelScope = currentSpan.makeCurrent(); + return otelScope::close; + } + /** * Creates a new instance of {@code OpenTelemetryTracingTracer}. * @@ -87,12 +104,7 @@ public void injectTraceContext(java.util.Map carrier) { * @param apiTracerContext the {@link ApiTracerContext} to use for recording spans */ OpenTelemetryTracingTracer(Tracer tracer, ApiTracerContext apiTracerContext) { - this.tracer = tracer; - this.apiTracerContext = apiTracerContext; - this.attemptSpanName = resolveAttemptSpanName(apiTracerContext); - this.attemptAttributes = new HashMap<>(); - this.parentContext = io.opentelemetry.context.Context.current(); - buildAttributes(); + this(tracer, apiTracerContext, resolveAttemptSpanName(apiTracerContext)); } /** @@ -106,12 +118,64 @@ public void injectTraceContext(java.util.Map carrier) { @InternalApi OpenTelemetryTracingTracer( Tracer tracer, ApiTracerContext apiTracerContext, String attemptSpanName) { + this(tracer, apiTracerContext, attemptSpanName, resolveOperationSpanName(attemptSpanName)); + } + + /** + * Creates a new instance of {@code OpenTelemetryTracingTracer} with explicitly provided attempt + * and operation span names. + * + * @param tracer the {@link Tracer} to use for recording spans + * @param apiTracerContext the {@link ApiTracerContext} to use for recording spans + * @param attemptSpanName the name of the individual attempt spans + * @param operationSpanName the name of the overall client request operation span + */ + @InternalApi + OpenTelemetryTracingTracer( + Tracer tracer, + ApiTracerContext apiTracerContext, + String attemptSpanName, + String operationSpanName) { this.tracer = tracer; - this.attemptSpanName = attemptSpanName; this.apiTracerContext = apiTracerContext; + this.attemptSpanName = attemptSpanName; + this.operationSpanName = operationSpanName; this.attemptAttributes = new HashMap<>(); this.parentContext = io.opentelemetry.context.Context.current(); buildAttributes(); + this.operationSpan = startOperationSpan(); + this.operationContext = parentContext.with(this.operationSpan); + } + + /** + * Starts and initializes the operation-level client request span (T3). + * + * @return the newly started {@link Span} for the overall operation + */ + private Span startOperationSpan() { + SpanBuilder operationSpanBuilder = tracer.spanBuilder(operationSpanName); + operationSpanBuilder.setSpanKind(SpanKind.INTERNAL); + operationSpanBuilder.setParent(parentContext); + operationSpanBuilder.setAllAttributes( + ObservabilityUtils.toOtelAttributes(this.attemptAttributes)); + return operationSpanBuilder.startSpan(); + } + + /** + * Derives the operation-level span name from the attempt span name. + * + * @param attemptSpanName the attempt span name + * @return the operation span name + */ + private static String resolveOperationSpanName(String attemptSpanName) { + if (!Strings.isNullOrEmpty(attemptSpanName)) { + if (attemptSpanName.endsWith("/attempt")) { + String name = attemptSpanName.substring(0, attemptSpanName.length() - "/attempt".length()); + return name.isEmpty() ? "operation" : name; + } + return "attempt".equals(attemptSpanName) ? "operation" : attemptSpanName; + } + return "operation"; } private static String resolveAttemptSpanName(ApiTracerContext apiTracerContext) { @@ -139,7 +203,7 @@ public void attemptStarted(Object request, int attemptNumber) { lock.lock(); try { // Prevent creating new attempt spans if the overall operation has already concluded. - if (operationCompleted) { + if (operationCompleted || operationSpan == null) { return; } // If a previous attempt was not explicitly closed before a retry started, @@ -166,8 +230,8 @@ public void attemptStarted(Object request, int attemptNumber) { // Attempt spans are of the CLIENT kind spanBuilder.setSpanKind(SpanKind.CLIENT); - // Link attempt span to parent context - spanBuilder.setParent(parentContext); + // Link attempt span to operation context (parent T3 span) + spanBuilder.setParent(operationContext); // Pass the combined attributes to the new SpanBuilder method spanBuilder.setAllAttributes(ObservabilityUtils.toOtelAttributes(currentAttemptAttributes)); @@ -185,35 +249,79 @@ public void attemptStarted(Object request, int attemptNumber) { /** * Signals that the overall logical operation succeeded. * - *

Marks the operation as complete and closes any remaining in-flight attempt span. + *

Closes any remaining in-flight attempt span and ends the operation span. */ @Override public void operationSucceeded() { - recordErrorAndEndAttempt(null, true); + recordErrorAndEndOperation(null); } /** * Signals that the overall logical operation was cancelled. * - *

Marks the operation as complete and closes any remaining in-flight attempt span with a - * {@link CancellationException}. + *

Closes any remaining in-flight attempt span with a {@link CancellationException} and ends + * the operation span with an ERROR status. */ @Override public void operationCancelled() { - recordErrorAndEndAttempt(new CancellationException(), true); + recordErrorAndEndOperation(new CancellationException()); } /** * Signals that the overall logical operation failed permanently. * - *

Marks the operation as complete and closes any remaining in-flight attempt span with the - * provided error details. + *

Closes any remaining in-flight attempt span and ends the operation span with the provided + * error details and an ERROR status. * * @param error the cause of the operation failure */ @Override public void operationFailed(Throwable error) { - recordErrorAndEndAttempt(error, true); + recordErrorAndEndOperation(error); + } + + /** + * Records error details and ends both the active attempt span and the operation span in a + * thread-safe manner. + * + * @param error the exception associated with the operation failure, or {@code null} if successful + */ + private void recordErrorAndEndOperation(@Nullable Throwable error) { + Span localOperationSpan; + Span localAttemptSpan; + lock.lock(); + try { + operationCompleted = true; + localOperationSpan = operationSpan; + if (localOperationSpan == null) { + return; + } + operationSpan = null; + localAttemptSpan = attemptSpan; + attemptSpan = null; + } finally { + lock.unlock(); + } + + if (localAttemptSpan != null) { + endAttemptSpan(localAttemptSpan, error); + } + + Map responseAttributes = + ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport()); + if (!responseAttributes.isEmpty()) { + localOperationSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); + } + + if (error != null) { + localOperationSpan.setStatus(StatusCode.ERROR); + if (!Strings.isNullOrEmpty(error.getMessage())) { + localOperationSpan.setAttribute( + ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage()); + } + } + + localOperationSpan.end(); } @Override @@ -286,24 +394,15 @@ public void attemptPermanentFailure(Throwable error) { recordErrorAndEndAttempt(error); } - private void recordErrorAndEndAttempt(@Nullable Throwable error) { - recordErrorAndEndAttempt(error, false); - } - /** * Records error details and ends the current attempt span in a thread-safe manner. * * @param error the exception associated with the attempt failure, or {@code null} if successful - * @param isOperationComplete {@code true} if this call marks the end of the entire logical - * operation, preventing subsequent retry attempts from starting */ - private void recordErrorAndEndAttempt(@Nullable Throwable error, boolean isOperationComplete) { + private void recordErrorAndEndAttempt(@Nullable Throwable error) { Span localAttemptSpan; lock.lock(); try { - if (isOperationComplete) { - operationCompleted = true; - } localAttemptSpan = attemptSpan; if (localAttemptSpan == null) { return; diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java index de7941522587..4a58a3b07bfd 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerFactoryTest.java @@ -229,7 +229,7 @@ void testNewTracer_withContext_grpc_usesFullMethodName() { tracerInstance.attemptStarted(null, 1); - verify(tracer).spanBuilder("google.cloud.v1.Service/Method"); + verify(tracer, atLeastOnce()).spanBuilder("google.cloud.v1.Service/Method"); } @ParameterizedTest @@ -256,7 +256,7 @@ void testNewTracer_withContext_http_usesHttpMethodAndPathTemplate( tracerInstance.attemptStarted(null, 1); - verify(tracer).spanBuilder(expectedSpanName); + verify(tracer, atLeastOnce()).spanBuilder(expectedSpanName); } @Test @@ -274,7 +274,7 @@ void testNewTracer_withContext_http_noHttpMethodOrPathTemplate_usesFullMethodNam tracerInstance.attemptStarted(null, 1); - verify(tracer).spanBuilder("google.cloud.v1.Service.Method"); + verify(tracer, atLeastOnce()).spanBuilder("google.cloud.v1.Service.Method"); } @Test @@ -310,7 +310,7 @@ void testNewTracer_mergesFactoryContext() { tracerInstance.attemptStarted(null, 1); ArgumentCaptor attributesCaptor = ArgumentCaptor.forClass(Attributes.class); - verify(spanBuilder).setAllAttributes(attributesCaptor.capture()); + verify(spanBuilder, atLeastOnce()).setAllAttributes(attributesCaptor.capture()); Attributes attributes = attributesCaptor.getValue(); assertThat(attributes.asMap()) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 31832fbc3d71..613770ec84ed 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -103,13 +103,18 @@ void testAttemptSpan_linkedToParentContextFromCallingThread() { parentSpan.end(); List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(2); + assertThat(finishedSpans).hasSize(3); // root, operation, attempt SpanData attemptSpan = finishedSpans.stream() - .filter(s -> s.getName().equals(FULL_METHOD_NAME)) + .filter(s -> s.getKind() == SpanKind.CLIENT) .findFirst() .orElseThrow(() -> new AssertionError("Attempt span not found")); + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); SpanData rootSpan = finishedSpans.stream() .filter(s -> s.getName().equals("application-parent-operation")) @@ -117,9 +122,12 @@ void testAttemptSpan_linkedToParentContextFromCallingThread() { .orElseThrow(() -> new AssertionError("Parent span not found")); assertThat(attemptSpan.getKind()).isEqualTo(SpanKind.CLIENT); - assertThat(attemptSpan.getParentSpanId()).isEqualTo(rootSpan.getSpanContext().getSpanId()); + assertThat(attemptSpan.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); + assertThat(operationSpan.getParentSpanId()).isEqualTo(rootSpan.getSpanContext().getSpanId()); assertThat(attemptSpan.getSpanContext().getTraceId()) .isEqualTo(rootSpan.getSpanContext().getTraceId()); + assertThat(operationSpan.getSpanContext().getTraceId()) + .isEqualTo(rootSpan.getSpanContext().getTraceId()); } @Test @@ -131,11 +139,21 @@ void testAttemptSpan_withoutParentContext_hasNoParent() { apiTracer.operationSucceeded(); List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(1); + assertThat(finishedSpans).hasSize(2); // operation, attempt - SpanData attemptSpan = finishedSpans.get(0); - assertThat(attemptSpan.getName()).isEqualTo(FULL_METHOD_NAME); - assertThat(attemptSpan.getParentSpanContext().isValid()).isFalse(); + SpanData attemptSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.CLIENT) + .findFirst() + .orElseThrow(() -> new AssertionError("Attempt span not found")); + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); + + assertThat(operationSpan.getParentSpanContext().isValid()).isFalse(); + assertThat(attemptSpan.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); } @Test @@ -153,7 +171,6 @@ void testSequentialAttempts_closesPreviousAttemptSpanAndLinksAllToParent() { apiTracer.attemptStarted(new Object(), 1); assertThat(spanExporter.getFinishedSpanItems()).hasSize(1); SpanData attempt0Span = spanExporter.getFinishedSpanItems().get(0); - assertThat(attempt0Span.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); // Complete attempt 1 and operation apiTracer.attemptSucceeded(); @@ -161,20 +178,31 @@ void testSequentialAttempts_closesPreviousAttemptSpanAndLinksAllToParent() { parentSpan.end(); List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, parent + assertThat(finishedSpans).hasSize(4); // attempt 0, attempt 1, operation, parent + + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); + + assertThat(attempt0Span.getParentSpanId()) + .isEqualTo(operationSpan.getSpanContext().getSpanId()); SpanData attempt1Span = finishedSpans.stream() .filter( s -> - s.getName().equals(FULL_METHOD_NAME) + s.getKind() == SpanKind.CLIENT && !s.getSpanContext() .getSpanId() .equals(attempt0Span.getSpanContext().getSpanId())) .findFirst() .orElseThrow(() -> new AssertionError("Attempt 1 span not found")); - assertThat(attempt1Span.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); + assertThat(attempt1Span.getParentSpanId()) + .isEqualTo(operationSpan.getSpanContext().getSpanId()); + assertThat(operationSpan.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); assertThat(attempt1Span.getSpanContext().getTraceId()) .isEqualTo(parentSpan.getSpanContext().getTraceId()); } @@ -193,15 +221,21 @@ void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { parentSpan.end(); List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(2); + assertThat(finishedSpans).hasSize(3); // parent, operation, attempt SpanData attemptSpan = finishedSpans.stream() - .filter(s -> s.getName().equals(FULL_METHOD_NAME)) + .filter(s -> s.getKind() == SpanKind.CLIENT) .findFirst() .orElseThrow(() -> new AssertionError("Attempt span not found")); + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); - assertThat(attemptSpan.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); + assertThat(attemptSpan.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); + assertThat(operationSpan.getParentSpanId()).isEqualTo(parentSpan.getSpanContext().getSpanId()); assertThat( attemptSpan .getAttributes() @@ -218,6 +252,9 @@ void testAttemptStarted_afterOperationCompleted_doesNotEmitNewSpan() { // Any attempts started after operation completed must be ignored apiTracer.attemptStarted(new Object(), 0); - assertThat(spanExporter.getFinishedSpanItems()).isEmpty(); + // Only the operation span was emitted and ended + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(1); + assertThat(finishedSpans.get(0).getKind()).isEqualTo(SpanKind.INTERNAL); } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 2bd2dd320ce4..7ccb6cde0840 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -66,6 +66,8 @@ class OpenTelemetryTracingTracerTest { @Mock private Tracer tracer; @Mock private SpanBuilder spanBuilder; @Mock private Span span; + @Mock private SpanBuilder operationSpanBuilder; + @Mock private Span operationSpan; private OpenTelemetryTracingTracer openTelemetryTracingTracer; private static final String ATTEMPT_SPAN_NAME = "Service/Method/attempt"; @@ -76,6 +78,20 @@ void setUp() { lenient().when(spanBuilder.setParent(any())).thenReturn(spanBuilder); lenient().when(spanBuilder.setAllAttributes(any(Attributes.class))).thenReturn(spanBuilder); lenient().when(spanBuilder.startSpan()).thenReturn(span); + + lenient() + .when(operationSpanBuilder.setSpanKind(any(SpanKind.class))) + .thenReturn(operationSpanBuilder); + lenient().when(operationSpanBuilder.setParent(any())).thenReturn(operationSpanBuilder); + lenient() + .when(operationSpanBuilder.setAllAttributes(any(Attributes.class))) + .thenReturn(operationSpanBuilder); + lenient().when(operationSpanBuilder.startSpan()).thenReturn(operationSpan); + lenient() + .when(operationSpan.storeInContext(any(io.opentelemetry.context.Context.class))) + .thenAnswer(invocation -> invocation.getArgument(0)); + lenient().when(tracer.spanBuilder("Service/Method")).thenReturn(operationSpanBuilder); + openTelemetryTracingTracer = new OpenTelemetryTracingTracer(tracer, ApiTracerContext.empty(), ATTEMPT_SPAN_NAME); } @@ -696,6 +712,7 @@ void testOperationSucceeded_endsActiveAttemptSpan() { openTelemetryTracingTracer.operationSucceeded(); verify(span).end(); + verify(operationSpan).end(); } @Test @@ -705,6 +722,8 @@ void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { verify(span).setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, "operation failed"); verify(span).end(); + verify(operationSpan).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); + verify(operationSpan).end(); } @Test @@ -715,6 +734,8 @@ void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { ArgumentCaptor attrsCaptor = ArgumentCaptor.forClass(Attributes.class); verify(span).setAllAttributes(attrsCaptor.capture()); verify(span).end(); + verify(operationSpan).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); + verify(operationSpan).end(); assertThat(attrsCaptor.getValue().asMap()) .containsEntry( @@ -722,6 +743,51 @@ void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { "CANCELLED"); } + @Test + void testInScope_withAttemptSpan() { + io.opentelemetry.context.Scope mockScope = mock(io.opentelemetry.context.Scope.class); + when(span.makeCurrent()).thenReturn(mockScope); + + openTelemetryTracingTracer.attemptStarted(new Object(), 1); + try (ApiTracer.Scope scope = openTelemetryTracingTracer.inScope()) { + verify(span).makeCurrent(); + } + verify(mockScope).close(); + } + + @Test + void testInScope_withOperationSpanFallback() { + io.opentelemetry.context.Scope mockScope = mock(io.opentelemetry.context.Scope.class); + when(operationSpan.makeCurrent()).thenReturn(mockScope); + + try (ApiTracer.Scope scope = openTelemetryTracingTracer.inScope()) { + verify(operationSpan).makeCurrent(); + } + verify(mockScope).close(); + } + + @Test + void testInjectTraceContext_withOperationSpanFallback() { + io.opentelemetry.api.trace.SpanContext mockSpanContext = + io.opentelemetry.api.trace.SpanContext.create( + "00000000000000000000000000000003", + "0000000000000004", + io.opentelemetry.api.trace.TraceFlags.getSampled(), + io.opentelemetry.api.trace.TraceState.getDefault()); + Span realSpan = Span.wrap(mockSpanContext); + when(operationSpanBuilder.startSpan()).thenReturn(realSpan); + + openTelemetryTracingTracer = + new OpenTelemetryTracingTracer(tracer, ApiTracerContext.empty(), ATTEMPT_SPAN_NAME); + + Map carrier = new java.util.HashMap<>(); + openTelemetryTracingTracer.injectTraceContext(carrier); + + assertThat(carrier).containsKey("traceparent"); + assertThat(carrier.get("traceparent")).contains("00000000000000000000000000000003"); + assertThat(carrier.get("traceparent")).contains("0000000000000004"); + } + @Test void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { Span span1 = mock(Span.class); From 349c14af70b97f4d24928dda21db1e2c7a7bc00e Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 11:34:09 -0400 Subject: [PATCH 09/22] fix(gax): address review feedback on span names and span completion helper --- .../tracing/OpenTelemetryTracingTracer.java | 45 +++++++------------ 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 70f3f7298869..f4018f6175d5 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -138,8 +138,9 @@ public Scope inScope() { String operationSpanName) { this.tracer = tracer; this.apiTracerContext = apiTracerContext; - this.attemptSpanName = attemptSpanName; this.operationSpanName = operationSpanName; + this.attemptSpanName = + attemptSpanName.equals(operationSpanName) ? attemptSpanName + "/attempt" : attemptSpanName; this.attemptAttributes = new HashMap<>(); this.parentContext = io.opentelemetry.context.Context.current(); buildAttributes(); @@ -242,7 +243,7 @@ public void attemptStarted(Object request, int attemptNumber) { } // End lingering previous attempt outside the lock to avoid holding the lock during callbacks. if (oldSpan != null) { - endAttemptSpan(oldSpan, null); + endSpan(oldSpan, null); } } @@ -304,24 +305,10 @@ private void recordErrorAndEndOperation(@Nullable Throwable error) { } if (localAttemptSpan != null) { - endAttemptSpan(localAttemptSpan, error); + endSpan(localAttemptSpan, error); } - Map responseAttributes = - ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport()); - if (!responseAttributes.isEmpty()) { - localOperationSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); - } - - if (error != null) { - localOperationSpan.setStatus(StatusCode.ERROR); - if (!Strings.isNullOrEmpty(error.getMessage())) { - localOperationSpan.setAttribute( - ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage()); - } - } - - localOperationSpan.end(); + endSpan(localOperationSpan, error); } @Override @@ -412,31 +399,33 @@ private void recordErrorAndEndAttempt(@Nullable Throwable error) { lock.unlock(); } - endAttemptSpan(localAttemptSpan, error); + endSpan(localAttemptSpan, error); } /** - * Attaches response status attributes and error messages to the attempt span and ends it. + * Attaches response status attributes and error messages to the span and ends it. * *

This method runs outside of synchronization locks to avoid blocking threads during * OpenTelemetry span completion callbacks. * - * @param localAttemptSpan the attempt span to finish - * @param error the exception that caused the attempt to end, or {@code null} if successful + * @param span the span to finish + * @param error the exception that caused the span to end, or {@code null} if successful */ - private void endAttemptSpan(Span localAttemptSpan, @Nullable Throwable error) { + private void endSpan(Span span, @Nullable Throwable error) { Map responseAttributes = ObservabilityUtils.getResponseAttributes(error, this.apiTracerContext.transport()); if (!responseAttributes.isEmpty()) { - localAttemptSpan.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); + span.setAllAttributes(ObservabilityUtils.toOtelAttributes(responseAttributes)); } - if (error != null && !Strings.isNullOrEmpty(error.getMessage())) { - localAttemptSpan.setAttribute( - ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage()); + if (error != null) { + span.setStatus(StatusCode.ERROR); + if (!Strings.isNullOrEmpty(error.getMessage())) { + span.setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, error.getMessage()); + } } - localAttemptSpan.end(); + span.end(); } @Override From b542ff110ca65fe86f92132ad892a4e06f87edd1 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:15:00 -0400 Subject: [PATCH 10/22] fix(gax): avoid potential NPE when resolving attempt span name --- .../com/google/api/gax/tracing/OpenTelemetryTracingTracer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index f4018f6175d5..e08dd54373ed 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -140,7 +140,7 @@ public Scope inScope() { this.apiTracerContext = apiTracerContext; this.operationSpanName = operationSpanName; this.attemptSpanName = - attemptSpanName.equals(operationSpanName) ? attemptSpanName + "/attempt" : attemptSpanName; + operationSpanName.equals(attemptSpanName) ? attemptSpanName + "/attempt" : attemptSpanName; this.attemptAttributes = new HashMap<>(); this.parentContext = io.opentelemetry.context.Context.current(); buildAttributes(); From 23b22fcb8db7887c6a43976da3de7804e87a8589 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:33:33 -0400 Subject: [PATCH 11/22] fix(tracing): record CancellationException on lingering attempt when superseded by retry --- .../com/google/api/gax/tracing/OpenTelemetryTracingTracer.java | 2 +- .../google/api/gax/tracing/OpenTelemetryTracingTracerTest.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index e08dd54373ed..1bd18b0faaee 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -243,7 +243,7 @@ public void attemptStarted(Object request, int attemptNumber) { } // End lingering previous attempt outside the lock to avoid holding the lock during callbacks. if (oldSpan != null) { - endSpan(oldSpan, null); + endSpan(oldSpan, new CancellationException("Attempt superseded by retry")); } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 7ccb6cde0840..1e6ca3180af2 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -799,6 +799,7 @@ void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { // Start a second attempt before the first attempt was ended openTelemetryTracingTracer.attemptStarted(new Object(), 1); + verify(span1).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); verify(span1).end(); verify(span2, never()).end(); From 018b3d80ec213a3c3118df590bb0244bd17be2c6 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:37:28 -0400 Subject: [PATCH 12/22] style(tracing): use simple names Map and HashMap in OpenTelemetryTracingTracer --- .../google/api/gax/tracing/OpenTelemetryTracingTracer.java | 4 ++-- .../api/gax/tracing/OpenTelemetryTracingTracerTest.java | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 1bd18b0faaee..baf5c31695c0 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -68,7 +68,7 @@ class OpenTelemetryTracingTracer implements ApiTracer { private volatile @Nullable Span attemptSpan; @Override - public void injectTraceContext(java.util.Map carrier) { + public void injectTraceContext(Map carrier) { Span currentAttempt = attemptSpan; Span spanToInject = currentAttempt != null ? currentAttempt : operationSpan; if (spanToInject != null) { @@ -317,7 +317,7 @@ public void attemptSucceeded() { } @Override - public void responseHeadersReceived(java.util.Map headers) { + public void responseHeadersReceived(Map headers) { // Snapshot to a local variable to prevent race conditions if another thread // clears attemptSpan concurrently. Span currentSpan = attemptSpan; diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 1e6ca3180af2..41a56372d8b7 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -53,6 +53,7 @@ import io.opentelemetry.api.trace.Tracer; import java.net.ConnectException; import java.net.SocketTimeoutException; +import java.util.HashMap; import java.util.Map; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -780,7 +781,7 @@ void testInjectTraceContext_withOperationSpanFallback() { openTelemetryTracingTracer = new OpenTelemetryTracingTracer(tracer, ApiTracerContext.empty(), ATTEMPT_SPAN_NAME); - Map carrier = new java.util.HashMap<>(); + Map carrier = new HashMap<>(); openTelemetryTracingTracer.injectTraceContext(carrier); assertThat(carrier).containsKey("traceparent"); From 4cab2e1fda7140a10a3c93cd33895c9309850ce8 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:45:55 -0400 Subject: [PATCH 13/22] style(tracing): use simple name StatusCode in OpenTelemetryTracingTracerTest --- .../api/gax/tracing/OpenTelemetryTracingTracerTest.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 41a56372d8b7..abeda4771d68 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -50,6 +50,7 @@ import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; import java.net.ConnectException; import java.net.SocketTimeoutException; @@ -723,7 +724,7 @@ void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { verify(span).setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, "operation failed"); verify(span).end(); - verify(operationSpan).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); + verify(operationSpan).setStatus(StatusCode.ERROR); verify(operationSpan).end(); } @@ -735,7 +736,7 @@ void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { ArgumentCaptor attrsCaptor = ArgumentCaptor.forClass(Attributes.class); verify(span).setAllAttributes(attrsCaptor.capture()); verify(span).end(); - verify(operationSpan).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); + verify(operationSpan).setStatus(StatusCode.ERROR); verify(operationSpan).end(); assertThat(attrsCaptor.getValue().asMap()) @@ -800,7 +801,7 @@ void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { // Start a second attempt before the first attempt was ended openTelemetryTracingTracer.attemptStarted(new Object(), 1); - verify(span1).setStatus(io.opentelemetry.api.trace.StatusCode.ERROR); + verify(span1).setStatus(StatusCode.ERROR); verify(span1).end(); verify(span2, never()).end(); From 8274051031089376af05f1b94e2f7aaa081dd457 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:50:54 -0400 Subject: [PATCH 14/22] fix(tracing): end superseded attempt span without error status --- .../com/google/api/gax/tracing/OpenTelemetryTracingTracer.java | 2 +- .../google/api/gax/tracing/OpenTelemetryTracingTracerTest.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index baf5c31695c0..66b8d8ed3c77 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -243,7 +243,7 @@ public void attemptStarted(Object request, int attemptNumber) { } // End lingering previous attempt outside the lock to avoid holding the lock during callbacks. if (oldSpan != null) { - endSpan(oldSpan, new CancellationException("Attempt superseded by retry")); + endSpan(oldSpan, null); } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index abeda4771d68..556845cab1a5 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -801,7 +801,6 @@ void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { // Start a second attempt before the first attempt was ended openTelemetryTracingTracer.attemptStarted(new Object(), 1); - verify(span1).setStatus(StatusCode.ERROR); verify(span1).end(); verify(span2, never()).end(); From 1e66751311517ce2cedc5d09936a7e8202edac29 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:57:25 -0400 Subject: [PATCH 15/22] style(tracing): use simple names Context and W3CTraceContextPropagator in OpenTelemetryTracingTracer --- .../api/gax/tracing/OpenTelemetryTracingTracer.java | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 66b8d8ed3c77..0e1444c52559 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -37,6 +37,8 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; +import io.opentelemetry.context.Context; import java.util.HashMap; import java.util.Map; import java.util.concurrent.CancellationException; @@ -57,9 +59,9 @@ class OpenTelemetryTracingTracer implements ApiTracer { private final ApiTracerContext apiTracerContext; // Captures the active trace context from the calling thread at RPC initiation. // This allows the operation span and attempt spans to link back to the caller's trace. - private final io.opentelemetry.context.Context parentContext; + private final Context parentContext; // Trace context containing the operationSpan, serving as the parent for attempt spans. - private final io.opentelemetry.context.Context operationContext; + private final Context operationContext; // Lock coordinates attempt transitions and operation completion across threads. private final ReentrantLock lock = new ReentrantLock(); private boolean operationCompleted; @@ -72,9 +74,8 @@ public void injectTraceContext(Map carrier) { Span currentAttempt = attemptSpan; Span spanToInject = currentAttempt != null ? currentAttempt : operationSpan; if (spanToInject != null) { - io.opentelemetry.context.Context context = - io.opentelemetry.context.Context.current().with(spanToInject); - io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator.getInstance() + Context context = Context.current().with(spanToInject); + W3CTraceContextPropagator.getInstance() .inject( context, carrier, @@ -142,7 +143,7 @@ public Scope inScope() { this.attemptSpanName = operationSpanName.equals(attemptSpanName) ? attemptSpanName + "/attempt" : attemptSpanName; this.attemptAttributes = new HashMap<>(); - this.parentContext = io.opentelemetry.context.Context.current(); + this.parentContext = Context.current(); buildAttributes(); this.operationSpan = startOperationSpan(); this.operationContext = parentContext.with(this.operationSpan); From 33b1d238898f2f0bd82a7ffdda45489f28ff7d21 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 14:47:59 -0400 Subject: [PATCH 16/22] style(tracing): static import OpenTelemetry StatusCode.ERROR to avoid collision with GAX StatusCode --- .../api/gax/tracing/OpenTelemetryTracingTracerTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 556845cab1a5..1655e3cb69d5 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -30,6 +30,7 @@ package com.google.api.gax.tracing; import static com.google.common.truth.Truth.assertThat; +import static io.opentelemetry.api.trace.StatusCode.ERROR; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; @@ -50,7 +51,6 @@ import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.SpanBuilder; import io.opentelemetry.api.trace.SpanKind; -import io.opentelemetry.api.trace.StatusCode; import io.opentelemetry.api.trace.Tracer; import java.net.ConnectException; import java.net.SocketTimeoutException; @@ -724,7 +724,7 @@ void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { verify(span).setAttribute(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE, "operation failed"); verify(span).end(); - verify(operationSpan).setStatus(StatusCode.ERROR); + verify(operationSpan).setStatus(ERROR); verify(operationSpan).end(); } @@ -736,7 +736,7 @@ void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { ArgumentCaptor attrsCaptor = ArgumentCaptor.forClass(Attributes.class); verify(span).setAllAttributes(attrsCaptor.capture()); verify(span).end(); - verify(operationSpan).setStatus(StatusCode.ERROR); + verify(operationSpan).setStatus(ERROR); verify(operationSpan).end(); assertThat(attrsCaptor.getValue().asMap()) From 412f398ee9d36f906f2ffa0fd837e980f5735a8d Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 15:26:03 -0400 Subject: [PATCH 17/22] fix(tracing): restore canonical attempt span name and suppress MustBeClosedChecker --- .../google/api/gax/tracing/OpenTelemetryTracingTracer.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 0e1444c52559..98cdb0375748 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -88,6 +88,7 @@ public void injectTraceContext(Map carrier) { } @Override + @SuppressWarnings("MustBeClosedChecker") public Scope inScope() { Span currentAttempt = attemptSpan; Span currentSpan = currentAttempt != null ? currentAttempt : operationSpan; @@ -140,8 +141,7 @@ public Scope inScope() { this.tracer = tracer; this.apiTracerContext = apiTracerContext; this.operationSpanName = operationSpanName; - this.attemptSpanName = - operationSpanName.equals(attemptSpanName) ? attemptSpanName + "/attempt" : attemptSpanName; + this.attemptSpanName = attemptSpanName; this.attemptAttributes = new HashMap<>(); this.parentContext = Context.current(); buildAttributes(); From 4b851adb7d4ef51cab69f943da9b61f0dbf5eb51 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 17:27:28 -0400 Subject: [PATCH 18/22] docs(tracing): add Javadoc and explanatory comments for T3 operation spans --- .../api/gax/tracing/OpenTelemetryTracingTracer.java | 12 ++++++++++++ .../OpenTelemetryTracingTracerIntegrationTest.java | 8 ++++++++ .../gax/tracing/OpenTelemetryTracingTracerTest.java | 8 ++++++++ 3 files changed, 28 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java index 98cdb0375748..a4a87463c26d 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/OpenTelemetryTracingTracer.java @@ -71,6 +71,9 @@ class OpenTelemetryTracingTracer implements ApiTracer { @Override public void injectTraceContext(Map carrier) { + // Prefer the active attempt span (T4) so outgoing RPC wire context reflects the specific + // attempt; + // fall back to the overall operation span (T3) if no attempt is currently in-flight. Span currentAttempt = attemptSpan; Span spanToInject = currentAttempt != null ? currentAttempt : operationSpan; if (spanToInject != null) { @@ -90,6 +93,8 @@ public void injectTraceContext(Map carrier) { @Override @SuppressWarnings("MustBeClosedChecker") public Scope inScope() { + // Attach the active attempt span to the current execution thread context; + // fall back to the overall operation span when between attempts. Span currentAttempt = attemptSpan; Span currentSpan = currentAttempt != null ? currentAttempt : operationSpan; if (currentSpan == null) { @@ -180,6 +185,12 @@ private static String resolveOperationSpanName(String attemptSpanName) { return "operation"; } + /** + * Resolves the canonical attempt-level span name based on transport and context. + * + * @param apiTracerContext the tracer context containing transport and method metadata + * @return the attempt span name + */ private static String resolveAttemptSpanName(ApiTracerContext apiTracerContext) { if (apiTracerContext.transport() == ApiTracerContext.Transport.GRPC) { // gRPC Uses the full method name as span name. @@ -195,6 +206,7 @@ private static String resolveAttemptSpanName(ApiTracerContext apiTracerContext) } } + /** Copies attempt-level attributes from the tracer context into the local attribute cache. */ private void buildAttributes() { this.attemptAttributes.putAll(this.apiTracerContext.getAttemptAttributes()); } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 613770ec84ed..4aa3a47fe807 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -91,6 +91,8 @@ void tearDown() { @Test void testAttemptSpan_linkedToParentContextFromCallingThread() { + // Verifies that when an application trace parent exists, the operation span (T3) + // links to the application parent, and the attempt span (T4) links to the operation span. Span parentSpan = tracer.spanBuilder("application-parent-operation").startSpan(); ApiTracer apiTracer; try (Scope scope = parentSpan.makeCurrent()) { @@ -132,6 +134,8 @@ void testAttemptSpan_linkedToParentContextFromCallingThread() { @Test void testAttemptSpan_withoutParentContext_hasNoParent() { + // Verifies that without an external parent context, the operation span acts as root + // and the attempt span is a child of the operation span. ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); apiTracer.attemptStarted(new Object(), 0); @@ -209,6 +213,8 @@ void testSequentialAttempts_closesPreviousAttemptSpanAndLinksAllToParent() { @Test void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { + // Verifies that when the operation fails permanently while an attempt is in-flight, + // both the attempt span and the operation span end with the error status and message. Span parentSpan = tracer.spanBuilder("application-parent-operation").startSpan(); ApiTracer apiTracer; try (Scope scope = parentSpan.makeCurrent()) { @@ -245,6 +251,8 @@ void testOperationFailed_endsActiveAttemptSpanWithErrorAttributes() { @Test void testAttemptStarted_afterOperationCompleted_doesNotEmitNewSpan() { + // Verifies that after operation completion, subsequent attemptStarted calls + // do not create orphan attempt spans. ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); apiTracer.operationSucceeded(); diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java index 1655e3cb69d5..6d1659ee0c71 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerTest.java @@ -747,6 +747,7 @@ void testOperationCancelled_endsActiveAttemptSpanWithCancellation() { @Test void testInScope_withAttemptSpan() { + // Verifies that inScope() activates the current attempt span if an attempt is currently active. io.opentelemetry.context.Scope mockScope = mock(io.opentelemetry.context.Scope.class); when(span.makeCurrent()).thenReturn(mockScope); @@ -759,6 +760,8 @@ void testInScope_withAttemptSpan() { @Test void testInScope_withOperationSpanFallback() { + // Verifies that inScope() falls back to activating the operation span when no attempt span is + // active. io.opentelemetry.context.Scope mockScope = mock(io.opentelemetry.context.Scope.class); when(operationSpan.makeCurrent()).thenReturn(mockScope); @@ -770,6 +773,8 @@ void testInScope_withOperationSpanFallback() { @Test void testInjectTraceContext_withOperationSpanFallback() { + // Verifies that injectTraceContext() injects the operation span context into the carrier + // when between attempts so that context propagation doesn't drop trace state. io.opentelemetry.api.trace.SpanContext mockSpanContext = io.opentelemetry.api.trace.SpanContext.create( "00000000000000000000000000000003", @@ -792,6 +797,8 @@ void testInjectTraceContext_withOperationSpanFallback() { @Test void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { + // Verifies that starting a new retry attempt cleanly closes any lingering previous attempt + // span. Span span1 = mock(Span.class); Span span2 = mock(Span.class); @@ -811,6 +818,7 @@ void testAttemptStarted_whenPreviousAttemptActive_closesOldSpan() { @Test void testAttemptStarted_afterOperationCompleted_doesNotStartNewSpan() { + // Verifies that after operation completion, late callbacks cannot spawn new attempt spans. openTelemetryTracingTracer.operationSucceeded(); // Attempting to start a new attempt after operation completion should be a no-op From ff8e44831511ed8f3c7830aa9ee3a53b798cf739 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 11:39:52 -0400 Subject: [PATCH 19/22] feat(gax): verify T3/T4 span hierarchy and retry aggregation --- ...TelemetryTracingTracerIntegrationTest.java | 95 +++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 4aa3a47fe807..71b92a56d3e9 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -44,7 +44,9 @@ import io.opentelemetry.sdk.trace.SdkTracerProvider; import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.export.SimpleSpanProcessor; +import java.time.Duration; import java.util.List; +import java.util.stream.Collectors; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -265,4 +267,97 @@ void testAttemptStarted_afterOperationCompleted_doesNotEmitNewSpan() { assertThat(finishedSpans).hasSize(1); assertThat(finishedSpans.get(0).getKind()).isEqualTo(SpanKind.INTERNAL); } + + @Test + void testRetrySucceeds_operationAggregatesSuccessAttributes() { + ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + + // Attempt 0 fails with transient error + apiTracer.attemptStarted(new Object(), 0); + apiTracer.attemptFailedDuration(new RuntimeException("transient 503"), Duration.ofMillis(10)); + + // Attempt 1 succeeds + apiTracer.attemptStarted(new Object(), 1); + apiTracer.attemptSucceeded(); + apiTracer.operationSucceeded(); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation + + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); + + List attemptSpans = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.CLIENT) + .collect(Collectors.toList()); + assertThat(attemptSpans).hasSize(2); + + for (SpanData attempt : attemptSpans) { + assertThat(attempt.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); + } + + assertThat(operationSpan.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.UNSET); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.RPC_RESPONSE_STATUS_ATTRIBUTE))) + .isEqualTo("OK"); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNull(); + } + + @Test + void testRetriesExhausted_operationAggregatesFailureAttributes() { + ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); + + // Attempt 0 fails with transient error + apiTracer.attemptStarted(new Object(), 0); + apiTracer.attemptFailedDuration(new RuntimeException("transient 503"), Duration.ofMillis(10)); + + // Attempt 1 fails and exhausts retries + apiTracer.attemptStarted(new Object(), 1); + RuntimeException finalError = new RuntimeException("unavailable: retries exhausted"); + apiTracer.attemptFailedRetriesExhausted(finalError); + apiTracer.operationFailed(finalError); + + List finishedSpans = spanExporter.getFinishedSpanItems(); + assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation + + SpanData operationSpan = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.INTERNAL) + .findFirst() + .orElseThrow(() -> new AssertionError("Operation span not found")); + + List attemptSpans = + finishedSpans.stream() + .filter(s -> s.getKind() == SpanKind.CLIENT) + .collect(Collectors.toList()); + assertThat(attemptSpans).hasSize(2); + + for (SpanData attempt : attemptSpans) { + assertThat(attempt.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); + } + + assertThat(operationSpan.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isEqualTo("unavailable: retries exhausted"); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNotNull(); + } } From 22ca25d3967ad1de8b6fdd9b78727ca4175a2007 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 13:43:42 -0400 Subject: [PATCH 20/22] test(tracing): explicitly verify exactly one internal operation span in retry tests --- .../OpenTelemetryTracingTracerIntegrationTest.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 71b92a56d3e9..50d77925e9e6 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -284,11 +284,12 @@ void testRetrySucceeds_operationAggregatesSuccessAttributes() { List finishedSpans = spanExporter.getFinishedSpanItems(); assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation - SpanData operationSpan = + List internalSpans = finishedSpans.stream() .filter(s -> s.getKind() == SpanKind.INTERNAL) - .findFirst() - .orElseThrow(() -> new AssertionError("Operation span not found")); + .collect(Collectors.toList()); + assertThat(internalSpans).hasSize(1); + SpanData operationSpan = internalSpans.get(0); List attemptSpans = finishedSpans.stream() @@ -331,11 +332,12 @@ void testRetriesExhausted_operationAggregatesFailureAttributes() { List finishedSpans = spanExporter.getFinishedSpanItems(); assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation - SpanData operationSpan = + List internalSpans = finishedSpans.stream() .filter(s -> s.getKind() == SpanKind.INTERNAL) - .findFirst() - .orElseThrow(() -> new AssertionError("Operation span not found")); + .collect(Collectors.toList()); + assertThat(internalSpans).hasSize(1); + SpanData operationSpan = internalSpans.get(0); List attemptSpans = finishedSpans.stream() From f27fdb36114903a4d5bf66164178a0deb9db8ba0 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 17:28:18 -0400 Subject: [PATCH 21/22] docs(tracing): add explanatory comments to T3/T4 hierarchy and retry tests --- .../OpenTelemetryTracingTracerIntegrationTest.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 50d77925e9e6..81a5949eed91 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -270,6 +270,10 @@ void testAttemptStarted_afterOperationCompleted_doesNotEmitNewSpan() { @Test void testRetrySucceeds_operationAggregatesSuccessAttributes() { + // Verifies that when a transient failure is retried and succeeds: + // 1. Exactly one overall INTERNAL operation span (T3) is created. + // 2. Both attempt spans (T4) have the operation span as their parent. + // 3. The operation span aggregates the successful status (OK) from the final attempt. ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); // Attempt 0 fails with transient error @@ -317,6 +321,10 @@ void testRetrySucceeds_operationAggregatesSuccessAttributes() { @Test void testRetriesExhausted_operationAggregatesFailureAttributes() { + // Verifies that when retries are exhausted: + // 1. Exactly one overall INTERNAL operation span (T3) is created. + // 2. Both attempt spans (T4) have the operation span as their parent. + // 3. The operation span aggregates the final ERROR status and error attributes. ApiTracer apiTracer = tracingFactory.newTracer(BaseApiTracer.getInstance(), TRACER_CONTEXT); // Attempt 0 fails with transient error From 8de17a01d584163bf775e037292bf9bbcc9afc10 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 5 Oct 2026 17:36:26 -0400 Subject: [PATCH 22/22] refactor(tracing): extract verifySpanHierarchyAndGetOperationSpan test helper --- ...TelemetryTracingTracerIntegrationTest.java | 63 +++++++++---------- 1 file changed, 29 insertions(+), 34 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java index 81a5949eed91..b479a6a5eccb 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/OpenTelemetryTracingTracerIntegrationTest.java @@ -285,25 +285,8 @@ void testRetrySucceeds_operationAggregatesSuccessAttributes() { apiTracer.attemptSucceeded(); apiTracer.operationSucceeded(); - List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation - - List internalSpans = - finishedSpans.stream() - .filter(s -> s.getKind() == SpanKind.INTERNAL) - .collect(Collectors.toList()); - assertThat(internalSpans).hasSize(1); - SpanData operationSpan = internalSpans.get(0); - - List attemptSpans = - finishedSpans.stream() - .filter(s -> s.getKind() == SpanKind.CLIENT) - .collect(Collectors.toList()); - assertThat(attemptSpans).hasSize(2); - - for (SpanData attempt : attemptSpans) { - assertThat(attempt.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); - } + SpanData operationSpan = + verifySpanHierarchyAndGetOperationSpan(spanExporter.getFinishedSpanItems()); assertThat(operationSpan.getStatus().getStatusCode()) .isEqualTo(io.opentelemetry.api.trace.StatusCode.UNSET); @@ -337,8 +320,32 @@ void testRetriesExhausted_operationAggregatesFailureAttributes() { apiTracer.attemptFailedRetriesExhausted(finalError); apiTracer.operationFailed(finalError); - List finishedSpans = spanExporter.getFinishedSpanItems(); - assertThat(finishedSpans).hasSize(3); // attempt 0, attempt 1, operation + SpanData operationSpan = + verifySpanHierarchyAndGetOperationSpan(spanExporter.getFinishedSpanItems()); + + assertThat(operationSpan.getStatus().getStatusCode()) + .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) + .isEqualTo("unavailable: retries exhausted"); + assertThat( + operationSpan + .getAttributes() + .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) + .isNotNull(); + } + + /** + * Verifies that exactly one internal operation span and two client attempt spans exist in the + * finished spans, and that each attempt span has the operation span as its parent. + * + * @param finishedSpans the list of recorded finished spans + * @return the verified operation {@link SpanData} + */ + private SpanData verifySpanHierarchyAndGetOperationSpan(List finishedSpans) { + assertThat(finishedSpans).hasSize(3); List internalSpans = finishedSpans.stream() @@ -356,18 +363,6 @@ void testRetriesExhausted_operationAggregatesFailureAttributes() { for (SpanData attempt : attemptSpans) { assertThat(attempt.getParentSpanId()).isEqualTo(operationSpan.getSpanContext().getSpanId()); } - - assertThat(operationSpan.getStatus().getStatusCode()) - .isEqualTo(io.opentelemetry.api.trace.StatusCode.ERROR); - assertThat( - operationSpan - .getAttributes() - .get(AttributeKey.stringKey(ObservabilityAttributes.STATUS_MESSAGE_ATTRIBUTE))) - .isEqualTo("unavailable: retries exhausted"); - assertThat( - operationSpan - .getAttributes() - .get(AttributeKey.stringKey(ObservabilityAttributes.ERROR_TYPE_ATTRIBUTE))) - .isNotNull(); + return operationSpan; } }