Skip to content

Commit 2cdfe4e

Browse files
jinseopkim0jinseopkim0
committed
feat(gax): correlate M3 client duration metrics with T3 span exemplars
1 parent 2f08818 commit 2cdfe4e

2 files changed

Lines changed: 212 additions & 28 deletions

File tree

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java‎

Lines changed: 70 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@
4545
@NullMarked
4646
@InternalApi
4747
class CompositeTracer extends BaseApiTracer {
48+
private static final Scope NO_OP_SCOPE = () -> {};
49+
4850
private final List<ApiTracer> children;
4951

5052
public CompositeTracer(List<ApiTracer> children) {
@@ -56,8 +58,12 @@ public Scope inScope() {
5658
final List<Scope> childScopes = new ArrayList<>(children.size());
5759

5860
try {
61+
// Enter each child scope in forward order, tracking only non-null scopes
5962
for (ApiTracer child : children) {
60-
childScopes.add(child.inScope());
63+
Scope scope = child.inScope();
64+
if (scope != null) {
65+
childScopes.add(scope);
66+
}
6167
}
6268
} catch (RuntimeException e) {
6369
for (int i = childScopes.size() - 1; i >= 0; i--) {
@@ -89,24 +95,51 @@ public Scope inScope() {
8995
};
9096
}
9197

98+
/**
99+
* Enters the tracer's ambient scope safely, returning a no-op {@link Scope} if entering fails.
100+
* This avoids allocating runnables, throwing exceptions, or returning null during lifecycle
101+
* notifications.
102+
*
103+
* <p>Entering the ambient scope ensures active trace span context is present on the thread so
104+
* that OpenTelemetry metric measurements recorded in lifecycle callbacks attach exemplars
105+
* pointing to the active span.
106+
*/
107+
private Scope enterScope() {
108+
try {
109+
return inScope();
110+
} catch (RuntimeException e) {
111+
// Ignore to prevent disrupting the lifecycle notification
112+
return NO_OP_SCOPE;
113+
}
114+
}
115+
92116
@Override
93117
public void operationSucceeded() {
94-
for (int i = children.size() - 1; i >= 0; i--) {
95-
children.get(i).operationSucceeded();
118+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
119+
try (Scope s = enterScope()) {
120+
for (int i = children.size() - 1; i >= 0; i--) {
121+
children.get(i).operationSucceeded();
122+
}
96123
}
97124
}
98125

99126
@Override
100127
public void operationCancelled() {
101-
for (int i = children.size() - 1; i >= 0; i--) {
102-
children.get(i).operationCancelled();
128+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
129+
try (Scope s = enterScope()) {
130+
for (int i = children.size() - 1; i >= 0; i--) {
131+
children.get(i).operationCancelled();
132+
}
103133
}
104134
}
105135

106136
@Override
107137
public void operationFailed(Throwable error) {
108-
for (int i = children.size() - 1; i >= 0; i--) {
109-
children.get(i).operationFailed(error);
138+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
139+
try (Scope s = enterScope()) {
140+
for (int i = children.size() - 1; i >= 0; i--) {
141+
children.get(i).operationFailed(error);
142+
}
110143
}
111144
}
112145

@@ -134,43 +167,61 @@ public void attemptStarted(Object request, int attemptNumber) {
134167

135168
@Override
136169
public void attemptSucceeded() {
137-
for (int i = children.size() - 1; i >= 0; i--) {
138-
children.get(i).attemptSucceeded();
170+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
171+
try (Scope s = enterScope()) {
172+
for (int i = children.size() - 1; i >= 0; i--) {
173+
children.get(i).attemptSucceeded();
174+
}
139175
}
140176
}
141177

142178
@Override
143179
public void attemptCancelled() {
144-
for (int i = children.size() - 1; i >= 0; i--) {
145-
children.get(i).attemptCancelled();
180+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
181+
try (Scope s = enterScope()) {
182+
for (int i = children.size() - 1; i >= 0; i--) {
183+
children.get(i).attemptCancelled();
184+
}
146185
}
147186
}
148187

149188
@Override
150189
public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) {
151-
for (int i = children.size() - 1; i >= 0; i--) {
152-
children.get(i).attemptFailed(error, delay);
190+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
191+
try (Scope s = enterScope()) {
192+
for (int i = children.size() - 1; i >= 0; i--) {
193+
children.get(i).attemptFailed(error, delay);
194+
}
153195
}
154196
}
155197

156198
@Override
157199
public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
158-
for (int i = children.size() - 1; i >= 0; i--) {
159-
children.get(i).attemptFailedDuration(error, delay);
200+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
201+
try (Scope s = enterScope()) {
202+
for (int i = children.size() - 1; i >= 0; i--) {
203+
children.get(i).attemptFailedDuration(error, delay);
204+
}
160205
}
161206
}
162207

163208
@Override
164209
public void attemptFailedRetriesExhausted(Throwable error) {
165-
for (int i = children.size() - 1; i >= 0; i--) {
166-
children.get(i).attemptFailedRetriesExhausted(error);
210+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
211+
try (Scope s = enterScope()) {
212+
for (int i = children.size() - 1; i >= 0; i--) {
213+
children.get(i).attemptFailedRetriesExhausted(error);
214+
}
167215
}
168216
}
169217

170218
@Override
171219
public void attemptPermanentFailure(Throwable error) {
172-
for (int i = children.size() - 1; i >= 0; i--) {
173-
children.get(i).attemptPermanentFailure(error);
220+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
221+
try (Scope s = enterScope()) {
222+
for (int i = children.size() - 1; i >= 0; i--) {
223+
children.get(i).attemptPermanentFailure(error);
224+
}
174225
}
175226
}
176227

‎sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java‎

Lines changed: 142 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -125,29 +125,90 @@ void testInScope_childScopeCloseThrows() {
125125
inOrder.verify(scope1).close();
126126
}
127127

128+
@Test
129+
void testInScope_nullChildScope_isSkipped() {
130+
ApiTracer.Scope scope1 =
131+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
132+
133+
when(child1.inScope()).thenReturn(scope1);
134+
when(child2.inScope()).thenReturn(null);
135+
136+
ApiTracer.Scope compositeScope = compositeTracer.inScope();
137+
compositeScope.close();
138+
139+
verify(scope1).close();
140+
}
141+
128142
@Test
129143
void testOperationSucceeded() {
144+
ApiTracer.Scope scope1 =
145+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
146+
ApiTracer.Scope scope2 =
147+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
148+
when(child1.inScope()).thenReturn(scope1);
149+
when(child2.inScope()).thenReturn(scope2);
150+
130151
compositeTracer.operationSucceeded();
131-
InOrder inOrder = inOrder(child2, child1);
152+
153+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
154+
inOrder.verify(child1).inScope();
155+
inOrder.verify(child2).inScope();
132156
inOrder.verify(child2).operationSucceeded();
133157
inOrder.verify(child1).operationSucceeded();
158+
inOrder.verify(scope2).close();
159+
inOrder.verify(scope1).close();
134160
}
135161

136162
@Test
137163
void testOperationCancelled() {
164+
ApiTracer.Scope scope1 =
165+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
166+
ApiTracer.Scope scope2 =
167+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
168+
when(child1.inScope()).thenReturn(scope1);
169+
when(child2.inScope()).thenReturn(scope2);
170+
138171
compositeTracer.operationCancelled();
139-
InOrder inOrder = inOrder(child2, child1);
172+
173+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
174+
inOrder.verify(child1).inScope();
175+
inOrder.verify(child2).inScope();
140176
inOrder.verify(child2).operationCancelled();
141177
inOrder.verify(child1).operationCancelled();
178+
inOrder.verify(scope2).close();
179+
inOrder.verify(scope1).close();
142180
}
143181

144182
@Test
145183
void testOperationFailed() {
184+
ApiTracer.Scope scope1 =
185+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
186+
ApiTracer.Scope scope2 =
187+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
188+
when(child1.inScope()).thenReturn(scope1);
189+
when(child2.inScope()).thenReturn(scope2);
190+
146191
Throwable error = new RuntimeException("test error");
147192
compositeTracer.operationFailed(error);
148-
InOrder inOrder = inOrder(child2, child1);
193+
194+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
195+
inOrder.verify(child1).inScope();
196+
inOrder.verify(child2).inScope();
149197
inOrder.verify(child2).operationFailed(error);
150198
inOrder.verify(child1).operationFailed(error);
199+
inOrder.verify(scope2).close();
200+
inOrder.verify(scope1).close();
201+
}
202+
203+
@Test
204+
void testOperationSucceeded_whenInScopeThrows_stillNotifiesChildren() {
205+
when(child1.inScope()).thenThrow(new RuntimeException("scope failure"));
206+
207+
compositeTracer.operationSucceeded();
208+
209+
InOrder inOrder = inOrder(child2, child1);
210+
inOrder.verify(child2).operationSucceeded();
211+
inOrder.verify(child1).operationSucceeded();
151212
}
152213

153214
@Test
@@ -176,57 +237,129 @@ void testAttemptStarted() {
176237

177238
@Test
178239
void testAttemptSucceeded() {
240+
ApiTracer.Scope scope1 =
241+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
242+
ApiTracer.Scope scope2 =
243+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
244+
when(child1.inScope()).thenReturn(scope1);
245+
when(child2.inScope()).thenReturn(scope2);
246+
179247
compositeTracer.attemptSucceeded();
180-
InOrder inOrder = inOrder(child2, child1);
248+
249+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
250+
inOrder.verify(child1).inScope();
251+
inOrder.verify(child2).inScope();
181252
inOrder.verify(child2).attemptSucceeded();
182253
inOrder.verify(child1).attemptSucceeded();
254+
inOrder.verify(scope2).close();
255+
inOrder.verify(scope1).close();
183256
}
184257

185258
@Test
186259
void testAttemptCancelled() {
260+
ApiTracer.Scope scope1 =
261+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
262+
ApiTracer.Scope scope2 =
263+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
264+
when(child1.inScope()).thenReturn(scope1);
265+
when(child2.inScope()).thenReturn(scope2);
266+
187267
compositeTracer.attemptCancelled();
188-
InOrder inOrder = inOrder(child2, child1);
268+
269+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
270+
inOrder.verify(child1).inScope();
271+
inOrder.verify(child2).inScope();
189272
inOrder.verify(child2).attemptCancelled();
190273
inOrder.verify(child1).attemptCancelled();
274+
inOrder.verify(scope2).close();
275+
inOrder.verify(scope1).close();
191276
}
192277

193278
@Test
194279
@SuppressWarnings("deprecation")
195280
void testAttemptFailedDeprecated() {
281+
ApiTracer.Scope scope1 =
282+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
283+
ApiTracer.Scope scope2 =
284+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
285+
when(child1.inScope()).thenReturn(scope1);
286+
when(child2.inScope()).thenReturn(scope2);
287+
196288
Throwable error = new RuntimeException("test error");
197289
org.threeten.bp.Duration delay = org.threeten.bp.Duration.ofSeconds(1);
198290
compositeTracer.attemptFailed(error, delay);
199-
InOrder inOrder = inOrder(child2, child1);
291+
292+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
293+
inOrder.verify(child1).inScope();
294+
inOrder.verify(child2).inScope();
200295
inOrder.verify(child2).attemptFailed(error, delay);
201296
inOrder.verify(child1).attemptFailed(error, delay);
297+
inOrder.verify(scope2).close();
298+
inOrder.verify(scope1).close();
202299
}
203300

204301
@Test
205302
void testAttemptFailedDuration() {
303+
ApiTracer.Scope scope1 =
304+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
305+
ApiTracer.Scope scope2 =
306+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
307+
when(child1.inScope()).thenReturn(scope1);
308+
when(child2.inScope()).thenReturn(scope2);
309+
206310
Throwable error = new RuntimeException("test error");
207311
java.time.Duration delay = java.time.Duration.ofSeconds(1);
208312
compositeTracer.attemptFailedDuration(error, delay);
209-
InOrder inOrder = inOrder(child2, child1);
313+
314+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
315+
inOrder.verify(child1).inScope();
316+
inOrder.verify(child2).inScope();
210317
inOrder.verify(child2).attemptFailedDuration(error, delay);
211318
inOrder.verify(child1).attemptFailedDuration(error, delay);
319+
inOrder.verify(scope2).close();
320+
inOrder.verify(scope1).close();
212321
}
213322

214323
@Test
215324
void testAttemptFailedRetriesExhausted() {
325+
ApiTracer.Scope scope1 =
326+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
327+
ApiTracer.Scope scope2 =
328+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
329+
when(child1.inScope()).thenReturn(scope1);
330+
when(child2.inScope()).thenReturn(scope2);
331+
216332
Throwable error = new RuntimeException("test error");
217333
compositeTracer.attemptFailedRetriesExhausted(error);
218-
InOrder inOrder = inOrder(child2, child1);
334+
335+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
336+
inOrder.verify(child1).inScope();
337+
inOrder.verify(child2).inScope();
219338
inOrder.verify(child2).attemptFailedRetriesExhausted(error);
220339
inOrder.verify(child1).attemptFailedRetriesExhausted(error);
340+
inOrder.verify(scope2).close();
341+
inOrder.verify(scope1).close();
221342
}
222343

223344
@Test
224345
void testAttemptPermanentFailure() {
346+
ApiTracer.Scope scope1 =
347+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
348+
ApiTracer.Scope scope2 =
349+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
350+
when(child1.inScope()).thenReturn(scope1);
351+
when(child2.inScope()).thenReturn(scope2);
352+
225353
Throwable error = new RuntimeException("test error");
226354
compositeTracer.attemptPermanentFailure(error);
227-
InOrder inOrder = inOrder(child2, child1);
355+
356+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
357+
inOrder.verify(child1).inScope();
358+
inOrder.verify(child2).inScope();
228359
inOrder.verify(child2).attemptPermanentFailure(error);
229360
inOrder.verify(child1).attemptPermanentFailure(error);
361+
inOrder.verify(scope2).close();
362+
inOrder.verify(scope1).close();
230363
}
231364

232365
@Test

0 commit comments

Comments
 (0)