Skip to content

Commit 24e64ee

Browse files
jinseopkim0jinseopkim0
committed
feat(gax): correlate M3 client duration metrics with T3 span exemplars
1 parent 2f08818 commit 24e64ee

3 files changed

Lines changed: 199 additions & 27 deletions

File tree

‎sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java‎

Lines changed: 65 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@
4545
@NullMarked
4646
@InternalApi
4747
class CompositeTracer extends BaseApiTracer {
48+
private static final Scope NO_OP_SCOPE = () -> {};
49+
4850
private final List<ApiTracer> children;
4951

5052
public CompositeTracer(List<ApiTracer> children) {
@@ -89,24 +91,51 @@ public Scope inScope() {
8991
};
9092
}
9193

94+
/**
95+
* Enters the tracer's ambient scope safely, returning a no-op {@link Scope} if entering fails.
96+
* This avoids allocating runnables, throwing exceptions, or returning null during lifecycle
97+
* notifications.
98+
*
99+
* <p>Entering the ambient scope ensures active trace span context is present on the thread so
100+
* that OpenTelemetry metric measurements recorded in lifecycle callbacks attach exemplars
101+
* pointing to the active span.
102+
*/
103+
private Scope enterScope() {
104+
try {
105+
return inScope();
106+
} catch (RuntimeException e) {
107+
// Ignore to prevent disrupting the lifecycle notification
108+
return NO_OP_SCOPE;
109+
}
110+
}
111+
92112
@Override
93113
public void operationSucceeded() {
94-
for (int i = children.size() - 1; i >= 0; i--) {
95-
children.get(i).operationSucceeded();
114+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
115+
try (Scope s = enterScope()) {
116+
for (int i = children.size() - 1; i >= 0; i--) {
117+
children.get(i).operationSucceeded();
118+
}
96119
}
97120
}
98121

99122
@Override
100123
public void operationCancelled() {
101-
for (int i = children.size() - 1; i >= 0; i--) {
102-
children.get(i).operationCancelled();
124+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
125+
try (Scope s = enterScope()) {
126+
for (int i = children.size() - 1; i >= 0; i--) {
127+
children.get(i).operationCancelled();
128+
}
103129
}
104130
}
105131

106132
@Override
107133
public void operationFailed(Throwable error) {
108-
for (int i = children.size() - 1; i >= 0; i--) {
109-
children.get(i).operationFailed(error);
134+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
135+
try (Scope s = enterScope()) {
136+
for (int i = children.size() - 1; i >= 0; i--) {
137+
children.get(i).operationFailed(error);
138+
}
110139
}
111140
}
112141

@@ -134,43 +163,61 @@ public void attemptStarted(Object request, int attemptNumber) {
134163

135164
@Override
136165
public void attemptSucceeded() {
137-
for (int i = children.size() - 1; i >= 0; i--) {
138-
children.get(i).attemptSucceeded();
166+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
167+
try (Scope s = enterScope()) {
168+
for (int i = children.size() - 1; i >= 0; i--) {
169+
children.get(i).attemptSucceeded();
170+
}
139171
}
140172
}
141173

142174
@Override
143175
public void attemptCancelled() {
144-
for (int i = children.size() - 1; i >= 0; i--) {
145-
children.get(i).attemptCancelled();
176+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
177+
try (Scope s = enterScope()) {
178+
for (int i = children.size() - 1; i >= 0; i--) {
179+
children.get(i).attemptCancelled();
180+
}
146181
}
147182
}
148183

149184
@Override
150185
public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) {
151-
for (int i = children.size() - 1; i >= 0; i--) {
152-
children.get(i).attemptFailed(error, delay);
186+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
187+
try (Scope s = enterScope()) {
188+
for (int i = children.size() - 1; i >= 0; i--) {
189+
children.get(i).attemptFailed(error, delay);
190+
}
153191
}
154192
}
155193

156194
@Override
157195
public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
158-
for (int i = children.size() - 1; i >= 0; i--) {
159-
children.get(i).attemptFailedDuration(error, delay);
196+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
197+
try (Scope s = enterScope()) {
198+
for (int i = children.size() - 1; i >= 0; i--) {
199+
children.get(i).attemptFailedDuration(error, delay);
200+
}
160201
}
161202
}
162203

163204
@Override
164205
public void attemptFailedRetriesExhausted(Throwable error) {
165-
for (int i = children.size() - 1; i >= 0; i--) {
166-
children.get(i).attemptFailedRetriesExhausted(error);
206+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
207+
try (Scope s = enterScope()) {
208+
for (int i = children.size() - 1; i >= 0; i--) {
209+
children.get(i).attemptFailedRetriesExhausted(error);
210+
}
167211
}
168212
}
169213

170214
@Override
171215
public void attemptPermanentFailure(Throwable error) {
172-
for (int i = children.size() - 1; i >= 0; i--) {
173-
children.get(i).attemptPermanentFailure(error);
216+
// Activate ambient span context so child tracers (e.g. metrics) can sample trace exemplars
217+
try (Scope s = enterScope()) {
218+
for (int i = children.size() - 1; i >= 0; i--) {
219+
children.get(i).attemptPermanentFailure(error);
220+
}
174221
}
175222
}
176223

‎sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerFactoryTest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,8 @@ void testNewTracerWithParentAndSpanName() {
5959

6060
ApiTracer tracer1 = mock(ApiTracer.class, Mockito.withSettings().withoutAnnotations());
6161
ApiTracer tracer2 = mock(ApiTracer.class, Mockito.withSettings().withoutAnnotations());
62+
when(tracer1.inScope()).thenReturn(() -> {});
63+
when(tracer2.inScope()).thenReturn(() -> {});
6264

6365
when(childFactory1.newTracer(parent, spanName, operationType)).thenReturn(tracer1);
6466
when(childFactory2.newTracer(parent, spanName, operationType)).thenReturn(tracer2);
@@ -81,6 +83,8 @@ void testNewTracerWithApiTracerContext() {
8183

8284
ApiTracer tracer1 = mock(ApiTracer.class, Mockito.withSettings().withoutAnnotations());
8385
ApiTracer tracer2 = mock(ApiTracer.class, Mockito.withSettings().withoutAnnotations());
86+
when(tracer1.inScope()).thenReturn(() -> {});
87+
when(tracer2.inScope()).thenReturn(() -> {});
8488

8589
when(childFactory1.newTracer(parent, context)).thenReturn(tracer1);
8690
when(childFactory2.newTracer(parent, context)).thenReturn(tracer2);
@@ -117,6 +121,8 @@ void testWithContext() {
117121

118122
ApiTracer tracer1 = mock(ApiTracer.class, Mockito.withSettings().withoutAnnotations());
119123
ApiTracer tracer2 = mock(ApiTracer.class, Mockito.withSettings().withoutAnnotations());
124+
when(tracer1.inScope()).thenReturn(() -> {});
125+
when(tracer2.inScope()).thenReturn(() -> {});
120126

121127
when(contextualizedFactory1.newTracer(parent, tracerContext)).thenReturn(tracer1);
122128
when(contextualizedFactory2.newTracer(parent, tracerContext)).thenReturn(tracer2);

‎sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java‎

Lines changed: 128 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -127,27 +127,74 @@ void testInScope_childScopeCloseThrows() {
127127

128128
@Test
129129
void testOperationSucceeded() {
130+
ApiTracer.Scope scope1 =
131+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
132+
ApiTracer.Scope scope2 =
133+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
134+
when(child1.inScope()).thenReturn(scope1);
135+
when(child2.inScope()).thenReturn(scope2);
136+
130137
compositeTracer.operationSucceeded();
131-
InOrder inOrder = inOrder(child2, child1);
138+
139+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
140+
inOrder.verify(child1).inScope();
141+
inOrder.verify(child2).inScope();
132142
inOrder.verify(child2).operationSucceeded();
133143
inOrder.verify(child1).operationSucceeded();
144+
inOrder.verify(scope2).close();
145+
inOrder.verify(scope1).close();
134146
}
135147

136148
@Test
137149
void testOperationCancelled() {
150+
ApiTracer.Scope scope1 =
151+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
152+
ApiTracer.Scope scope2 =
153+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
154+
when(child1.inScope()).thenReturn(scope1);
155+
when(child2.inScope()).thenReturn(scope2);
156+
138157
compositeTracer.operationCancelled();
139-
InOrder inOrder = inOrder(child2, child1);
158+
159+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
160+
inOrder.verify(child1).inScope();
161+
inOrder.verify(child2).inScope();
140162
inOrder.verify(child2).operationCancelled();
141163
inOrder.verify(child1).operationCancelled();
164+
inOrder.verify(scope2).close();
165+
inOrder.verify(scope1).close();
142166
}
143167

144168
@Test
145169
void testOperationFailed() {
170+
ApiTracer.Scope scope1 =
171+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
172+
ApiTracer.Scope scope2 =
173+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
174+
when(child1.inScope()).thenReturn(scope1);
175+
when(child2.inScope()).thenReturn(scope2);
176+
146177
Throwable error = new RuntimeException("test error");
147178
compositeTracer.operationFailed(error);
148-
InOrder inOrder = inOrder(child2, child1);
179+
180+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
181+
inOrder.verify(child1).inScope();
182+
inOrder.verify(child2).inScope();
149183
inOrder.verify(child2).operationFailed(error);
150184
inOrder.verify(child1).operationFailed(error);
185+
inOrder.verify(scope2).close();
186+
inOrder.verify(scope1).close();
187+
}
188+
189+
@Test
190+
void testOperationSucceeded_whenInScopeThrows_stillNotifiesChildren() {
191+
when(child1.inScope()).thenThrow(new RuntimeException("scope failure"));
192+
193+
compositeTracer.operationSucceeded();
194+
195+
InOrder inOrder = inOrder(child2, child1);
196+
inOrder.verify(child2).operationSucceeded();
197+
inOrder.verify(child1).operationSucceeded();
151198
}
152199

153200
@Test
@@ -176,57 +223,129 @@ void testAttemptStarted() {
176223

177224
@Test
178225
void testAttemptSucceeded() {
226+
ApiTracer.Scope scope1 =
227+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
228+
ApiTracer.Scope scope2 =
229+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
230+
when(child1.inScope()).thenReturn(scope1);
231+
when(child2.inScope()).thenReturn(scope2);
232+
179233
compositeTracer.attemptSucceeded();
180-
InOrder inOrder = inOrder(child2, child1);
234+
235+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
236+
inOrder.verify(child1).inScope();
237+
inOrder.verify(child2).inScope();
181238
inOrder.verify(child2).attemptSucceeded();
182239
inOrder.verify(child1).attemptSucceeded();
240+
inOrder.verify(scope2).close();
241+
inOrder.verify(scope1).close();
183242
}
184243

185244
@Test
186245
void testAttemptCancelled() {
246+
ApiTracer.Scope scope1 =
247+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
248+
ApiTracer.Scope scope2 =
249+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
250+
when(child1.inScope()).thenReturn(scope1);
251+
when(child2.inScope()).thenReturn(scope2);
252+
187253
compositeTracer.attemptCancelled();
188-
InOrder inOrder = inOrder(child2, child1);
254+
255+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
256+
inOrder.verify(child1).inScope();
257+
inOrder.verify(child2).inScope();
189258
inOrder.verify(child2).attemptCancelled();
190259
inOrder.verify(child1).attemptCancelled();
260+
inOrder.verify(scope2).close();
261+
inOrder.verify(scope1).close();
191262
}
192263

193264
@Test
194265
@SuppressWarnings("deprecation")
195266
void testAttemptFailedDeprecated() {
267+
ApiTracer.Scope scope1 =
268+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
269+
ApiTracer.Scope scope2 =
270+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
271+
when(child1.inScope()).thenReturn(scope1);
272+
when(child2.inScope()).thenReturn(scope2);
273+
196274
Throwable error = new RuntimeException("test error");
197275
org.threeten.bp.Duration delay = org.threeten.bp.Duration.ofSeconds(1);
198276
compositeTracer.attemptFailed(error, delay);
199-
InOrder inOrder = inOrder(child2, child1);
277+
278+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
279+
inOrder.verify(child1).inScope();
280+
inOrder.verify(child2).inScope();
200281
inOrder.verify(child2).attemptFailed(error, delay);
201282
inOrder.verify(child1).attemptFailed(error, delay);
283+
inOrder.verify(scope2).close();
284+
inOrder.verify(scope1).close();
202285
}
203286

204287
@Test
205288
void testAttemptFailedDuration() {
289+
ApiTracer.Scope scope1 =
290+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
291+
ApiTracer.Scope scope2 =
292+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
293+
when(child1.inScope()).thenReturn(scope1);
294+
when(child2.inScope()).thenReturn(scope2);
295+
206296
Throwable error = new RuntimeException("test error");
207297
java.time.Duration delay = java.time.Duration.ofSeconds(1);
208298
compositeTracer.attemptFailedDuration(error, delay);
209-
InOrder inOrder = inOrder(child2, child1);
299+
300+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
301+
inOrder.verify(child1).inScope();
302+
inOrder.verify(child2).inScope();
210303
inOrder.verify(child2).attemptFailedDuration(error, delay);
211304
inOrder.verify(child1).attemptFailedDuration(error, delay);
305+
inOrder.verify(scope2).close();
306+
inOrder.verify(scope1).close();
212307
}
213308

214309
@Test
215310
void testAttemptFailedRetriesExhausted() {
311+
ApiTracer.Scope scope1 =
312+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
313+
ApiTracer.Scope scope2 =
314+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
315+
when(child1.inScope()).thenReturn(scope1);
316+
when(child2.inScope()).thenReturn(scope2);
317+
216318
Throwable error = new RuntimeException("test error");
217319
compositeTracer.attemptFailedRetriesExhausted(error);
218-
InOrder inOrder = inOrder(child2, child1);
320+
321+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
322+
inOrder.verify(child1).inScope();
323+
inOrder.verify(child2).inScope();
219324
inOrder.verify(child2).attemptFailedRetriesExhausted(error);
220325
inOrder.verify(child1).attemptFailedRetriesExhausted(error);
326+
inOrder.verify(scope2).close();
327+
inOrder.verify(scope1).close();
221328
}
222329

223330
@Test
224331
void testAttemptPermanentFailure() {
332+
ApiTracer.Scope scope1 =
333+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
334+
ApiTracer.Scope scope2 =
335+
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
336+
when(child1.inScope()).thenReturn(scope1);
337+
when(child2.inScope()).thenReturn(scope2);
338+
225339
Throwable error = new RuntimeException("test error");
226340
compositeTracer.attemptPermanentFailure(error);
227-
InOrder inOrder = inOrder(child2, child1);
341+
342+
InOrder inOrder = inOrder(child1, child2, scope2, scope1);
343+
inOrder.verify(child1).inScope();
344+
inOrder.verify(child2).inScope();
228345
inOrder.verify(child2).attemptPermanentFailure(error);
229346
inOrder.verify(child1).attemptPermanentFailure(error);
347+
inOrder.verify(scope2).close();
348+
inOrder.verify(scope1).close();
230349
}
231350

232351
@Test

0 commit comments

Comments
 (0)