diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java
index 118b2d89a334..41eed5c078c7 100644
--- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java
+++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java
@@ -40,6 +40,12 @@
* A composite implementation of {@link ApiTracer} that delegates all tracing events to a list of
* underlying tracers.
*
+ *
Completion and failure callbacks execute asynchronously on background worker threads where no
+ * trace span is active on thread-local storage. Wrapping those callbacks in {@code try (Scope
+ * ignored = inScope())} briefly activates the current span on the callback thread before any child
+ * tracer ends it, allowing sibling tracers (such as logging and metrics) to correlate error logs
+ * and metric exemplars with the active span before the scope closes on the same thread.
+ *
*
For internal use only.
*/
@NullMarked
@@ -51,13 +57,22 @@ public CompositeTracer(List children) {
this.children = ImmutableList.copyOf(children);
}
+ /**
+ * Opens a scope on each child tracer in forward order and returns a composite scope that closes
+ * them in reverse (LIFO) order. Null scopes (for example, from unstubbed {@link ApiTracer} mocks
+ * in tests) are ignored.
+ */
+ @SuppressWarnings("java:S2589")
@Override
public Scope inScope() {
final List childScopes = new ArrayList<>(children.size());
try {
for (ApiTracer child : children) {
- childScopes.add(child.inScope());
+ Scope childScope = child.inScope();
+ if (childScope != null) {
+ childScopes.add(childScope);
+ }
}
} catch (RuntimeException e) {
for (int i = childScopes.size() - 1; i >= 0; i--) {
@@ -89,24 +104,42 @@ public Scope inScope() {
};
}
+ /**
+ * Activates the operation span on the callback thread before any child ends it so sibling tracers
+ * (such as metrics exemplars) observe the active span during completion.
+ */
@Override
public void operationSucceeded() {
- for (int i = children.size() - 1; i >= 0; i--) {
- children.get(i).operationSucceeded();
+ try (Scope ignored = inScope()) {
+ for (int i = children.size() - 1; i >= 0; i--) {
+ children.get(i).operationSucceeded();
+ }
}
}
+ /**
+ * Activates the operation span on the callback thread before any child ends it so sibling tracers
+ * (such as metrics exemplars and cancellation logs) observe the active span.
+ */
@Override
public void operationCancelled() {
- for (int i = children.size() - 1; i >= 0; i--) {
- children.get(i).operationCancelled();
+ try (Scope ignored = inScope()) {
+ for (int i = children.size() - 1; i >= 0; i--) {
+ children.get(i).operationCancelled();
+ }
}
}
+ /**
+ * Activates the operation span on the callback thread before any child ends it so sibling tracers
+ * (such as metrics exemplars and ERROR-level logs) observe the active span.
+ */
@Override
public void operationFailed(Throwable error) {
- for (int i = children.size() - 1; i >= 0; i--) {
- children.get(i).operationFailed(error);
+ try (Scope ignored = inScope()) {
+ for (int i = children.size() - 1; i >= 0; i--) {
+ children.get(i).operationFailed(error);
+ }
}
}
@@ -132,45 +165,81 @@ public void attemptStarted(Object request, int attemptNumber) {
}
}
+ /**
+ * Activates the attempt span on the callback thread before any child ends it so sibling tracers
+ * observe the active attempt span during completion.
+ */
@Override
public void attemptSucceeded() {
- for (int i = children.size() - 1; i >= 0; i--) {
- children.get(i).attemptSucceeded();
+ try (Scope ignored = inScope()) {
+ for (int i = children.size() - 1; i >= 0; i--) {
+ children.get(i).attemptSucceeded();
+ }
}
}
+ /**
+ * Activates the attempt span on the callback thread before any child ends it so sibling tracers
+ * observe the active attempt span on cancellation.
+ */
@Override
public void attemptCancelled() {
- for (int i = children.size() - 1; i >= 0; i--) {
- children.get(i).attemptCancelled();
+ try (Scope ignored = inScope()) {
+ for (int i = children.size() - 1; i >= 0; i--) {
+ children.get(i).attemptCancelled();
+ }
}
}
+ /**
+ * Activates the attempt span on the callback thread before any child ends it so sibling tracers
+ * (such as DEBUG-level retry logs) observe the active attempt span.
+ */
@Override
public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) {
- for (int i = children.size() - 1; i >= 0; i--) {
- children.get(i).attemptFailed(error, delay);
+ try (Scope ignored = inScope()) {
+ for (int i = children.size() - 1; i >= 0; i--) {
+ children.get(i).attemptFailed(error, delay);
+ }
}
}
+ /**
+ * Activates the attempt span on the callback thread before any child ends it so sibling tracers
+ * (such as DEBUG-level retry logs) observe the active attempt span.
+ */
@Override
public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
- for (int i = children.size() - 1; i >= 0; i--) {
- children.get(i).attemptFailedDuration(error, delay);
+ try (Scope ignored = inScope()) {
+ for (int i = children.size() - 1; i >= 0; i--) {
+ children.get(i).attemptFailedDuration(error, delay);
+ }
}
}
+ /**
+ * Activates the attempt span on the callback thread before any child ends it so sibling tracers
+ * (such as DEBUG-level exhausted-retry logs) observe the active attempt span.
+ */
@Override
public void attemptFailedRetriesExhausted(Throwable error) {
- for (int i = children.size() - 1; i >= 0; i--) {
- children.get(i).attemptFailedRetriesExhausted(error);
+ try (Scope ignored = inScope()) {
+ for (int i = children.size() - 1; i >= 0; i--) {
+ children.get(i).attemptFailedRetriesExhausted(error);
+ }
}
}
+ /**
+ * Activates the attempt span on the callback thread before any child ends it so sibling tracers
+ * (such as DEBUG-level permanent-failure logs) observe the active attempt span.
+ */
@Override
public void attemptPermanentFailure(Throwable error) {
- for (int i = children.size() - 1; i >= 0; i--) {
- children.get(i).attemptPermanentFailure(error);
+ try (Scope ignored = inScope()) {
+ for (int i = children.size() - 1; i >= 0; i--) {
+ children.get(i).attemptPermanentFailure(error);
+ }
}
}
diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java
index 02e5012dc32c..01143ee7747e 100644
--- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java
+++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java
@@ -29,6 +29,7 @@
*/
package com.google.api.gax.tracing;
+import static com.google.common.truth.Truth.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -39,8 +40,19 @@
import static org.mockito.Mockito.when;
import com.google.common.collect.ImmutableMap;
+import io.opentelemetry.api.trace.Span;
+import io.opentelemetry.api.trace.SpanContext;
+import io.opentelemetry.api.trace.TraceFlags;
+import io.opentelemetry.api.trace.TraceState;
+import java.time.Duration;
import java.util.Arrays;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InOrder;
@@ -293,4 +305,154 @@ void testRequestUrlResolved() {
inOrder.verify(child1).requestUrlResolved("the-url");
inOrder.verify(child2).requestUrlResolved("the-url");
}
+
+ @Test
+ void testInScope_nullChildScopeIgnored() {
+ ApiTracer.Scope scope2 =
+ mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
+ when(child1.inScope()).thenReturn(null);
+ when(child2.inScope()).thenReturn(scope2);
+
+ try (ApiTracer.Scope ignored = compositeTracer.inScope()) {
+ verify(child1).inScope();
+ verify(child2).inScope();
+ }
+
+ verify(scope2).close();
+ }
+
+ @Test
+ void testCompletionMethods_enterAndCloseScopeAroundChildren() {
+ ApiTracer.Scope scope1 =
+ mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
+ ApiTracer.Scope scope2 =
+ mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
+ when(child1.inScope()).thenReturn(scope1);
+ when(child2.inScope()).thenReturn(scope2);
+
+ Throwable error = new RuntimeException("test error");
+ compositeTracer.operationFailed(error);
+
+ // Scopes open in forward order, callbacks dispatch in reverse order, and scopes close in LIFO
+ // order.
+ InOrder inOrder = inOrder(child1, child2, scope1, scope2);
+ inOrder.verify(child1).inScope();
+ inOrder.verify(child2).inScope();
+ inOrder.verify(child2).operationFailed(error);
+ inOrder.verify(child1).operationFailed(error);
+ inOrder.verify(scope2).close();
+ inOrder.verify(scope1).close();
+ }
+
+ @Test
+ void testNestedCompositeTracer_crossThreadScoping() throws Exception {
+ Span operationSpan =
+ Span.wrap(
+ SpanContext.create(
+ "00000000000000000000000000000001",
+ "0000000000000010",
+ TraceFlags.getSampled(),
+ TraceState.getDefault()));
+ Span attemptSpan =
+ Span.wrap(
+ SpanContext.create(
+ "00000000000000000000000000000001",
+ "0000000000000020",
+ TraceFlags.getSampled(),
+ TraceState.getDefault()));
+
+ AtomicReference activeSpanRef = new AtomicReference<>(operationSpan);
+ List outerLoggingContexts = new CopyOnWriteArrayList<>();
+ List innerMetricsContexts = new CopyOnWriteArrayList<>();
+
+ ApiTracer outerLoggingTracer =
+ new BaseApiTracer() {
+ @Override
+ public void attemptFailedDuration(Throwable error, Duration delay) {
+ outerLoggingContexts.add(Span.current().getSpanContext());
+ }
+
+ @Override
+ public void operationFailed(Throwable error) {
+ outerLoggingContexts.add(Span.current().getSpanContext());
+ }
+ };
+
+ ApiTracer innerMetricsTracer =
+ new BaseApiTracer() {
+ @Override
+ public void operationFailed(Throwable error) {
+ innerMetricsContexts.add(Span.current().getSpanContext());
+ }
+ };
+
+ ApiTracer innerTracingTracer =
+ new BaseApiTracer() {
+ @Override
+ public Scope inScope() {
+ Span current = activeSpanRef.get();
+ if (current == null) {
+ return () -> {};
+ }
+ @SuppressWarnings("MustBeClosedChecker")
+ io.opentelemetry.context.Scope otelScope = current.makeCurrent();
+ return otelScope::close;
+ }
+
+ @Override
+ public void attemptStarted(Object request, int attemptNumber) {
+ activeSpanRef.set(attemptSpan);
+ }
+
+ @Override
+ public void attemptFailedDuration(Throwable error, Duration delay) {
+ // Simulates OpenTelemetryTracingTracer ending attemptSpan and falling back to
+ // operationSpan.
+ activeSpanRef.set(operationSpan);
+ }
+
+ @Override
+ public void operationFailed(Throwable error) {
+ activeSpanRef.set(null);
+ }
+ };
+
+ // Mirrors ClientContext: CompositeTracer(Logging, CompositeTracer(Metrics, Tracing))
+ CompositeTracer nestedTracer =
+ new CompositeTracer(
+ Arrays.asList(
+ outerLoggingTracer,
+ new CompositeTracer(Arrays.asList(innerMetricsTracer, innerTracingTracer))));
+
+ // Start the attempt on the caller thread; no long-lived scope should remain open.
+ nestedTracer.attemptStarted(new Object(), 0);
+ assertThat(Span.current().getSpanContext().isValid()).isFalse();
+
+ // Dispatch failure callbacks on a separate worker thread, matching async GAX RPC completion.
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ try {
+ executor
+ .submit(
+ () -> {
+ RuntimeException err = new RuntimeException("transient failure");
+ nestedTracer.attemptFailedDuration(err, Duration.ofMillis(10));
+ assertThat(Span.current().getSpanContext().isValid()).isFalse();
+
+ nestedTracer.operationFailed(err);
+ assertThat(Span.current().getSpanContext().isValid()).isFalse();
+ })
+ .get(5, TimeUnit.SECONDS);
+ } finally {
+ executor.shutdownNow();
+ }
+
+ // Attempt failure logged under attemptSpan; operation failure logged/recorded under
+ // operationSpan.
+ assertThat(outerLoggingContexts).hasSize(2);
+ assertThat(outerLoggingContexts.get(0)).isEqualTo(attemptSpan.getSpanContext());
+ assertThat(outerLoggingContexts.get(1)).isEqualTo(operationSpan.getSpanContext());
+
+ assertThat(innerMetricsContexts).hasSize(1);
+ assertThat(innerMetricsContexts.get(0)).isEqualTo(operationSpan.getSpanContext());
+ }
}