diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 118b2d89a334..41eed5c078c7 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -40,6 +40,12 @@ * A composite implementation of {@link ApiTracer} that delegates all tracing events to a list of * underlying tracers. * + *

Completion and failure callbacks execute asynchronously on background worker threads where no + * trace span is active on thread-local storage. Wrapping those callbacks in {@code try (Scope + * ignored = inScope())} briefly activates the current span on the callback thread before any child + * tracer ends it, allowing sibling tracers (such as logging and metrics) to correlate error logs + * and metric exemplars with the active span before the scope closes on the same thread. + * *

For internal use only. */ @NullMarked @@ -51,13 +57,22 @@ public CompositeTracer(List children) { this.children = ImmutableList.copyOf(children); } + /** + * Opens a scope on each child tracer in forward order and returns a composite scope that closes + * them in reverse (LIFO) order. Null scopes (for example, from unstubbed {@link ApiTracer} mocks + * in tests) are ignored. + */ + @SuppressWarnings("java:S2589") @Override public Scope inScope() { final List childScopes = new ArrayList<>(children.size()); try { for (ApiTracer child : children) { - childScopes.add(child.inScope()); + Scope childScope = child.inScope(); + if (childScope != null) { + childScopes.add(childScope); + } } } catch (RuntimeException e) { for (int i = childScopes.size() - 1; i >= 0; i--) { @@ -89,24 +104,42 @@ public Scope inScope() { }; } + /** + * Activates the operation span on the callback thread before any child ends it so sibling tracers + * (such as metrics exemplars) observe the active span during completion. + */ @Override public void operationSucceeded() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationSucceeded(); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationSucceeded(); + } } } + /** + * Activates the operation span on the callback thread before any child ends it so sibling tracers + * (such as metrics exemplars and cancellation logs) observe the active span. + */ @Override public void operationCancelled() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationCancelled(); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationCancelled(); + } } } + /** + * Activates the operation span on the callback thread before any child ends it so sibling tracers + * (such as metrics exemplars and ERROR-level logs) observe the active span. + */ @Override public void operationFailed(Throwable error) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationFailed(error); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationFailed(error); + } } } @@ -132,45 +165,81 @@ public void attemptStarted(Object request, int attemptNumber) { } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * observe the active attempt span during completion. + */ @Override public void attemptSucceeded() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptSucceeded(); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptSucceeded(); + } } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * observe the active attempt span on cancellation. + */ @Override public void attemptCancelled() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptCancelled(); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptCancelled(); + } } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * (such as DEBUG-level retry logs) observe the active attempt span. + */ @Override public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptFailed(error, delay); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptFailed(error, delay); + } } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * (such as DEBUG-level retry logs) observe the active attempt span. + */ @Override public void attemptFailedDuration(Throwable error, java.time.Duration delay) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptFailedDuration(error, delay); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptFailedDuration(error, delay); + } } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * (such as DEBUG-level exhausted-retry logs) observe the active attempt span. + */ @Override public void attemptFailedRetriesExhausted(Throwable error) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptFailedRetriesExhausted(error); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptFailedRetriesExhausted(error); + } } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * (such as DEBUG-level permanent-failure logs) observe the active attempt span. + */ @Override public void attemptPermanentFailure(Throwable error) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptPermanentFailure(error); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptPermanentFailure(error); + } } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java index 02e5012dc32c..01143ee7747e 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java @@ -29,6 +29,7 @@ */ package com.google.api.gax.tracing; +import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -39,8 +40,19 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import java.time.Duration; import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InOrder; @@ -293,4 +305,154 @@ void testRequestUrlResolved() { inOrder.verify(child1).requestUrlResolved("the-url"); inOrder.verify(child2).requestUrlResolved("the-url"); } + + @Test + void testInScope_nullChildScopeIgnored() { + ApiTracer.Scope scope2 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + when(child1.inScope()).thenReturn(null); + when(child2.inScope()).thenReturn(scope2); + + try (ApiTracer.Scope ignored = compositeTracer.inScope()) { + verify(child1).inScope(); + verify(child2).inScope(); + } + + verify(scope2).close(); + } + + @Test + void testCompletionMethods_enterAndCloseScopeAroundChildren() { + ApiTracer.Scope scope1 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + ApiTracer.Scope scope2 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + when(child1.inScope()).thenReturn(scope1); + when(child2.inScope()).thenReturn(scope2); + + Throwable error = new RuntimeException("test error"); + compositeTracer.operationFailed(error); + + // Scopes open in forward order, callbacks dispatch in reverse order, and scopes close in LIFO + // order. + InOrder inOrder = inOrder(child1, child2, scope1, scope2); + inOrder.verify(child1).inScope(); + inOrder.verify(child2).inScope(); + inOrder.verify(child2).operationFailed(error); + inOrder.verify(child1).operationFailed(error); + inOrder.verify(scope2).close(); + inOrder.verify(scope1).close(); + } + + @Test + void testNestedCompositeTracer_crossThreadScoping() throws Exception { + Span operationSpan = + Span.wrap( + SpanContext.create( + "00000000000000000000000000000001", + "0000000000000010", + TraceFlags.getSampled(), + TraceState.getDefault())); + Span attemptSpan = + Span.wrap( + SpanContext.create( + "00000000000000000000000000000001", + "0000000000000020", + TraceFlags.getSampled(), + TraceState.getDefault())); + + AtomicReference activeSpanRef = new AtomicReference<>(operationSpan); + List outerLoggingContexts = new CopyOnWriteArrayList<>(); + List innerMetricsContexts = new CopyOnWriteArrayList<>(); + + ApiTracer outerLoggingTracer = + new BaseApiTracer() { + @Override + public void attemptFailedDuration(Throwable error, Duration delay) { + outerLoggingContexts.add(Span.current().getSpanContext()); + } + + @Override + public void operationFailed(Throwable error) { + outerLoggingContexts.add(Span.current().getSpanContext()); + } + }; + + ApiTracer innerMetricsTracer = + new BaseApiTracer() { + @Override + public void operationFailed(Throwable error) { + innerMetricsContexts.add(Span.current().getSpanContext()); + } + }; + + ApiTracer innerTracingTracer = + new BaseApiTracer() { + @Override + public Scope inScope() { + Span current = activeSpanRef.get(); + if (current == null) { + return () -> {}; + } + @SuppressWarnings("MustBeClosedChecker") + io.opentelemetry.context.Scope otelScope = current.makeCurrent(); + return otelScope::close; + } + + @Override + public void attemptStarted(Object request, int attemptNumber) { + activeSpanRef.set(attemptSpan); + } + + @Override + public void attemptFailedDuration(Throwable error, Duration delay) { + // Simulates OpenTelemetryTracingTracer ending attemptSpan and falling back to + // operationSpan. + activeSpanRef.set(operationSpan); + } + + @Override + public void operationFailed(Throwable error) { + activeSpanRef.set(null); + } + }; + + // Mirrors ClientContext: CompositeTracer(Logging, CompositeTracer(Metrics, Tracing)) + CompositeTracer nestedTracer = + new CompositeTracer( + Arrays.asList( + outerLoggingTracer, + new CompositeTracer(Arrays.asList(innerMetricsTracer, innerTracingTracer)))); + + // Start the attempt on the caller thread; no long-lived scope should remain open. + nestedTracer.attemptStarted(new Object(), 0); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + + // Dispatch failure callbacks on a separate worker thread, matching async GAX RPC completion. + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + executor + .submit( + () -> { + RuntimeException err = new RuntimeException("transient failure"); + nestedTracer.attemptFailedDuration(err, Duration.ofMillis(10)); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + + nestedTracer.operationFailed(err); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + }) + .get(5, TimeUnit.SECONDS); + } finally { + executor.shutdownNow(); + } + + // Attempt failure logged under attemptSpan; operation failure logged/recorded under + // operationSpan. + assertThat(outerLoggingContexts).hasSize(2); + assertThat(outerLoggingContexts.get(0)).isEqualTo(attemptSpan.getSpanContext()); + assertThat(outerLoggingContexts.get(1)).isEqualTo(operationSpan.getSpanContext()); + + assertThat(innerMetricsContexts).hasSize(1); + assertThat(innerMetricsContexts.get(0)).isEqualTo(operationSpan.getSpanContext()); + } }