Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@
* A composite implementation of {@link ApiTracer} that delegates all tracing events to a list of
* underlying tracers.
*
* <p>Completion and failure callbacks execute asynchronously on background worker threads where no
* trace span is active on thread-local storage. Wrapping those callbacks in {@code try (Scope
* ignored = inScope())} briefly activates the current span on the callback thread before any child
* tracer ends it, allowing sibling tracers (such as logging and metrics) to correlate error logs
* and metric exemplars with the active span before the scope closes on the same thread.
*
* <p>For internal use only.
*/
@NullMarked
Expand All @@ -51,13 +57,22 @@ public CompositeTracer(List<ApiTracer> children) {
this.children = ImmutableList.copyOf(children);
}

/**
* Opens a scope on each child tracer in forward order and returns a composite scope that closes
* them in reverse (LIFO) order. Null scopes (for example, from unstubbed {@link ApiTracer} mocks
* in tests) are ignored.
*/
@SuppressWarnings("java:S2589")
@Override
public Scope inScope() {
final List<Scope> childScopes = new ArrayList<>(children.size());
Comment thread
jinseopkim0 marked this conversation as resolved.
Resolved

try {
for (ApiTracer child : children) {
childScopes.add(child.inScope());
Scope childScope = child.inScope();
Comment thread
jinseopkim0 marked this conversation as resolved.
Resolved
if (childScope != null) {
childScopes.add(childScope);
}
}
} catch (RuntimeException e) {
for (int i = childScopes.size() - 1; i >= 0; i--) {
Expand Down Expand Up @@ -89,24 +104,42 @@ public Scope inScope() {
};
}

/**
* Activates the operation span on the callback thread before any child ends it so sibling tracers
* (such as metrics exemplars) observe the active span during completion.
*/
@Override
public void operationSucceeded() {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationSucceeded();
try (Scope ignored = inScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationSucceeded();
}
}
}

/**
* Activates the operation span on the callback thread before any child ends it so sibling tracers
* (such as metrics exemplars and cancellation logs) observe the active span.
*/
@Override
public void operationCancelled() {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationCancelled();
try (Scope ignored = inScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationCancelled();
}
}
}

/**
* Activates the operation span on the callback thread before any child ends it so sibling tracers
* (such as metrics exemplars and ERROR-level logs) observe the active span.
*/
@Override
public void operationFailed(Throwable error) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationFailed(error);
try (Scope ignored = inScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationFailed(error);
}
}
}

Expand All @@ -132,45 +165,81 @@ public void attemptStarted(Object request, int attemptNumber) {
}
}

/**
* Activates the attempt span on the callback thread before any child ends it so sibling tracers
* observe the active attempt span during completion.
*/
@Override
public void attemptSucceeded() {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptSucceeded();
try (Scope ignored = inScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptSucceeded();
}
}
}

/**
* Activates the attempt span on the callback thread before any child ends it so sibling tracers
* observe the active attempt span on cancellation.
*/
@Override
public void attemptCancelled() {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptCancelled();
try (Scope ignored = inScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptCancelled();
}
}
}

/**
* Activates the attempt span on the callback thread before any child ends it so sibling tracers
* (such as DEBUG-level retry logs) observe the active attempt span.
*/
@Override
public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailed(error, delay);
try (Scope ignored = inScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailed(error, delay);
}
}
}

/**
* Activates the attempt span on the callback thread before any child ends it so sibling tracers
* (such as DEBUG-level retry logs) observe the active attempt span.
*/
@Override
public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailedDuration(error, delay);
try (Scope ignored = inScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailedDuration(error, delay);
}
}
}

/**
* Activates the attempt span on the callback thread before any child ends it so sibling tracers
* (such as DEBUG-level exhausted-retry logs) observe the active attempt span.
*/
@Override
public void attemptFailedRetriesExhausted(Throwable error) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailedRetriesExhausted(error);
try (Scope ignored = inScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailedRetriesExhausted(error);
}
}
}

/**
* Activates the attempt span on the callback thread before any child ends it so sibling tracers
* (such as DEBUG-level permanent-failure logs) observe the active attempt span.
*/
@Override
public void attemptPermanentFailure(Throwable error) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptPermanentFailure(error);
try (Scope ignored = inScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptPermanentFailure(error);
}
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
*/
package com.google.api.gax.tracing;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
Expand All @@ -39,8 +40,19 @@
import static org.mockito.Mockito.when;

import com.google.common.collect.ImmutableMap;
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.SpanContext;
import io.opentelemetry.api.trace.TraceFlags;
import io.opentelemetry.api.trace.TraceState;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InOrder;
Expand Down Expand Up @@ -293,4 +305,154 @@ void testRequestUrlResolved() {
inOrder.verify(child1).requestUrlResolved("the-url");
inOrder.verify(child2).requestUrlResolved("the-url");
}

@Test
void testInScope_nullChildScopeIgnored() {
ApiTracer.Scope scope2 =
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
when(child1.inScope()).thenReturn(null);
when(child2.inScope()).thenReturn(scope2);

try (ApiTracer.Scope ignored = compositeTracer.inScope()) {
verify(child1).inScope();
verify(child2).inScope();
}

verify(scope2).close();
}

@Test
void testCompletionMethods_enterAndCloseScopeAroundChildren() {
ApiTracer.Scope scope1 =
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
ApiTracer.Scope scope2 =
mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations());
when(child1.inScope()).thenReturn(scope1);
when(child2.inScope()).thenReturn(scope2);

Throwable error = new RuntimeException("test error");
compositeTracer.operationFailed(error);

// Scopes open in forward order, callbacks dispatch in reverse order, and scopes close in LIFO
// order.
InOrder inOrder = inOrder(child1, child2, scope1, scope2);
inOrder.verify(child1).inScope();
inOrder.verify(child2).inScope();
inOrder.verify(child2).operationFailed(error);
inOrder.verify(child1).operationFailed(error);
inOrder.verify(scope2).close();
inOrder.verify(scope1).close();
}

@Test
void testNestedCompositeTracer_crossThreadScoping() throws Exception {
Span operationSpan =
Span.wrap(
SpanContext.create(
"00000000000000000000000000000001",
"0000000000000010",
TraceFlags.getSampled(),
TraceState.getDefault()));
Span attemptSpan =
Span.wrap(
SpanContext.create(
"00000000000000000000000000000001",
"0000000000000020",
TraceFlags.getSampled(),
TraceState.getDefault()));

AtomicReference<Span> activeSpanRef = new AtomicReference<>(operationSpan);
List<SpanContext> outerLoggingContexts = new CopyOnWriteArrayList<>();
List<SpanContext> innerMetricsContexts = new CopyOnWriteArrayList<>();

ApiTracer outerLoggingTracer =
new BaseApiTracer() {
@Override
public void attemptFailedDuration(Throwable error, Duration delay) {
outerLoggingContexts.add(Span.current().getSpanContext());
}

@Override
public void operationFailed(Throwable error) {
outerLoggingContexts.add(Span.current().getSpanContext());
}
};

ApiTracer innerMetricsTracer =
new BaseApiTracer() {
@Override
public void operationFailed(Throwable error) {
innerMetricsContexts.add(Span.current().getSpanContext());
}
};

ApiTracer innerTracingTracer =
new BaseApiTracer() {
@Override
public Scope inScope() {
Span current = activeSpanRef.get();
if (current == null) {
return () -> {};
}
@SuppressWarnings("MustBeClosedChecker")
io.opentelemetry.context.Scope otelScope = current.makeCurrent();
return otelScope::close;
}

@Override
public void attemptStarted(Object request, int attemptNumber) {
activeSpanRef.set(attemptSpan);
}

@Override
public void attemptFailedDuration(Throwable error, Duration delay) {
// Simulates OpenTelemetryTracingTracer ending attemptSpan and falling back to
// operationSpan.
activeSpanRef.set(operationSpan);
}

@Override
public void operationFailed(Throwable error) {
activeSpanRef.set(null);
}
};

// Mirrors ClientContext: CompositeTracer(Logging, CompositeTracer(Metrics, Tracing))
CompositeTracer nestedTracer =
new CompositeTracer(
Arrays.asList(
outerLoggingTracer,
new CompositeTracer(Arrays.asList(innerMetricsTracer, innerTracingTracer))));

// Start the attempt on the caller thread; no long-lived scope should remain open.
nestedTracer.attemptStarted(new Object(), 0);
assertThat(Span.current().getSpanContext().isValid()).isFalse();

// Dispatch failure callbacks on a separate worker thread, matching async GAX RPC completion.
ExecutorService executor = Executors.newSingleThreadExecutor();
try {
executor
.submit(
() -> {
RuntimeException err = new RuntimeException("transient failure");
nestedTracer.attemptFailedDuration(err, Duration.ofMillis(10));
assertThat(Span.current().getSpanContext().isValid()).isFalse();

nestedTracer.operationFailed(err);
assertThat(Span.current().getSpanContext().isValid()).isFalse();
})
.get(5, TimeUnit.SECONDS);
} finally {
executor.shutdownNow();
}

// Attempt failure logged under attemptSpan; operation failure logged/recorded under
// operationSpan.
assertThat(outerLoggingContexts).hasSize(2);
assertThat(outerLoggingContexts.get(0)).isEqualTo(attemptSpan.getSpanContext());
assertThat(outerLoggingContexts.get(1)).isEqualTo(operationSpan.getSpanContext());

assertThat(innerMetricsContexts).hasSize(1);
assertThat(innerMetricsContexts.get(0)).isEqualTo(operationSpan.getSpanContext());
}
}
Loading