From 050b117d6ce20cfdd4e676fa051da52ef2964e9e Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 11:14:06 -0500 Subject: [PATCH 01/11] feat(gax): enter inScope() during CompositeTracer completion callbacks --- .../api/gax/tracing/CompositeTracer.java | 59 ++++--- .../api/gax/tracing/CompositeTracerTest.java | 153 ++++++++++++++++++ 2 files changed, 193 insertions(+), 19 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 118b2d89a334..71f476aeac6b 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -57,7 +57,10 @@ public Scope inScope() { try { for (ApiTracer child : children) { - childScopes.add(child.inScope()); + Scope childScope = child.inScope(); + if (childScope != null) { + childScopes.add(childScope); + } } } catch (RuntimeException e) { for (int i = childScopes.size() - 1; i >= 0; i--) { @@ -91,22 +94,28 @@ public Scope inScope() { @Override public void operationSucceeded() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationSucceeded(); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationSucceeded(); + } } } @Override public void operationCancelled() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationCancelled(); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationCancelled(); + } } } @Override public void operationFailed(Throwable error) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).operationFailed(error); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).operationFailed(error); + } } } @@ -134,43 +143,55 @@ public void attemptStarted(Object request, int attemptNumber) { @Override public void attemptSucceeded() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptSucceeded(); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptSucceeded(); + } } } @Override public void attemptCancelled() { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptCancelled(); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptCancelled(); + } } } @Override public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptFailed(error, delay); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptFailed(error, delay); + } } } @Override public void attemptFailedDuration(Throwable error, java.time.Duration delay) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptFailedDuration(error, delay); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptFailedDuration(error, delay); + } } } @Override public void attemptFailedRetriesExhausted(Throwable error) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptFailedRetriesExhausted(error); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptFailedRetriesExhausted(error); + } } } @Override public void attemptPermanentFailure(Throwable error) { - for (int i = children.size() - 1; i >= 0; i--) { - children.get(i).attemptPermanentFailure(error); + try (Scope ignored = inScope()) { + for (int i = children.size() - 1; i >= 0; i--) { + children.get(i).attemptPermanentFailure(error); + } } } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java index 02e5012dc32c..d98297347726 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java @@ -29,6 +29,7 @@ */ package com.google.api.gax.tracing; +import static com.google.common.truth.Truth.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -39,8 +40,19 @@ import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceFlags; +import io.opentelemetry.api.trace.TraceState; +import java.time.Duration; import java.util.Arrays; +import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.InOrder; @@ -293,4 +305,145 @@ void testRequestUrlResolved() { inOrder.verify(child1).requestUrlResolved("the-url"); inOrder.verify(child2).requestUrlResolved("the-url"); } + + @Test + void testInScope_nullChildScopeIgnored() { + ApiTracer.Scope scope2 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + when(child1.inScope()).thenReturn(null); + when(child2.inScope()).thenReturn(scope2); + + try (ApiTracer.Scope ignored = compositeTracer.inScope()) { + verify(child1).inScope(); + verify(child2).inScope(); + } + + verify(scope2).close(); + } + + @Test + void testCompletionMethods_enterAndCloseScopeAroundChildren() { + ApiTracer.Scope scope1 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + ApiTracer.Scope scope2 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + when(child1.inScope()).thenReturn(scope1); + when(child2.inScope()).thenReturn(scope2); + + Throwable error = new RuntimeException("test error"); + compositeTracer.operationFailed(error); + + InOrder inOrder = inOrder(child1, child2, scope1, scope2); + inOrder.verify(child1).inScope(); + inOrder.verify(child2).inScope(); + inOrder.verify(child2).operationFailed(error); + inOrder.verify(child1).operationFailed(error); + inOrder.verify(scope2).close(); + inOrder.verify(scope1).close(); + } + + @Test + void testNestedCompositeTracer_crossThreadScoping() throws Exception { + Span operationSpan = + Span.wrap( + SpanContext.create( + "00000000000000000000000000000001", + "0000000000000010", + TraceFlags.getSampled(), + TraceState.getDefault())); + Span attemptSpan = + Span.wrap( + SpanContext.create( + "00000000000000000000000000000001", + "0000000000000020", + TraceFlags.getSampled(), + TraceState.getDefault())); + + AtomicReference activeSpanRef = new AtomicReference<>(operationSpan); + List outerLoggingContexts = new CopyOnWriteArrayList<>(); + List innerMetricsContexts = new CopyOnWriteArrayList<>(); + + ApiTracer outerLoggingTracer = + new BaseApiTracer() { + @Override + public void attemptFailedDuration(Throwable error, Duration delay) { + outerLoggingContexts.add(Span.current().getSpanContext()); + } + + @Override + public void operationFailed(Throwable error) { + outerLoggingContexts.add(Span.current().getSpanContext()); + } + }; + + ApiTracer innerMetricsTracer = + new BaseApiTracer() { + @Override + public void operationFailed(Throwable error) { + innerMetricsContexts.add(Span.current().getSpanContext()); + } + }; + + ApiTracer innerTracingTracer = + new BaseApiTracer() { + @Override + public Scope inScope() { + Span current = activeSpanRef.get(); + if (current == null) { + return () -> {}; + } + io.opentelemetry.context.Scope otelScope = current.makeCurrent(); + return otelScope::close; + } + + @Override + public void attemptStarted(Object request, int attemptNumber) { + activeSpanRef.set(attemptSpan); + } + + @Override + public void attemptFailedDuration(Throwable error, Duration delay) { + activeSpanRef.set(operationSpan); + } + + @Override + public void operationFailed(Throwable error) { + activeSpanRef.set(null); + } + }; + + // Mirrors ClientContext: CompositeTracer(Logging, CompositeTracer(Metrics, Tracing)) + CompositeTracer nestedTracer = + new CompositeTracer( + Arrays.asList( + outerLoggingTracer, + new CompositeTracer(Arrays.asList(innerMetricsTracer, innerTracingTracer)))); + + nestedTracer.attemptStarted(new Object(), 0); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + executor + .submit( + () -> { + RuntimeException err = new RuntimeException("transient failure"); + nestedTracer.attemptFailedDuration(err, Duration.ofMillis(10)); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + + nestedTracer.operationFailed(err); + assertThat(Span.current().getSpanContext().isValid()).isFalse(); + }) + .get(5, TimeUnit.SECONDS); + } finally { + executor.shutdownNow(); + } + + assertThat(outerLoggingContexts).hasSize(2); + assertThat(outerLoggingContexts.get(0)).isEqualTo(attemptSpan.getSpanContext()); + assertThat(outerLoggingContexts.get(1)).isEqualTo(operationSpan.getSpanContext()); + + assertThat(innerMetricsContexts).hasSize(1); + assertThat(innerMetricsContexts.get(0)).isEqualTo(operationSpan.getSpanContext()); + } } From ae0fedf71d1b5b5c94cccb6d4e5948e9b89c51b5 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 11:46:15 -0500 Subject: [PATCH 02/11] fix(gax): suppress MustBeClosedChecker and SonarCloud java:S2589 in CompositeTracer --- .../main/java/com/google/api/gax/tracing/CompositeTracer.java | 2 ++ .../java/com/google/api/gax/tracing/CompositeTracerTest.java | 1 + 2 files changed, 3 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 71f476aeac6b..a077dda8960e 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -51,6 +51,8 @@ public CompositeTracer(List children) { this.children = ImmutableList.copyOf(children); } + // Unstubbed Mockito mocks of ApiTracer may return null despite @NullMarked. + @SuppressWarnings("java:S2589") @Override public Scope inScope() { final List childScopes = new ArrayList<>(children.size()); diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java index d98297347726..681204005066 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java @@ -392,6 +392,7 @@ public Scope inScope() { if (current == null) { return () -> {}; } + @SuppressWarnings("MustBeClosedChecker") io.opentelemetry.context.Scope otelScope = current.makeCurrent(); return otelScope::close; } From 04e2f8e4c480b280354c05fd7b28869c95cc2149 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 11:57:18 -0500 Subject: [PATCH 03/11] fix(gax): close child scopes on Error and make CompositeTracer.inScope() idempotent --- .../api/gax/tracing/CompositeTracer.java | 31 +++++++++----- .../api/gax/tracing/CompositeTracerTest.java | 40 +++++++++++++++++++ 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index a077dda8960e..0eeb1f4f0ce8 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -34,6 +34,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import org.jspecify.annotations.NullMarked; /** @@ -51,8 +52,9 @@ public CompositeTracer(List children) { this.children = ImmutableList.copyOf(children); } - // Unstubbed Mockito mocks of ApiTracer may return null despite @NullMarked. - @SuppressWarnings("java:S2589") + // Unstubbed Mockito mocks of ApiTracer may return null despite @NullMarked, and Error is caught + // alongside RuntimeException solely to close already-opened scopes before rethrowing. + @SuppressWarnings({"java:S2589", "java:S1181"}) @Override public Scope inScope() { final List childScopes = new ArrayList<>(children.size()); @@ -64,32 +66,39 @@ public Scope inScope() { childScopes.add(childScope); } } - } catch (RuntimeException e) { + } catch (RuntimeException | Error e) { for (int i = childScopes.size() - 1; i >= 0; i--) { try { childScopes.get(i).close(); - } catch (RuntimeException suppressed) { + } catch (RuntimeException | Error suppressed) { e.addSuppressed(suppressed); } } throw e; } + AtomicBoolean closed = new AtomicBoolean(); return () -> { - RuntimeException exception = null; + if (!closed.compareAndSet(false, true)) { + return; + } + Throwable primaryError = null; for (int i = childScopes.size() - 1; i >= 0; i--) { try { childScopes.get(i).close(); - } catch (RuntimeException e) { - if (exception == null) { - exception = e; + } catch (RuntimeException | Error e) { + if (primaryError == null) { + primaryError = e; } else { - exception.addSuppressed(e); + primaryError.addSuppressed(e); } } } - if (exception != null) { - throw exception; + if (primaryError instanceof RuntimeException) { + throw (RuntimeException) primaryError; + } + if (primaryError instanceof Error) { + throw (Error) primaryError; } }; } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java index 681204005066..e650f39d6216 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java @@ -83,6 +83,7 @@ void testInScope_lifoOrder() { ApiTracer.Scope compositeScope = compositeTracer.inScope(); compositeScope.close(); + compositeScope.close(); verify(child1).inScope(); verify(child2).inScope(); @@ -90,6 +91,7 @@ void testInScope_lifoOrder() { InOrder inOrder = inOrder(scope2, scope1); inOrder.verify(scope2).close(); inOrder.verify(scope1).close(); + inOrder.verifyNoMoreInteractions(); } @Test @@ -137,6 +139,44 @@ void testInScope_childScopeCloseThrows() { inOrder.verify(scope1).close(); } + @Test + void testInScope_childInScopeThrowsError() { + ApiTracer.Scope scope1 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + AssertionError error = new AssertionError("Creation Error"); + RuntimeException cleanupFailure = new RuntimeException("Cleanup failure"); + + when(child1.inScope()).thenReturn(scope1); + when(child2.inScope()).thenThrow(error); + doThrow(cleanupFailure).when(scope1).close(); + + AssertionError thrown = assertThrows(AssertionError.class, () -> compositeTracer.inScope()); + + assertEquals(error, thrown); + assertThat(Arrays.asList(thrown.getSuppressed())).contains(cleanupFailure); + verify(scope1).close(); + } + + @Test + void testInScope_childScopeCloseThrowsError() { + ApiTracer.Scope scope1 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + ApiTracer.Scope scope2 = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + AssertionError error2 = new AssertionError("Scope 2 close Error"); + + when(child1.inScope()).thenReturn(scope1); + when(child2.inScope()).thenReturn(scope2); + doThrow(error2).when(scope2).close(); + + ApiTracer.Scope compositeScope = compositeTracer.inScope(); + + AssertionError thrown = assertThrows(AssertionError.class, compositeScope::close); + assertEquals(error2, thrown); + verify(scope2).close(); + verify(scope1).close(); + } + @Test void testOperationSucceeded() { compositeTracer.operationSucceeded(); From 70421d7c1534a6b4ed517fc946e547386e1b22e0 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 12:04:19 -0500 Subject: [PATCH 04/11] fix(gax): replace AtomicBoolean with local CompositeScope in CompositeTracer.inScope() --- .../api/gax/tracing/CompositeTracer.java | 49 ++++++++++--------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 0eeb1f4f0ce8..b1f12d6c0e16 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -34,7 +34,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; import org.jspecify.annotations.NullMarked; /** @@ -77,30 +76,36 @@ public Scope inScope() { throw e; } - AtomicBoolean closed = new AtomicBoolean(); - return () -> { - if (!closed.compareAndSet(false, true)) { - return; - } - Throwable primaryError = null; - for (int i = childScopes.size() - 1; i >= 0; i--) { - try { - childScopes.get(i).close(); - } catch (RuntimeException | Error e) { - if (primaryError == null) { - primaryError = e; - } else { - primaryError.addSuppressed(e); + class CompositeScope implements Scope { + private volatile boolean closed; + + @Override + public void close() { + if (closed) { + return; + } + closed = true; + Throwable primaryError = null; + for (int i = childScopes.size() - 1; i >= 0; i--) { + try { + childScopes.get(i).close(); + } catch (RuntimeException | Error e) { + if (primaryError == null) { + primaryError = e; + } else { + primaryError.addSuppressed(e); + } } } + if (primaryError instanceof RuntimeException) { + throw (RuntimeException) primaryError; + } + if (primaryError instanceof Error) { + throw (Error) primaryError; + } } - if (primaryError instanceof RuntimeException) { - throw (RuntimeException) primaryError; - } - if (primaryError instanceof Error) { - throw (Error) primaryError; - } - }; + } + return new CompositeScope(); } @Override From 27557cd493b3e12a31beb2c5347b5d5b5e41f282 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 12:12:01 -0500 Subject: [PATCH 05/11] fix(gax): add empty and single-child fast paths to CompositeTracer.inScope() --- .../api/gax/tracing/CompositeTracer.java | 9 +++++++ .../api/gax/tracing/CompositeTracerTest.java | 24 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index b1f12d6c0e16..a6bcca4b9770 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -45,6 +45,8 @@ @NullMarked @InternalApi class CompositeTracer extends BaseApiTracer { + private static final Scope NOOP_SCOPE = () -> {}; + private final List children; public CompositeTracer(List children) { @@ -56,6 +58,13 @@ public CompositeTracer(List children) { @SuppressWarnings({"java:S2589", "java:S1181"}) @Override public Scope inScope() { + if (children.isEmpty()) { + return NOOP_SCOPE; + } + if (children.size() == 1) { + Scope scope = children.get(0).inScope(); + return scope != null ? scope : NOOP_SCOPE; + } final List childScopes = new ArrayList<>(children.size()); try { diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java index e650f39d6216..54010fdb3128 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java @@ -46,6 +46,7 @@ import io.opentelemetry.api.trace.TraceState; import java.time.Duration; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; @@ -361,6 +362,29 @@ void testInScope_nullChildScopeIgnored() { verify(scope2).close(); } + @Test + void testInScope_emptyAndSingleChildFastPath() { + CompositeTracer emptyTracer = new CompositeTracer(Collections.emptyList()); + try (ApiTracer.Scope ignored = emptyTracer.inScope()) { + // no-op scope closes cleanly + } + + ApiTracer.Scope singleScope = + mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); + when(child1.inScope()).thenReturn(singleScope); + CompositeTracer singleChildTracer = new CompositeTracer(Collections.singletonList(child1)); + try (ApiTracer.Scope scope = singleChildTracer.inScope()) { + assertThat(scope).isSameInstanceAs(singleScope); + } + verify(singleScope).close(); + + when(child2.inScope()).thenReturn(null); + CompositeTracer singleNullScopeTracer = new CompositeTracer(Collections.singletonList(child2)); + try (ApiTracer.Scope ignored = singleNullScopeTracer.inScope()) { + verify(child2).inScope(); + } + } + @Test void testCompletionMethods_enterAndCloseScopeAroundChildren() { ApiTracer.Scope scope1 = From 0ec36953c1b0529ec87d0894f4522fbe491609c2 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 12:18:48 -0500 Subject: [PATCH 06/11] fix(gax): guard CompositeScope.closed with ReentrantLock --- .../api/gax/tracing/CompositeTracer.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index a6bcca4b9770..f613a5afd0b2 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -31,9 +31,11 @@ import com.google.api.core.InternalApi; import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.locks.ReentrantLock; import org.jspecify.annotations.NullMarked; /** @@ -86,14 +88,22 @@ public Scope inScope() { } class CompositeScope implements Scope { - private volatile boolean closed; + private final ReentrantLock lock = new ReentrantLock(); + + @GuardedBy("lock") + private boolean closed; @Override public void close() { - if (closed) { - return; + lock.lock(); + try { + if (closed) { + return; + } + closed = true; + } finally { + lock.unlock(); } - closed = true; Throwable primaryError = null; for (int i = childScopes.size() - 1; i >= 0; i--) { try { From 60029641ca712155bf9ff1364e725c8e5ad034c7 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 12:23:11 -0500 Subject: [PATCH 07/11] fix(gax): use thread-bound anonymous Scope in CompositeTracer.inScope() --- .../api/gax/tracing/CompositeTracer.java | 22 +++++-------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index f613a5afd0b2..4e9b0ca7b6cb 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -31,11 +31,9 @@ import com.google.api.core.InternalApi; import com.google.common.collect.ImmutableList; -import com.google.errorprone.annotations.concurrent.GuardedBy; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.locks.ReentrantLock; import org.jspecify.annotations.NullMarked; /** @@ -87,23 +85,16 @@ public Scope inScope() { throw e; } - class CompositeScope implements Scope { - private final ReentrantLock lock = new ReentrantLock(); - - @GuardedBy("lock") + // Scope is thread-bound and must be closed on the same thread that opened it. + return new Scope() { private boolean closed; @Override public void close() { - lock.lock(); - try { - if (closed) { - return; - } - closed = true; - } finally { - lock.unlock(); + if (closed) { + return; } + closed = true; Throwable primaryError = null; for (int i = childScopes.size() - 1; i >= 0; i--) { try { @@ -123,8 +114,7 @@ public void close() { throw (Error) primaryError; } } - } - return new CompositeScope(); + }; } @Override From 7437a55548d6ecc48b1186aa2e1e7ba33da6a117 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 12:37:05 -0500 Subject: [PATCH 08/11] fix(gax): skip anonymous Scope wrapper when childScopes has 0 or 1 active scope --- .../java/com/google/api/gax/tracing/CompositeTracer.java | 7 +++++++ .../com/google/api/gax/tracing/CompositeTracerTest.java | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 4e9b0ca7b6cb..bb959829728b 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -85,6 +85,13 @@ public Scope inScope() { throw e; } + if (childScopes.isEmpty()) { + return NOOP_SCOPE; + } + if (childScopes.size() == 1) { + return childScopes.get(0); + } + // Scope is thread-bound and must be closed on the same thread that opened it. return new Scope() { private boolean closed; diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java index 54010fdb3128..a4e6a2d10780 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java @@ -360,6 +360,11 @@ void testInScope_nullChildScopeIgnored() { } verify(scope2).close(); + + when(child2.inScope()).thenReturn(null); + try (ApiTracer.Scope ignored = compositeTracer.inScope()) { + // all-null childScopes returns no-op scope cleanly + } } @Test From 7338bf47c792df8988d2db9dbfddc9c567a9bdce Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 17:06:52 -0500 Subject: [PATCH 09/11] fix(gax): simplify CompositeTracer.inScope() to minimal null-guard implementation --- .../api/gax/tracing/CompositeTracer.java | 64 +++++------------ .../api/gax/tracing/CompositeTracerTest.java | 69 ------------------- 2 files changed, 17 insertions(+), 116 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index bb959829728b..a077dda8960e 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -45,26 +45,16 @@ @NullMarked @InternalApi class CompositeTracer extends BaseApiTracer { - private static final Scope NOOP_SCOPE = () -> {}; - private final List children; public CompositeTracer(List children) { this.children = ImmutableList.copyOf(children); } - // Unstubbed Mockito mocks of ApiTracer may return null despite @NullMarked, and Error is caught - // alongside RuntimeException solely to close already-opened scopes before rethrowing. - @SuppressWarnings({"java:S2589", "java:S1181"}) + // Unstubbed Mockito mocks of ApiTracer may return null despite @NullMarked. + @SuppressWarnings("java:S2589") @Override public Scope inScope() { - if (children.isEmpty()) { - return NOOP_SCOPE; - } - if (children.size() == 1) { - Scope scope = children.get(0).inScope(); - return scope != null ? scope : NOOP_SCOPE; - } final List childScopes = new ArrayList<>(children.size()); try { @@ -74,52 +64,32 @@ public Scope inScope() { childScopes.add(childScope); } } - } catch (RuntimeException | Error e) { + } catch (RuntimeException e) { for (int i = childScopes.size() - 1; i >= 0; i--) { try { childScopes.get(i).close(); - } catch (RuntimeException | Error suppressed) { + } catch (RuntimeException suppressed) { e.addSuppressed(suppressed); } } throw e; } - if (childScopes.isEmpty()) { - return NOOP_SCOPE; - } - if (childScopes.size() == 1) { - return childScopes.get(0); - } - - // Scope is thread-bound and must be closed on the same thread that opened it. - return new Scope() { - private boolean closed; - - @Override - public void close() { - if (closed) { - return; - } - closed = true; - Throwable primaryError = null; - for (int i = childScopes.size() - 1; i >= 0; i--) { - try { - childScopes.get(i).close(); - } catch (RuntimeException | Error e) { - if (primaryError == null) { - primaryError = e; - } else { - primaryError.addSuppressed(e); - } + return () -> { + RuntimeException exception = null; + for (int i = childScopes.size() - 1; i >= 0; i--) { + try { + childScopes.get(i).close(); + } catch (RuntimeException e) { + if (exception == null) { + exception = e; + } else { + exception.addSuppressed(e); } } - if (primaryError instanceof RuntimeException) { - throw (RuntimeException) primaryError; - } - if (primaryError instanceof Error) { - throw (Error) primaryError; - } + } + if (exception != null) { + throw exception; } }; } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java index a4e6a2d10780..681204005066 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java @@ -46,7 +46,6 @@ import io.opentelemetry.api.trace.TraceState; import java.time.Duration; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.CopyOnWriteArrayList; @@ -84,7 +83,6 @@ void testInScope_lifoOrder() { ApiTracer.Scope compositeScope = compositeTracer.inScope(); compositeScope.close(); - compositeScope.close(); verify(child1).inScope(); verify(child2).inScope(); @@ -92,7 +90,6 @@ void testInScope_lifoOrder() { InOrder inOrder = inOrder(scope2, scope1); inOrder.verify(scope2).close(); inOrder.verify(scope1).close(); - inOrder.verifyNoMoreInteractions(); } @Test @@ -140,44 +137,6 @@ void testInScope_childScopeCloseThrows() { inOrder.verify(scope1).close(); } - @Test - void testInScope_childInScopeThrowsError() { - ApiTracer.Scope scope1 = - mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); - AssertionError error = new AssertionError("Creation Error"); - RuntimeException cleanupFailure = new RuntimeException("Cleanup failure"); - - when(child1.inScope()).thenReturn(scope1); - when(child2.inScope()).thenThrow(error); - doThrow(cleanupFailure).when(scope1).close(); - - AssertionError thrown = assertThrows(AssertionError.class, () -> compositeTracer.inScope()); - - assertEquals(error, thrown); - assertThat(Arrays.asList(thrown.getSuppressed())).contains(cleanupFailure); - verify(scope1).close(); - } - - @Test - void testInScope_childScopeCloseThrowsError() { - ApiTracer.Scope scope1 = - mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); - ApiTracer.Scope scope2 = - mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); - AssertionError error2 = new AssertionError("Scope 2 close Error"); - - when(child1.inScope()).thenReturn(scope1); - when(child2.inScope()).thenReturn(scope2); - doThrow(error2).when(scope2).close(); - - ApiTracer.Scope compositeScope = compositeTracer.inScope(); - - AssertionError thrown = assertThrows(AssertionError.class, compositeScope::close); - assertEquals(error2, thrown); - verify(scope2).close(); - verify(scope1).close(); - } - @Test void testOperationSucceeded() { compositeTracer.operationSucceeded(); @@ -360,34 +319,6 @@ void testInScope_nullChildScopeIgnored() { } verify(scope2).close(); - - when(child2.inScope()).thenReturn(null); - try (ApiTracer.Scope ignored = compositeTracer.inScope()) { - // all-null childScopes returns no-op scope cleanly - } - } - - @Test - void testInScope_emptyAndSingleChildFastPath() { - CompositeTracer emptyTracer = new CompositeTracer(Collections.emptyList()); - try (ApiTracer.Scope ignored = emptyTracer.inScope()) { - // no-op scope closes cleanly - } - - ApiTracer.Scope singleScope = - mock(ApiTracer.Scope.class, Mockito.withSettings().withoutAnnotations()); - when(child1.inScope()).thenReturn(singleScope); - CompositeTracer singleChildTracer = new CompositeTracer(Collections.singletonList(child1)); - try (ApiTracer.Scope scope = singleChildTracer.inScope()) { - assertThat(scope).isSameInstanceAs(singleScope); - } - verify(singleScope).close(); - - when(child2.inScope()).thenReturn(null); - CompositeTracer singleNullScopeTracer = new CompositeTracer(Collections.singletonList(child2)); - try (ApiTracer.Scope ignored = singleNullScopeTracer.inScope()) { - verify(child2).inScope(); - } } @Test From daacf14e1cab2b3e41c426fa2dd18d49cdeef63a Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 18:06:09 -0500 Subject: [PATCH 10/11] chore(gax): add explanatory comments to CompositeTracer callback scoping --- .../com/google/api/gax/tracing/CompositeTracer.java | 10 ++++++++++ .../google/api/gax/tracing/CompositeTracerTest.java | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index a077dda8960e..5d5faddc10f7 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -40,6 +40,12 @@ * A composite implementation of {@link ApiTracer} that delegates all tracing events to a list of * underlying tracers. * + *

Completion and failure callbacks execute asynchronously on background worker threads where no + * trace span is active on thread-local storage. Wrapping those callbacks in {@code try (Scope + * ignored = inScope())} briefly activates the current span on the callback thread before any child + * tracer ends it, allowing sibling tracers (such as logging and metrics) to correlate error logs + * and metric exemplars with the active span before the scope closes on the same thread. + * *

For internal use only. */ @NullMarked @@ -94,6 +100,8 @@ public Scope inScope() { }; } + // Activate the operation span on the callback thread before any child ends it so sibling + // tracers (e.g., metrics exemplars and operation error logs) observe the active span. @Override public void operationSucceeded() { try (Scope ignored = inScope()) { @@ -143,6 +151,8 @@ public void attemptStarted(Object request, int attemptNumber) { } } + // Activate the attempt span on the callback thread before any child ends it so sibling + // tracers (e.g., attempt debug error logs) observe the active span. @Override public void attemptSucceeded() { try (Scope ignored = inScope()) { diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java index 681204005066..01143ee7747e 100644 --- a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerTest.java @@ -333,6 +333,8 @@ void testCompletionMethods_enterAndCloseScopeAroundChildren() { Throwable error = new RuntimeException("test error"); compositeTracer.operationFailed(error); + // Scopes open in forward order, callbacks dispatch in reverse order, and scopes close in LIFO + // order. InOrder inOrder = inOrder(child1, child2, scope1, scope2); inOrder.verify(child1).inScope(); inOrder.verify(child2).inScope(); @@ -404,6 +406,8 @@ public void attemptStarted(Object request, int attemptNumber) { @Override public void attemptFailedDuration(Throwable error, Duration delay) { + // Simulates OpenTelemetryTracingTracer ending attemptSpan and falling back to + // operationSpan. activeSpanRef.set(operationSpan); } @@ -420,9 +424,11 @@ public void operationFailed(Throwable error) { outerLoggingTracer, new CompositeTracer(Arrays.asList(innerMetricsTracer, innerTracingTracer)))); + // Start the attempt on the caller thread; no long-lived scope should remain open. nestedTracer.attemptStarted(new Object(), 0); assertThat(Span.current().getSpanContext().isValid()).isFalse(); + // Dispatch failure callbacks on a separate worker thread, matching async GAX RPC completion. ExecutorService executor = Executors.newSingleThreadExecutor(); try { executor @@ -440,6 +446,8 @@ public void operationFailed(Throwable error) { executor.shutdownNow(); } + // Attempt failure logged under attemptSpan; operation failure logged/recorded under + // operationSpan. assertThat(outerLoggingContexts).hasSize(2); assertThat(outerLoggingContexts.get(0)).isEqualTo(attemptSpan.getSpanContext()); assertThat(outerLoggingContexts.get(1)).isEqualTo(operationSpan.getSpanContext()); From 2d0dc1d77b9c19543dca7f9980bc1054d86eff48 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Mon, 16 Nov 2026 18:50:59 -0500 Subject: [PATCH 11/11] chore(gax): add Javadoc comments to all modified CompositeTracer methods --- .../api/gax/tracing/CompositeTracer.java | 46 +++++++++++++++++-- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 5d5faddc10f7..41eed5c078c7 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -57,7 +57,11 @@ public CompositeTracer(List children) { this.children = ImmutableList.copyOf(children); } - // Unstubbed Mockito mocks of ApiTracer may return null despite @NullMarked. + /** + * Opens a scope on each child tracer in forward order and returns a composite scope that closes + * them in reverse (LIFO) order. Null scopes (for example, from unstubbed {@link ApiTracer} mocks + * in tests) are ignored. + */ @SuppressWarnings("java:S2589") @Override public Scope inScope() { @@ -100,8 +104,10 @@ public Scope inScope() { }; } - // Activate the operation span on the callback thread before any child ends it so sibling - // tracers (e.g., metrics exemplars and operation error logs) observe the active span. + /** + * Activates the operation span on the callback thread before any child ends it so sibling tracers + * (such as metrics exemplars) observe the active span during completion. + */ @Override public void operationSucceeded() { try (Scope ignored = inScope()) { @@ -111,6 +117,10 @@ public void operationSucceeded() { } } + /** + * Activates the operation span on the callback thread before any child ends it so sibling tracers + * (such as metrics exemplars and cancellation logs) observe the active span. + */ @Override public void operationCancelled() { try (Scope ignored = inScope()) { @@ -120,6 +130,10 @@ public void operationCancelled() { } } + /** + * Activates the operation span on the callback thread before any child ends it so sibling tracers + * (such as metrics exemplars and ERROR-level logs) observe the active span. + */ @Override public void operationFailed(Throwable error) { try (Scope ignored = inScope()) { @@ -151,8 +165,10 @@ public void attemptStarted(Object request, int attemptNumber) { } } - // Activate the attempt span on the callback thread before any child ends it so sibling - // tracers (e.g., attempt debug error logs) observe the active span. + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * observe the active attempt span during completion. + */ @Override public void attemptSucceeded() { try (Scope ignored = inScope()) { @@ -162,6 +178,10 @@ public void attemptSucceeded() { } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * observe the active attempt span on cancellation. + */ @Override public void attemptCancelled() { try (Scope ignored = inScope()) { @@ -171,6 +191,10 @@ public void attemptCancelled() { } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * (such as DEBUG-level retry logs) observe the active attempt span. + */ @Override public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) { try (Scope ignored = inScope()) { @@ -180,6 +204,10 @@ public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) { } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * (such as DEBUG-level retry logs) observe the active attempt span. + */ @Override public void attemptFailedDuration(Throwable error, java.time.Duration delay) { try (Scope ignored = inScope()) { @@ -189,6 +217,10 @@ public void attemptFailedDuration(Throwable error, java.time.Duration delay) { } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * (such as DEBUG-level exhausted-retry logs) observe the active attempt span. + */ @Override public void attemptFailedRetriesExhausted(Throwable error) { try (Scope ignored = inScope()) { @@ -198,6 +230,10 @@ public void attemptFailedRetriesExhausted(Throwable error) { } } + /** + * Activates the attempt span on the callback thread before any child ends it so sibling tracers + * (such as DEBUG-level permanent-failure logs) observe the active attempt span. + */ @Override public void attemptPermanentFailure(Throwable error) { try (Scope ignored = inScope()) {