From f28113d93a9575423e038deb6ae2001ea873bc12 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 16 Nov 2026 22:34:01 +0000 Subject: [PATCH 1/2] fix(bigtable): remove shadow attemptSettings from MutateRowsAttemptCallable --- .../mutaterows/MutateRowsAttemptCallable.java | 23 ++- .../MutateRowsAttemptCallableTest.java | 160 +++++++++++++++++- 2 files changed, 168 insertions(+), 15 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java index 4796ad48a094..2b8273852c9f 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java @@ -19,9 +19,10 @@ import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.retrying.ResultRetryAlgorithm; +import com.google.api.gax.retrying.ResultRetryAlgorithmWithContext; import com.google.api.gax.retrying.RetryAlgorithm; import com.google.api.gax.retrying.RetryingFuture; -import com.google.api.gax.retrying.TimedAttemptSettings; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ApiException; import com.google.api.gax.rpc.ApiExceptionFactory; @@ -110,8 +111,7 @@ public Object getTransportCode() { @Nullable private List originalIndexes; @Nonnull private final Set retryableCodes; @Nonnull private final List permanentFailures; - @Nonnull private final RetryAlgorithm retryAlgorithm; - @Nonnull private TimedAttemptSettings attemptSettings; + @Nonnull private final RetryAlgorithm retryAlgorithm; // Parent controller private RetryingFuture externalFuture; @@ -140,14 +140,12 @@ public List apply(Throwable throwable) { @Nonnull MutateRowsRequest originalRequest, @Nonnull ApiCallContext callContext, @Nonnull Set retryableCodes, - @Nonnull RetryAlgorithm retryAlgorithm) { + @Nonnull RetryAlgorithm retryAlgorithm) { this.innerCallable = Preconditions.checkNotNull(innerCallable, "innerCallable"); this.currentRequest = Preconditions.checkNotNull(originalRequest, "currentRequest"); this.callContext = Preconditions.checkNotNull(callContext, "callContext"); this.retryableCodes = Preconditions.checkNotNull(retryableCodes, "retryableCodes"); this.retryAlgorithm = retryAlgorithm; - // TODO: pass in the callContext so that the retry setting can be overridden per call - this.attemptSettings = retryAlgorithm.createFirstAttempt(null); permanentFailures = Lists.newArrayList(); } @@ -235,15 +233,13 @@ private void handleAttemptError(Throwable rpcError) { MutateRowsRequest.Builder builder = lastRequest.toBuilder().clearEntries(); List newOriginalIndexes = Lists.newArrayList(); - attemptSettings = retryAlgorithm.createNextAttempt(null, entryError, null, attemptSettings); - for (int i = 0; i < currentRequest.getEntriesCount(); i++) { int origIndex = getOriginalIndex(i); FailedMutation failedMutation = FailedMutation.create(origIndex, entryError); allFailures.add(failedMutation); - if (!retryAlgorithm.shouldRetry(null, failedMutation.getError(), null, attemptSettings)) { + if (!shouldRetryBasedOnResult(failedMutation.getError())) { permanentFailures.add(failedMutation); } else { // Schedule the mutation entry for the next RPC by adding it to the request builder and @@ -260,6 +256,15 @@ private void handleAttemptError(Throwable rpcError) { rpcError, entryError.getStatusCode(), allFailures.build(), builder.getEntriesCount() > 0); } + private boolean shouldRetryBasedOnResult(Throwable throwable) { + ResultRetryAlgorithm resultAlgorithm = retryAlgorithm.getResultAlgorithm(); + if (resultAlgorithm instanceof ResultRetryAlgorithmWithContext) { + return ((ResultRetryAlgorithmWithContext) resultAlgorithm) + .shouldRetry(callContext, throwable, null); + } + return resultAlgorithm.shouldRetry(throwable, null); + } + /** * Handle entry level failures. All new response entries are inspected for failure. If any * transient failures are found, their corresponding mutations are scheduled for the next RPC. The diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java index 65f4f8a324f2..ed803da420c0 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java @@ -21,14 +21,18 @@ import com.google.api.core.AbstractApiFuture; import com.google.api.core.ApiFuture; import com.google.api.core.ApiFutures; +import com.google.api.core.NanoClock; import com.google.api.gax.grpc.GrpcCallContext; import com.google.api.gax.grpc.GrpcStatusCode; +import com.google.api.gax.retrying.ExponentialRetryAlgorithm; import com.google.api.gax.retrying.RetryAlgorithm; import com.google.api.gax.retrying.RetrySettings; import com.google.api.gax.retrying.RetryingFuture; +import com.google.api.gax.retrying.ScheduledRetryingExecutor; import com.google.api.gax.retrying.TimedAttemptSettings; import com.google.api.gax.rpc.ApiCallContext; import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.ServerStreamingCallable; import com.google.api.gax.rpc.StatusCode.Code; import com.google.api.gax.rpc.UnaryCallable; import com.google.api.gax.rpc.UnavailableException; @@ -37,6 +41,7 @@ import com.google.bigtable.v2.MutateRowsResponse; import com.google.cloud.bigtable.data.v2.models.MutateRowsException; import com.google.cloud.bigtable.data.v2.models.MutateRowsException.FailedMutation; +import com.google.cloud.bigtable.gaxx.retrying.RetryInfoRetryAlgorithm; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import com.google.protobuf.ByteString; @@ -45,6 +50,10 @@ import java.util.List; import java.util.Set; import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -66,7 +75,7 @@ public class MutateRowsAttemptCallableTest { private Set retryCodes; private ApiCallContext callContext; private MockRetryingFuture parentFuture; - private final RetryAlgorithm mockRetryAlgorithm = + private final RetryAlgorithm mockRetryAlgorithm = Mockito.mock(RetryAlgorithm.class, Mockito.withSettings().withoutAnnotations()); @Before @@ -75,12 +84,13 @@ public void setUp() { retryCodes = ImmutableSet.of(Code.DEADLINE_EXCEEDED, Code.UNAVAILABLE); callContext = GrpcCallContext.createDefault(); parentFuture = new MockRetryingFuture(); + Mockito.when(mockRetryAlgorithm.getResultAlgorithm()) + .thenReturn(new MutateRowsPartialErrorRetryAlgorithm(new RetryInfoRetryAlgorithm<>())); Mockito.when(mockRetryAlgorithm.shouldRetry(any(), any(), any(), any())) - .thenAnswer( - input -> { - Throwable throwable = input.getArgument(1); - return ((ApiException) throwable).isRetryable(); - }); + .thenAnswer(input -> { + Throwable throwable = input.getArgument(1); + return ((ApiException) throwable).isRetryable(); + }); } @Test @@ -408,6 +418,144 @@ public ApiFuture> futureCall( assertThat(failedMutations.get(1).getError().getCause()).isEqualTo(rpcError); } + @Test + public void partialSuccessThenRpcErrorTest() throws Exception { + MutateRowsRequest request = + MutateRowsRequest.newBuilder() + .addEntries(Entry.newBuilder().setRowKey(ByteString.copyFromUtf8("0-ok"))) + .addEntries(Entry.newBuilder().setRowKey(ByteString.copyFromUtf8("1-unavailable"))) + .addEntries(Entry.newBuilder().setRowKey(ByteString.copyFromUtf8("2-invalid"))) + .build(); + + final UnavailableException rpcError = + new UnavailableException( + "fake error", null, GrpcStatusCode.of(io.grpc.Status.Code.UNAVAILABLE), true); + AtomicInteger attempt = new AtomicInteger(); + + UnaryCallable> callable = + new UnaryCallable>() { + @Override + public ApiFuture> futureCall( + MutateRowsRequest req, ApiCallContext ctx) { + if (attempt.incrementAndGet() == 1) { + return ApiFutures.immediateFuture( + Lists.newArrayList( + MutateRowsResponse.newBuilder() + .addEntries( + MutateRowsResponse.Entry.newBuilder() + .setIndex(0) + .setStatus(OK_STATUS_PROTO)) + .addEntries( + MutateRowsResponse.Entry.newBuilder() + .setIndex(1) + .setStatus(TRANSIENT_ERROR_STATUS_PROTO)) + .addEntries( + MutateRowsResponse.Entry.newBuilder() + .setIndex(2) + .setStatus(PERMENANT_ERROR_STATUS_PROTO)) + .build())); + } + return ApiFutures.immediateFailedFuture(rpcError); + } + }; + + RetryAlgorithm realRetryAlgorithm = + new RetryAlgorithm<>( + new MutateRowsPartialErrorRetryAlgorithm(new RetryInfoRetryAlgorithm<>()), + new ExponentialRetryAlgorithm( + RetrySettings.newBuilder().setMaxAttempts(2).build(), NanoClock.getDefaultClock())); + + MutateRowsAttemptCallable attemptCallable = + new MutateRowsAttemptCallable( + callable, request, callContext, retryCodes, realRetryAlgorithm); + attemptCallable.setExternalFuture(parentFuture); + + // Attempt 1: partial failure + attemptCallable.call(); + MutateRowsAttemptResult attempt1Result = parentFuture.attemptFuture.get(); + assertThat(attempt1Result.getIsRetryable()).isTrue(); + assertThat(attempt1Result.getFailedMutations()).hasSize(2); + + // Attempt 2: RPC-level failure + attemptCallable.call(); + Throwable actualError = null; + try { + parentFuture.attemptFuture.get(); + } catch (ExecutionException e) { + actualError = e.getCause(); + } + assertThat(actualError).isInstanceOf(MutateRowsException.class); + MutateRowsException mutateRowsException = (MutateRowsException) actualError; + assertThat(mutateRowsException.isRetryable()).isTrue(); + assertThat(mutateRowsException.getFailedMutations()).hasSize(2); + assertThat(mutateRowsException.getFailedMutations().get(0).getIndex()).isEqualTo(2); + assertThat(mutateRowsException.getFailedMutations().get(0).getError().isRetryable()).isFalse(); + assertThat(mutateRowsException.getFailedMutations().get(1).getIndex()).isEqualTo(1); + assertThat(mutateRowsException.getFailedMutations().get(1).getError().isRetryable()).isTrue(); + } + + @Test + public void perCallRetrySettingsMaxAttemptsOverrideTest() { + MutateRowsRequest request = + MutateRowsRequest.newBuilder().addEntries(Entry.getDefaultInstance()).build(); + final UnavailableException rpcError = + new UnavailableException( + "fake error", null, GrpcStatusCode.of(io.grpc.Status.Code.UNAVAILABLE), true); + AtomicInteger attempts = new AtomicInteger(); + + ServerStreamingCallable streamingCallable = + new ServerStreamingCallable() { + @Override + public void call( + MutateRowsRequest req, + com.google.api.gax.rpc.ResponseObserver responseObserver, + ApiCallContext ctx) { + attempts.incrementAndGet(); + responseObserver.onError(rpcError); + } + }; + + RetrySettings defaultRetrySettings = + RetrySettings.newBuilder() + .setMaxAttempts(2) + .setInitialRetryDelayDuration(Duration.ofMillis(1)) + .setMaxRetryDelayDuration(Duration.ofMillis(5)) + .setRetryDelayMultiplier(1.0) + .setTotalTimeoutDuration(Duration.ofSeconds(5)) + .build(); + RetryAlgorithm retryAlgorithm = + new RetryAlgorithm<>( + new MutateRowsPartialErrorRetryAlgorithm(new RetryInfoRetryAlgorithm<>()), + new ExponentialRetryAlgorithm(defaultRetrySettings, NanoClock.getDefaultClock())); + + ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); + try { + ScheduledRetryingExecutor retryingExecutor = + new ScheduledRetryingExecutor<>(retryAlgorithm, executorService); + MutateRowsRetryingCallable retryingCallable = + new MutateRowsRetryingCallable( + callContext.withRetrySettings(defaultRetrySettings), + streamingCallable, + retryingExecutor, + retryCodes, + retryAlgorithm); + + ApiCallContext perCallContext = + callContext.withRetrySettings(defaultRetrySettings.toBuilder().setMaxAttempts(4).build()); + + Throwable actualError = null; + try { + retryingCallable.futureCall(request, perCallContext).get(); + } catch (Throwable t) { + actualError = t.getCause(); + } + assertThat(actualError).isInstanceOf(MutateRowsException.class); + assertThat(attempts.get()).isEqualTo(4); + } finally { + executorService.shutdownNow(); + } + } + // static class MockInnerCallable extends UnaryCallable> { From 1520859be144dbfae2d697438f14ac6b48a960f5 Mon Sep 17 00:00:00 2001 From: Igor Bernstein Date: Mon, 16 Nov 2026 22:56:21 +0000 Subject: [PATCH 2/2] fix(bigtable): address review feedback on MutateRowsAttemptCallable --- .../data/v2/stub/mutaterows/MutateRowsAttemptCallable.java | 2 +- .../v2/stub/mutaterows/MutateRowsAttemptCallableTest.java | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java index 2b8273852c9f..5f17d83e8663 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallable.java @@ -145,7 +145,7 @@ public List apply(Throwable throwable) { this.currentRequest = Preconditions.checkNotNull(originalRequest, "currentRequest"); this.callContext = Preconditions.checkNotNull(callContext, "callContext"); this.retryableCodes = Preconditions.checkNotNull(retryableCodes, "retryableCodes"); - this.retryAlgorithm = retryAlgorithm; + this.retryAlgorithm = Preconditions.checkNotNull(retryAlgorithm, "retryAlgorithm"); permanentFailures = Lists.newArrayList(); } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java index ed803da420c0..a44c12c931cc 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/stub/mutaterows/MutateRowsAttemptCallableTest.java @@ -53,6 +53,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.junit.Before; import org.junit.Test; @@ -495,7 +496,7 @@ public ApiFuture> futureCall( } @Test - public void perCallRetrySettingsMaxAttemptsOverrideTest() { + public void perCallRetrySettingsMaxAttemptsOverrideTest() throws Exception { MutateRowsRequest request = MutateRowsRequest.newBuilder().addEntries(Entry.getDefaultInstance()).build(); final UnavailableException rpcError = @@ -545,8 +546,8 @@ public void call( Throwable actualError = null; try { - retryingCallable.futureCall(request, perCallContext).get(); - } catch (Throwable t) { + retryingCallable.futureCall(request, perCallContext).get(10, TimeUnit.SECONDS); + } catch (ExecutionException t) { actualError = t.getCause(); } assertThat(actualError).isInstanceOf(MutateRowsException.class);