Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@
import com.google.api.core.ApiFuture;
import com.google.api.core.ApiFutures;
import com.google.api.gax.grpc.GrpcStatusCode;
import com.google.api.gax.retrying.ResultRetryAlgorithm;
import com.google.api.gax.retrying.ResultRetryAlgorithmWithContext;
import com.google.api.gax.retrying.RetryAlgorithm;
import com.google.api.gax.retrying.RetryingFuture;
import com.google.api.gax.retrying.TimedAttemptSettings;
import com.google.api.gax.rpc.ApiCallContext;
import com.google.api.gax.rpc.ApiException;
import com.google.api.gax.rpc.ApiExceptionFactory;
Expand Down Expand Up @@ -110,8 +111,7 @@ public Object getTransportCode() {
@Nullable private List<Integer> originalIndexes;
@Nonnull private final Set<StatusCode.Code> retryableCodes;
@Nonnull private final List<FailedMutation> permanentFailures;
@Nonnull private final RetryAlgorithm<MutateRowsRequest> retryAlgorithm;
@Nonnull private TimedAttemptSettings attemptSettings;
@Nonnull private final RetryAlgorithm<MutateRowsAttemptResult> retryAlgorithm;

// Parent controller
private RetryingFuture<MutateRowsAttemptResult> externalFuture;
Expand Down Expand Up @@ -140,14 +140,12 @@ public List<MutateRowsResponse> apply(Throwable throwable) {
@Nonnull MutateRowsRequest originalRequest,
@Nonnull ApiCallContext callContext,
@Nonnull Set<StatusCode.Code> retryableCodes,
@Nonnull RetryAlgorithm<MutateRowsRequest> retryAlgorithm) {
@Nonnull RetryAlgorithm<MutateRowsAttemptResult> retryAlgorithm) {
this.innerCallable = Preconditions.checkNotNull(innerCallable, "innerCallable");
this.currentRequest = Preconditions.checkNotNull(originalRequest, "currentRequest");
this.callContext = Preconditions.checkNotNull(callContext, "callContext");
this.retryableCodes = Preconditions.checkNotNull(retryableCodes, "retryableCodes");
this.retryAlgorithm = retryAlgorithm;
// TODO: pass in the callContext so that the retry setting can be overridden per call
this.attemptSettings = retryAlgorithm.createFirstAttempt(null);
this.retryAlgorithm = Preconditions.checkNotNull(retryAlgorithm, "retryAlgorithm");

permanentFailures = Lists.newArrayList();
}
Expand Down Expand Up @@ -235,15 +233,13 @@ private void handleAttemptError(Throwable rpcError) {
MutateRowsRequest.Builder builder = lastRequest.toBuilder().clearEntries();
List<Integer> newOriginalIndexes = Lists.newArrayList();

attemptSettings = retryAlgorithm.createNextAttempt(null, entryError, null, attemptSettings);

for (int i = 0; i < currentRequest.getEntriesCount(); i++) {
int origIndex = getOriginalIndex(i);

FailedMutation failedMutation = FailedMutation.create(origIndex, entryError);
allFailures.add(failedMutation);

if (!retryAlgorithm.shouldRetry(null, failedMutation.getError(), null, attemptSettings)) {
if (!shouldRetryBasedOnResult(failedMutation.getError())) {
permanentFailures.add(failedMutation);
} else {
// Schedule the mutation entry for the next RPC by adding it to the request builder and
Expand All @@ -260,6 +256,15 @@ private void handleAttemptError(Throwable rpcError) {
rpcError, entryError.getStatusCode(), allFailures.build(), builder.getEntriesCount() > 0);
}

private boolean shouldRetryBasedOnResult(Throwable throwable) {
ResultRetryAlgorithm<?> resultAlgorithm = retryAlgorithm.getResultAlgorithm();
if (resultAlgorithm instanceof ResultRetryAlgorithmWithContext) {
return ((ResultRetryAlgorithmWithContext<?>) resultAlgorithm)
.shouldRetry(callContext, throwable, null);
}
return resultAlgorithm.shouldRetry(throwable, null);
}

/**
* Handle entry level failures. All new response entries are inspected for failure. If any
* transient failures are found, their corresponding mutations are scheduled for the next RPC. The
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,18 @@
import com.google.api.core.AbstractApiFuture;
import com.google.api.core.ApiFuture;
import com.google.api.core.ApiFutures;
import com.google.api.core.NanoClock;
import com.google.api.gax.grpc.GrpcCallContext;
import com.google.api.gax.grpc.GrpcStatusCode;
import com.google.api.gax.retrying.ExponentialRetryAlgorithm;
import com.google.api.gax.retrying.RetryAlgorithm;
import com.google.api.gax.retrying.RetrySettings;
import com.google.api.gax.retrying.RetryingFuture;
import com.google.api.gax.retrying.ScheduledRetryingExecutor;
import com.google.api.gax.retrying.TimedAttemptSettings;
import com.google.api.gax.rpc.ApiCallContext;
import com.google.api.gax.rpc.ApiException;
import com.google.api.gax.rpc.ServerStreamingCallable;
import com.google.api.gax.rpc.StatusCode.Code;
import com.google.api.gax.rpc.UnaryCallable;
import com.google.api.gax.rpc.UnavailableException;
Expand All @@ -37,6 +41,7 @@
import com.google.bigtable.v2.MutateRowsResponse;
import com.google.cloud.bigtable.data.v2.models.MutateRowsException;
import com.google.cloud.bigtable.data.v2.models.MutateRowsException.FailedMutation;
import com.google.cloud.bigtable.gaxx.retrying.RetryInfoRetryAlgorithm;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.protobuf.ByteString;
Expand All @@ -45,6 +50,11 @@
import java.util.List;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
Expand All @@ -66,7 +76,7 @@ public class MutateRowsAttemptCallableTest {
private Set<Code> retryCodes;
private ApiCallContext callContext;
private MockRetryingFuture parentFuture;
private final RetryAlgorithm<MutateRowsRequest> mockRetryAlgorithm =
private final RetryAlgorithm<MutateRowsAttemptResult> mockRetryAlgorithm =
Mockito.mock(RetryAlgorithm.class, Mockito.withSettings().withoutAnnotations());

@Before
Expand All @@ -75,12 +85,13 @@ public void setUp() {
retryCodes = ImmutableSet.of(Code.DEADLINE_EXCEEDED, Code.UNAVAILABLE);
callContext = GrpcCallContext.createDefault();
parentFuture = new MockRetryingFuture();
Mockito.when(mockRetryAlgorithm.getResultAlgorithm())
.thenReturn(new MutateRowsPartialErrorRetryAlgorithm(new RetryInfoRetryAlgorithm<>()));
Mockito.when(mockRetryAlgorithm.shouldRetry(any(), any(), any(), any()))
.thenAnswer(
input -> {
Throwable throwable = input.getArgument(1);
return ((ApiException) throwable).isRetryable();
});
.thenAnswer(input -> {
Throwable throwable = input.getArgument(1);
return ((ApiException) throwable).isRetryable();
});
}

@Test
Expand Down Expand Up @@ -408,6 +419,144 @@ public ApiFuture<List<MutateRowsResponse>> futureCall(
assertThat(failedMutations.get(1).getError().getCause()).isEqualTo(rpcError);
}

@Test
public void partialSuccessThenRpcErrorTest() throws Exception {
MutateRowsRequest request =
MutateRowsRequest.newBuilder()
.addEntries(Entry.newBuilder().setRowKey(ByteString.copyFromUtf8("0-ok")))
.addEntries(Entry.newBuilder().setRowKey(ByteString.copyFromUtf8("1-unavailable")))
.addEntries(Entry.newBuilder().setRowKey(ByteString.copyFromUtf8("2-invalid")))
.build();

final UnavailableException rpcError =
new UnavailableException(
"fake error", null, GrpcStatusCode.of(io.grpc.Status.Code.UNAVAILABLE), true);
AtomicInteger attempt = new AtomicInteger();

UnaryCallable<MutateRowsRequest, List<MutateRowsResponse>> callable =
new UnaryCallable<MutateRowsRequest, List<MutateRowsResponse>>() {
@Override
public ApiFuture<List<MutateRowsResponse>> futureCall(
MutateRowsRequest req, ApiCallContext ctx) {
if (attempt.incrementAndGet() == 1) {
return ApiFutures.immediateFuture(
Lists.newArrayList(
MutateRowsResponse.newBuilder()
.addEntries(
MutateRowsResponse.Entry.newBuilder()
.setIndex(0)
.setStatus(OK_STATUS_PROTO))
.addEntries(
MutateRowsResponse.Entry.newBuilder()
.setIndex(1)
.setStatus(TRANSIENT_ERROR_STATUS_PROTO))
.addEntries(
MutateRowsResponse.Entry.newBuilder()
.setIndex(2)
.setStatus(PERMENANT_ERROR_STATUS_PROTO))
.build()));
}
return ApiFutures.immediateFailedFuture(rpcError);
}
};

RetryAlgorithm<MutateRowsAttemptResult> realRetryAlgorithm =
new RetryAlgorithm<>(
new MutateRowsPartialErrorRetryAlgorithm(new RetryInfoRetryAlgorithm<>()),
new ExponentialRetryAlgorithm(
RetrySettings.newBuilder().setMaxAttempts(2).build(), NanoClock.getDefaultClock()));

MutateRowsAttemptCallable attemptCallable =
new MutateRowsAttemptCallable(
callable, request, callContext, retryCodes, realRetryAlgorithm);
attemptCallable.setExternalFuture(parentFuture);

// Attempt 1: partial failure
attemptCallable.call();
MutateRowsAttemptResult attempt1Result = parentFuture.attemptFuture.get();
assertThat(attempt1Result.getIsRetryable()).isTrue();
assertThat(attempt1Result.getFailedMutations()).hasSize(2);

// Attempt 2: RPC-level failure
attemptCallable.call();
Throwable actualError = null;
try {
parentFuture.attemptFuture.get();
} catch (ExecutionException e) {
actualError = e.getCause();
}
assertThat(actualError).isInstanceOf(MutateRowsException.class);
MutateRowsException mutateRowsException = (MutateRowsException) actualError;
assertThat(mutateRowsException.isRetryable()).isTrue();
assertThat(mutateRowsException.getFailedMutations()).hasSize(2);
assertThat(mutateRowsException.getFailedMutations().get(0).getIndex()).isEqualTo(2);
assertThat(mutateRowsException.getFailedMutations().get(0).getError().isRetryable()).isFalse();
assertThat(mutateRowsException.getFailedMutations().get(1).getIndex()).isEqualTo(1);
assertThat(mutateRowsException.getFailedMutations().get(1).getError().isRetryable()).isTrue();
}

@Test
public void perCallRetrySettingsMaxAttemptsOverrideTest() throws Exception {
MutateRowsRequest request =
MutateRowsRequest.newBuilder().addEntries(Entry.getDefaultInstance()).build();
final UnavailableException rpcError =
new UnavailableException(
"fake error", null, GrpcStatusCode.of(io.grpc.Status.Code.UNAVAILABLE), true);
AtomicInteger attempts = new AtomicInteger();

ServerStreamingCallable<MutateRowsRequest, MutateRowsResponse> streamingCallable =
new ServerStreamingCallable<MutateRowsRequest, MutateRowsResponse>() {
@Override
public void call(
MutateRowsRequest req,
com.google.api.gax.rpc.ResponseObserver<MutateRowsResponse> responseObserver,
ApiCallContext ctx) {
attempts.incrementAndGet();
responseObserver.onError(rpcError);
}
};

RetrySettings defaultRetrySettings =
RetrySettings.newBuilder()
.setMaxAttempts(2)
.setInitialRetryDelayDuration(Duration.ofMillis(1))
.setMaxRetryDelayDuration(Duration.ofMillis(5))
.setRetryDelayMultiplier(1.0)
.setTotalTimeoutDuration(Duration.ofSeconds(5))
.build();
RetryAlgorithm<MutateRowsAttemptResult> retryAlgorithm =
new RetryAlgorithm<>(
new MutateRowsPartialErrorRetryAlgorithm(new RetryInfoRetryAlgorithm<>()),
new ExponentialRetryAlgorithm(defaultRetrySettings, NanoClock.getDefaultClock()));

ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor();
try {
ScheduledRetryingExecutor<MutateRowsAttemptResult> retryingExecutor =
new ScheduledRetryingExecutor<>(retryAlgorithm, executorService);
MutateRowsRetryingCallable retryingCallable =
new MutateRowsRetryingCallable(
callContext.withRetrySettings(defaultRetrySettings),
streamingCallable,
retryingExecutor,
retryCodes,
retryAlgorithm);

ApiCallContext perCallContext =
callContext.withRetrySettings(defaultRetrySettings.toBuilder().setMaxAttempts(4).build());

Throwable actualError = null;
try {
retryingCallable.futureCall(request, perCallContext).get(10, TimeUnit.SECONDS);
} catch (ExecutionException t) {
actualError = t.getCause();
}
Comment on lines +547 to +552

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

Use a timeout on the .get() call to prevent the test from hanging indefinitely if the asynchronous execution fails or deadlocks. Additionally, catch ExecutionException specifically so that other exceptions (like TimeoutException) can propagate and fail the test clearly.

Suggested change
Throwable actualError = null;
try {
retryingCallable.futureCall(request, perCallContext).get();
} catch (Throwable t) {
actualError = t.getCause();
}
Throwable actualError = null;
try {
retryingCallable.futureCall(request, perCallContext).get(10, java.util.concurrent.TimeUnit.SECONDS);
} catch (ExecutionException t) {
actualError = t.getCause();
}
References
  1. Avoid using very short timeouts (such as 200ms) when waiting for asynchronous events or mock server requests in tests, as they can cause flakiness in busy CI/CD environments. Use a larger timeout (e.g., 2000ms) to ensure reliability.

assertThat(actualError).isInstanceOf(MutateRowsException.class);
assertThat(attempts.get()).isEqualTo(4);
} finally {
executorService.shutdownNow();
}
}

// <editor-fold desc="Helpers">
static class MockInnerCallable
extends UnaryCallable<MutateRowsRequest, List<MutateRowsResponse>> {
Expand Down
Loading