Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -345,26 +345,38 @@ private ApiException createEntryError(com.google.rpc.Status protoStatus) {
.withDescription(protoStatus.getMessage());

StatusCode gaxStatusCode = GrpcStatusCode.of(grpcStatus.getCode());
Set<StatusCode.Code> effectiveRetryableCodes =
callContext.getRetryableCodes() != null ? callContext.getRetryableCodes() : retryableCodes;

return ApiExceptionFactory.createException(
grpcStatus.asRuntimeException(),
gaxStatusCode,
retryableCodes.contains(gaxStatusCode.getCode()));
effectiveRetryableCodes.contains(gaxStatusCode.getCode()));
}

/**
* Create a synthetic {@link ApiException} for an individual entry. When the entire RPC fails, it
* implies that all entries failed as well. This helper is used to make that behavior explicit.
* The generated exception will have the overall error as its cause.
*/
private static ApiException createSyntheticErrorForRpcFailure(Throwable overallRequestError) {
private ApiException createSyntheticErrorForRpcFailure(Throwable overallRequestError) {
if (overallRequestError instanceof ApiException) {
ApiException requestApiException = (ApiException) overallRequestError;
// When no per-call retryableCodes override is present on callContext, preserve
// requestApiException.isRetryable() so both the stub's configured retryableCodes (applied by
// GrpcRawCallableFactory) and ConvertExceptionCallable's retryable flag on transient
// INTERNAL errors (e.g. RST_STREAM / GOAWAY) are honored.
boolean isRetryable =
callContext.getRetryableCodes() != null
? callContext
.getRetryableCodes()
.contains(requestApiException.getStatusCode().getCode())
: requestApiException.isRetryable();
Comment on lines +369 to +374

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

When callContext.getRetryableCodes() is null, createSyntheticErrorForRpcFailure falls back to requestApiException.isRetryable(). However, requestApiException.isRetryable() only reflects the hardcoded default of the exception class (e.g., false for AbortedException), ignoring any custom retryableCodes configured at the stub level (which are stored in this.retryableCodes).

This creates an inconsistency where entry-level errors (via createEntryError) correctly honor the stub's configured retryableCodes when no per-call override is present, but RPC-level errors (via createSyntheticErrorForRpcFailure) ignore them.

To fix this and ensure consistency, we should fall back to this.retryableCodes instead of requestApiException.isRetryable() when callContext.getRetryableCodes() is null.

      Set<StatusCode.Code> effectiveRetryableCodes =
          callContext.getRetryableCodes() != null ? callContext.getRetryableCodes() : retryableCodes;
      boolean isRetryable =
          effectiveRetryableCodes.contains(requestApiException.getStatusCode().getCode());


return ApiExceptionFactory.createException(
overallRequestError,
requestApiException.getStatusCode(),
requestApiException.isRetryable(),
isRetryable,
requestApiException.getErrorDetails());
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.google.api.gax.retrying.RetrySettings;
import com.google.api.gax.retrying.RetryingFuture;
import com.google.api.gax.retrying.TimedAttemptSettings;
import com.google.api.gax.rpc.AbortedException;
import com.google.api.gax.rpc.ApiCallContext;
import com.google.api.gax.rpc.ApiException;
import com.google.api.gax.rpc.StatusCode.Code;
Expand Down Expand Up @@ -76,11 +77,10 @@ public void setUp() {
callContext = GrpcCallContext.createDefault();
parentFuture = new MockRetryingFuture();
Mockito.when(mockRetryAlgorithm.shouldRetry(any(), any(), any(), any()))
.thenAnswer(
input -> {
Throwable throwable = input.getArgument(1);
return ((ApiException) throwable).isRetryable();
});
.thenAnswer(input -> {
Throwable throwable = input.getArgument(1);
return ((ApiException) throwable).isRetryable();
});
}

@Test
Expand Down Expand Up @@ -408,6 +408,119 @@ public ApiFuture<List<MutateRowsResponse>> futureCall(
assertThat(failedMutations.get(1).getError().getCause()).isEqualTo(rpcError);
}

@Test
public void perCallRetryableCodesEntryErrorTest() throws Exception {
MutateRowsRequest request =
MutateRowsRequest.newBuilder()
.addEntries(Entry.newBuilder().setRowKey(ByteString.copyFromUtf8("0-aborted")))
.addEntries(Entry.newBuilder().setRowKey(ByteString.copyFromUtf8("1-unavailable")))
.build();

Status abortedStatusProto =
Status.newBuilder().setCode(com.google.rpc.Code.ABORTED_VALUE).build();
innerCallable.response.add(
MutateRowsResponse.newBuilder()
.addEntries(
MutateRowsResponse.Entry.newBuilder().setIndex(0).setStatus(abortedStatusProto))
.addEntries(
MutateRowsResponse.Entry.newBuilder()
.setIndex(1)
.setStatus(TRANSIENT_ERROR_STATUS_PROTO))
.build());

ApiCallContext customCallContext =
callContext.withRetryableCodes(ImmutableSet.of(Code.ABORTED));
MutateRowsAttemptCallable attemptCallable =
new MutateRowsAttemptCallable(
innerCallable, request, customCallContext, retryCodes, mockRetryAlgorithm);
attemptCallable.setExternalFuture(parentFuture);
attemptCallable.call();

MutateRowsAttemptResult result = parentFuture.attemptFuture.get();
assertThat(result.getIsRetryable()).isTrue();
List<FailedMutation> failedMutations = result.getFailedMutations();
assertThat(failedMutations).hasSize(2);

assertThat(failedMutations.get(0).getIndex()).isEqualTo(0);
assertThat(failedMutations.get(0).getError().getStatusCode().getCode()).isEqualTo(Code.ABORTED);
assertThat(failedMutations.get(0).getError().isRetryable()).isTrue();

assertThat(failedMutations.get(1).getIndex()).isEqualTo(1);
assertThat(failedMutations.get(1).getError().getStatusCode().getCode())
.isEqualTo(Code.UNAVAILABLE);
assertThat(failedMutations.get(1).getError().isRetryable()).isFalse();
}

@Test
public void perCallRetryableCodesRpcErrorTest() {
MutateRowsRequest request =
MutateRowsRequest.newBuilder().addEntries(Entry.getDefaultInstance()).build();

// Stub default marks ABORTED as non-retryable and UNAVAILABLE as retryable, but per-call
// context overrides retryableCodes to {ABORTED}.
final AbortedException abortedError =
new AbortedException(
"aborted", null, GrpcStatusCode.of(io.grpc.Status.Code.ABORTED), false);
final UnavailableException unavailableError =
new UnavailableException(
"unavailable", null, GrpcStatusCode.of(io.grpc.Status.Code.UNAVAILABLE), true);

ApiCallContext customCallContext =
callContext.withRetryableCodes(ImmutableSet.of(Code.ABORTED));

UnaryCallable<MutateRowsRequest, List<MutateRowsResponse>> abortedCallable =
new UnaryCallable<MutateRowsRequest, List<MutateRowsResponse>>() {
@Override
public ApiFuture<List<MutateRowsResponse>> futureCall(
MutateRowsRequest req, ApiCallContext ctx) {
return ApiFutures.immediateFailedFuture(abortedError);
}
};

MutateRowsAttemptCallable attemptCallable1 =
new MutateRowsAttemptCallable(
abortedCallable, request, customCallContext, retryCodes, mockRetryAlgorithm);
attemptCallable1.setExternalFuture(parentFuture);
attemptCallable1.call();

Throwable actualError1 = null;
try {
parentFuture.attemptFuture.get();
} catch (Throwable t) {
actualError1 = t.getCause();
}
assertThat(actualError1).isInstanceOf(MutateRowsException.class);
MutateRowsException ex1 = (MutateRowsException) actualError1;
assertThat(ex1.isRetryable()).isTrue();
assertThat(ex1.getFailedMutations().get(0).getError().isRetryable()).isTrue();

UnaryCallable<MutateRowsRequest, List<MutateRowsResponse>> unavailableCallable =
new UnaryCallable<MutateRowsRequest, List<MutateRowsResponse>>() {
@Override
public ApiFuture<List<MutateRowsResponse>> futureCall(
MutateRowsRequest req, ApiCallContext ctx) {
return ApiFutures.immediateFailedFuture(unavailableError);
}
};

MutateRowsAttemptCallable attemptCallable2 =
new MutateRowsAttemptCallable(
unavailableCallable, request, customCallContext, retryCodes, mockRetryAlgorithm);
attemptCallable2.setExternalFuture(parentFuture);
attemptCallable2.call();

Throwable actualError2 = null;
try {
parentFuture.attemptFuture.get();
} catch (Throwable t) {
actualError2 = t.getCause();
}
assertThat(actualError2).isInstanceOf(MutateRowsException.class);
MutateRowsException ex2 = (MutateRowsException) actualError2;
assertThat(ex2.isRetryable()).isFalse();
assertThat(ex2.getFailedMutations().get(0).getError().isRetryable()).isFalse();
}

// <editor-fold desc="Helpers">
static class MockInnerCallable
extends UnaryCallable<MutateRowsRequest, List<MutateRowsResponse>> {
Expand Down
Loading