Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,14 @@ private enum PoolState {
@GuardedBy("poolLock")
private int consecutiveFailures = 0;

@GuardedBy("poolLock")
@Nullable
private Status lastCloseStatus = null;

@GuardedBy("poolLock")
@Nullable
private Metadata lastCloseTrailers = null;

/**
* When the client fallback to a non-session AFE session creation will return unimplemented
* errors. In which case the requests should fallback to classic client instead of waiting for an
Expand Down Expand Up @@ -598,6 +606,8 @@ private void onSessionReady(SessionHandle handle, OpenSessionResponse ignored) {
new Object[] {handle.getSession().getLogName(), handle.getSession().getState()});

consecutiveFailures = 0;
lastCloseStatus = null;
lastCloseTrailers = null;
consecutiveUnimplementedFailures = 0;

if (poolState != PoolState.STARTED) {
Expand Down Expand Up @@ -729,13 +739,15 @@ private void onSessionClose(
// rather than session establishment failures and should not fail pending vRPCs.
if (prevState == SessionState.STARTING) {
consecutiveFailures++;
lastCloseStatus = status;
lastCloseTrailers = trailers;
Comment thread
mutianf marked this conversation as resolved.
Resolved
if (status.getCode() == Status.Code.UNIMPLEMENTED) {
consecutiveUnimplementedFailures++;
} else {
consecutiveUnimplementedFailures = 0;
}
// TODO: decide if max consecutive failures should be capped per client
if (consecutiveFailures >= getMaxConsecutiveFailures(configManager)) {
if (isFailingWithNoEstablishedSessions()) {
toBeClosed = popClosableRpcs();
}
}
Expand All @@ -757,14 +769,7 @@ private void onSessionClose(

if (!toBeClosed.isEmpty()) {
// vRPCs failing with consecutive failures should fail
VRpcResult result =
VRpcResult.createRejectedError(
Status.fromCode(status.getCode())
.withDescription(
String.format(
"Session failed with consecutive failures. Most recent server status: %s,"
+ " metadata: %s.",
status, trailers)));
VRpcResult result = createConsecutiveFailureResult(status, trailers);
for (PendingVRpc<?, ?> vrpc : toBeClosed) {
try {
vrpc.cancelWithResult(result);
Expand All @@ -775,6 +780,29 @@ private void onSessionClose(
}
}

@GuardedBy("poolLock")
private boolean hasEstablishedSession() {
return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0;
}

@GuardedBy("poolLock")
private boolean isFailingWithNoEstablishedSessions() {
return consecutiveFailures >= getMaxConsecutiveFailures(configManager)
&& lastCloseStatus != null
&& !hasEstablishedSession();
}

private static VRpcResult createConsecutiveFailureResult(
Status status, @Nullable Metadata trailers) {
Status errorStatus = status.isOk() ? Status.UNAVAILABLE : Status.fromCode(status.getCode());
return VRpcResult.createRejectedError(
errorStatus.withDescription(
String.format(
"Session failed with consecutive failures. Most recent server status: %s,"
+ " metadata: %s.",
status, trailers)));
}

@GuardedBy("poolLock")
private void tryDrainPendingRpcs() {
while (!pendingRpcs.isEmpty()) {
Expand Down Expand Up @@ -895,7 +923,7 @@ public boolean hasSession() {
return false;
}
try {
return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0;
return hasEstablishedSession();
} finally {
poolLock.unlock();
}
Expand Down Expand Up @@ -970,6 +998,10 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener<RespT> listener) {
});
return;
}
if (isFailingWithNoEstablishedSessions()) {
cancelWithResult(createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers));
return;
}
// Only arm the deadline monitor after we've committed to queueing; otherwise the
// fast-fail early return above would leak a timer that fires later and emits a phantom
// tracer.onAttemptFinish on the Active state's stale listener.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1448,6 +1448,112 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except
assertThat(sessionPool.hasSession()).isTrue();
}

@Test
@SuppressWarnings("GuardedBy")
void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception {
com.google.rpc.Status errorStatus =
com.google.rpc.Status.newBuilder()
.setCode(Code.INTERNAL_VALUE)
.setMessage("fake internal error")
.build();
OpenFakeSessionRequest request =
OpenFakeSessionRequest.newBuilder()
.setStreamError(
StreamError.newBuilder()
.setStatus(errorStatus)
.setDelay(Durations.fromMillis(0))
.build())
.build();
sessionPool.start(request, new Metadata());

// Wait until all initial SessionCreationBudget slots have been exhausted and closed on the
// client
ReentrantLock poolLock = extractPoolLock(sessionPool);
Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures");
consecutiveFailuresField.setAccessible(true);
long deadline = System.currentTimeMillis() + 5000;
while (System.currentTimeMillis() < deadline) {
poolLock.lock();
try {
if (sessionPool.sessions.getAllSessions().isEmpty()
&& consecutiveFailuresField.getInt(sessionPool) >= 10) {
break;
}
} finally {
poolLock.unlock();
}
Thread.sleep(10);
}

// A new call arriving during the 60s budget lockout must fail immediately instead of queueing
UnaryResponseFuture<SessionFakeScriptedResponse> f = new UnaryResponseFuture<>();
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> rpc =
sessionPool.newCall(FakeDescriptor.SCRIPTED);
rpc.start(
SessionFakeScriptedRequest.getDefaultInstance(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
f);

ExecutionException ex =
assertThrows(ExecutionException.class, () -> f.get(1, TimeUnit.SECONDS));
assertThat(ex.getCause())
.hasMessageThat()
.contains("INTERNAL: Session failed with consecutive failures.");
}

@Test
@SuppressWarnings("GuardedBy")
void consecutiveFailuresPreservesPendingRpcsWhilePoolHasInUseSessions() throws Exception {
sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata());
waitForOpenRequestCount(5, Duration.ofSeconds(5));
assertPoolServesVRpc(sessionPool, "initial started pool");

ReentrantLock poolLock = extractPoolLock(sessionPool);
Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures");
consecutiveFailuresField.setAccessible(true);

UnaryResponseFuture<SessionFakeScriptedResponse> checkedOutFuture = new UnaryResponseFuture<>();
UnaryResponseFuture<SessionFakeScriptedResponse> pendingFuture = new UnaryResponseFuture<>();
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> checkedOutRpc;
poolLock.lock();
try {
// Mark 4 of the 5 ready sessions closing, then check out the remaining 1 ready session via
// newCall() so readyCount == 0 and inUseCount == 1.
List<SessionHandle> handles = new ArrayList<>(sessionPool.sessions.getAllSessions());
for (int i = 1; i < handles.size(); i++) {
handles.get(i).onSessionClosing();
}
checkedOutRpc = sessionPool.newCall(FakeDescriptor.SCRIPTED);
assertThat(sessionPool.sessions.getStats().getReadyCount()).isEqualTo(0);
assertThat(sessionPool.sessions.getStats().getInUseCount()).isEqualTo(1);

// Simulate consecutive STARTING failures reaching threshold while inUseCount == 1.
Field lastCloseStatusField = SessionPoolImpl.class.getDeclaredField("lastCloseStatus");
lastCloseStatusField.setAccessible(true);
lastCloseStatusField.set(sessionPool, Status.INTERNAL.withDescription("fake internal error"));
consecutiveFailuresField.setInt(sessionPool, 10);

// A new PendingVRpc started while inUseCount > 0 must queue instead of failing fast.
VRpc<SessionFakeScriptedRequest, SessionFakeScriptedResponse> pendingRpc =
sessionPool.newCall(FakeDescriptor.SCRIPTED);
pendingRpc.start(
SessionFakeScriptedRequest.newBuilder().setTag(0).build(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
pendingFuture);
assertThat(pendingFuture.isDone()).isFalse();
} finally {
poolLock.unlock();
}

// Complete the checked-out RPC so its session returns to the pool and drains pendingRpc.
checkedOutRpc.start(
SessionFakeScriptedRequest.newBuilder().setTag(0).build(),
VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer),
checkedOutFuture);
assertThat(checkedOutFuture.get(5, TimeUnit.SECONDS)).isNotNull();
assertThat(pendingFuture.get(5, TimeUnit.SECONDS)).isNotNull();
}

private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException {
long deadline = System.currentTimeMillis() + timeout.toMillis();
while (System.currentTimeMillis() < deadline) {
Expand Down
Loading