diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index c1abfb90232c..b1d1e8ede269 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -144,6 +144,14 @@ private enum PoolState { @GuardedBy("poolLock") private int consecutiveFailures = 0; + @GuardedBy("poolLock") + @Nullable + private Status lastCloseStatus = null; + + @GuardedBy("poolLock") + @Nullable + private Metadata lastCloseTrailers = null; + /** * When the client fallback to a non-session AFE session creation will return unimplemented * errors. In which case the requests should fallback to classic client instead of waiting for an @@ -598,6 +606,8 @@ private void onSessionReady(SessionHandle handle, OpenSessionResponse ignored) { new Object[] {handle.getSession().getLogName(), handle.getSession().getState()}); consecutiveFailures = 0; + lastCloseStatus = null; + lastCloseTrailers = null; consecutiveUnimplementedFailures = 0; if (poolState != PoolState.STARTED) { @@ -729,13 +739,15 @@ private void onSessionClose( // rather than session establishment failures and should not fail pending vRPCs. if (prevState == SessionState.STARTING) { consecutiveFailures++; + lastCloseStatus = status; + lastCloseTrailers = trailers; if (status.getCode() == Status.Code.UNIMPLEMENTED) { consecutiveUnimplementedFailures++; } else { consecutiveUnimplementedFailures = 0; } // TODO: decide if max consecutive failures should be capped per client - if (consecutiveFailures >= getMaxConsecutiveFailures(configManager)) { + if (isFailingWithNoEstablishedSessions()) { toBeClosed = popClosableRpcs(); } } @@ -757,14 +769,7 @@ private void onSessionClose( if (!toBeClosed.isEmpty()) { // vRPCs failing with consecutive failures should fail - VRpcResult result = - VRpcResult.createRejectedError( - Status.fromCode(status.getCode()) - .withDescription( - String.format( - "Session failed with consecutive failures. Most recent server status: %s," - + " metadata: %s.", - status, trailers))); + VRpcResult result = createConsecutiveFailureResult(status, trailers); for (PendingVRpc vrpc : toBeClosed) { try { vrpc.cancelWithResult(result); @@ -775,6 +780,29 @@ private void onSessionClose( } } + @GuardedBy("poolLock") + private boolean hasEstablishedSession() { + return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0; + } + + @GuardedBy("poolLock") + private boolean isFailingWithNoEstablishedSessions() { + return consecutiveFailures >= getMaxConsecutiveFailures(configManager) + && lastCloseStatus != null + && !hasEstablishedSession(); + } + + private static VRpcResult createConsecutiveFailureResult( + Status status, @Nullable Metadata trailers) { + Status errorStatus = status.isOk() ? Status.UNAVAILABLE : Status.fromCode(status.getCode()); + return VRpcResult.createRejectedError( + errorStatus.withDescription( + String.format( + "Session failed with consecutive failures. Most recent server status: %s," + + " metadata: %s.", + status, trailers))); + } + @GuardedBy("poolLock") private void tryDrainPendingRpcs() { while (!pendingRpcs.isEmpty()) { @@ -895,7 +923,7 @@ public boolean hasSession() { return false; } try { - return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0; + return hasEstablishedSession(); } finally { poolLock.unlock(); } @@ -970,6 +998,10 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { }); return; } + if (isFailingWithNoEstablishedSessions()) { + cancelWithResult(createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers)); + return; + } // Only arm the deadline monitor after we've committed to queueing; otherwise the // fast-fail early return above would leak a timer that fires later and emits a phantom // tracer.onAttemptFinish on the Active state's stale listener. diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index 55afab0b79dd..fc9a0affc14e 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -1448,6 +1448,112 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except assertThat(sessionPool.hasSession()).isTrue(); } + @Test + @SuppressWarnings("GuardedBy") + void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception { + com.google.rpc.Status errorStatus = + com.google.rpc.Status.newBuilder() + .setCode(Code.INTERNAL_VALUE) + .setMessage("fake internal error") + .build(); + OpenFakeSessionRequest request = + OpenFakeSessionRequest.newBuilder() + .setStreamError( + StreamError.newBuilder() + .setStatus(errorStatus) + .setDelay(Durations.fromMillis(0)) + .build()) + .build(); + sessionPool.start(request, new Metadata()); + + // Wait until all initial SessionCreationBudget slots have been exhausted and closed on the + // client + ReentrantLock poolLock = extractPoolLock(sessionPool); + Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures"); + consecutiveFailuresField.setAccessible(true); + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + poolLock.lock(); + try { + if (sessionPool.sessions.getAllSessions().isEmpty() + && consecutiveFailuresField.getInt(sessionPool) >= 10) { + break; + } + } finally { + poolLock.unlock(); + } + Thread.sleep(10); + } + + // A new call arriving during the 60s budget lockout must fail immediately instead of queueing + UnaryResponseFuture f = new UnaryResponseFuture<>(); + VRpc rpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + rpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + f); + + ExecutionException ex = + assertThrows(ExecutionException.class, () -> f.get(1, TimeUnit.SECONDS)); + assertThat(ex.getCause()) + .hasMessageThat() + .contains("INTERNAL: Session failed with consecutive failures."); + } + + @Test + @SuppressWarnings("GuardedBy") + void consecutiveFailuresPreservesPendingRpcsWhilePoolHasInUseSessions() throws Exception { + sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata()); + waitForOpenRequestCount(5, Duration.ofSeconds(5)); + assertPoolServesVRpc(sessionPool, "initial started pool"); + + ReentrantLock poolLock = extractPoolLock(sessionPool); + Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures"); + consecutiveFailuresField.setAccessible(true); + + UnaryResponseFuture checkedOutFuture = new UnaryResponseFuture<>(); + UnaryResponseFuture pendingFuture = new UnaryResponseFuture<>(); + VRpc checkedOutRpc; + poolLock.lock(); + try { + // Mark 4 of the 5 ready sessions closing, then check out the remaining 1 ready session via + // newCall() so readyCount == 0 and inUseCount == 1. + List handles = new ArrayList<>(sessionPool.sessions.getAllSessions()); + for (int i = 1; i < handles.size(); i++) { + handles.get(i).onSessionClosing(); + } + checkedOutRpc = sessionPool.newCall(FakeDescriptor.SCRIPTED); + assertThat(sessionPool.sessions.getStats().getReadyCount()).isEqualTo(0); + assertThat(sessionPool.sessions.getStats().getInUseCount()).isEqualTo(1); + + // Simulate consecutive STARTING failures reaching threshold while inUseCount == 1. + Field lastCloseStatusField = SessionPoolImpl.class.getDeclaredField("lastCloseStatus"); + lastCloseStatusField.setAccessible(true); + lastCloseStatusField.set(sessionPool, Status.INTERNAL.withDescription("fake internal error")); + consecutiveFailuresField.setInt(sessionPool, 10); + + // A new PendingVRpc started while inUseCount > 0 must queue instead of failing fast. + VRpc pendingRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + pendingRpc.start( + SessionFakeScriptedRequest.newBuilder().setTag(0).build(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + pendingFuture); + assertThat(pendingFuture.isDone()).isFalse(); + } finally { + poolLock.unlock(); + } + + // Complete the checked-out RPC so its session returns to the pool and drains pendingRpc. + checkedOutRpc.start( + SessionFakeScriptedRequest.newBuilder().setTag(0).build(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + checkedOutFuture); + assertThat(checkedOutFuture.get(5, TimeUnit.SECONDS)).isNotNull(); + assertThat(pendingFuture.get(5, TimeUnit.SECONDS)).isNotNull(); + } + private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException { long deadline = System.currentTimeMillis() + timeout.toMillis(); while (System.currentTimeMillis() < deadline) {