From b787011e6d624195037d90480452f675bd95947d Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 20:57:55 +0000 Subject: [PATCH 1/7] fix(bigtable): bound SessionPool pendingRpcs and fast-fail during consecutive failures # Conflicts: # java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java # Conflicts: # java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java --- .../data/v2/internal/session/PoolSizer.java | 8 ++++ .../v2/internal/session/SessionPoolImpl.java | 38 +++++++++++++++++++ .../v2/internal/session/PoolSizerTest.java | 7 ++++ .../internal/session/SessionPoolImplTest.java | 36 ++++++++++++++++++ 4 files changed, 89 insertions(+) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java index 3e9d166d9ae0..1a5ebd56b42c 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java @@ -100,6 +100,14 @@ public int getScaleDelta() { } } + /** Returns the maximum number of pending vRPCs allowed in the pool queue. */ + int getMaxPendingRpcs() { + if (isDisabled()) { + return 0; + } + return maxIdleSessions * Math.max(1, pendingVRpcsPerSession); + } + /** Returns true if the session should be replaced */ @CheckReturnValue public boolean handleGoAway(GoAwayResponse msg) { diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index c1abfb90232c..82c78051d9e6 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -144,6 +144,12 @@ private enum PoolState { @GuardedBy("poolLock") private int consecutiveFailures = 0; + @GuardedBy("poolLock") + private Status lastCloseStatus = Status.UNAVAILABLE; + + @GuardedBy("poolLock") + private Metadata lastCloseTrailers = new Metadata(); + /** * When the client fallback to a non-session AFE session creation will return unimplemented * errors. In which case the requests should fallback to classic client instead of waiting for an @@ -729,6 +735,8 @@ private void onSessionClose( // rather than session establishment failures and should not fail pending vRPCs. if (prevState == SessionState.STARTING) { consecutiveFailures++; + lastCloseStatus = status; + lastCloseTrailers = trailers; if (status.getCode() == Status.Code.UNIMPLEMENTED) { consecutiveUnimplementedFailures++; } else { @@ -970,6 +978,36 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { }); return; } + if (consecutiveFailures >= getMaxConsecutiveFailures(configManager) + && sessions.getStats().getReadyCount() == 0) { + if (poolSizer.handleNewCall()) { + createSession(openParams); + } + VRpcResult result = + VRpcResult.createRejectedError( + Status.fromCode(lastCloseStatus.getCode()) + .withDescription( + String.format( + "Session failed with consecutive failures. Most recent server status:" + + " %s, metadata: %s.", + lastCloseStatus, lastCloseTrailers))); + cancelWithResult(result); + return; + } + int maxPendingRpcs = poolSizer.getMaxPendingRpcs(); + if (maxPendingRpcs > 0 && pendingRpcs.size() >= maxPendingRpcs) { + if (poolSizer.handleNewCall()) { + createSession(openParams); + } + VRpcResult result = + VRpcResult.createRejectedError( + Status.RESOURCE_EXHAUSTED.withDescription( + String.format( + "SessionPool pending RPC queue is full (%d/%d)", + pendingRpcs.size(), maxPendingRpcs))); + cancelWithResult(result); + return; + } // Only arm the deadline monitor after we've committed to queueing; otherwise the // fast-fail early return above would leak a timer that fires later and emits a phantom // tracer.onAttemptFinish on the Active state's stale listener. diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java index a343c7973ce8..1a693d1c7bfd 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java @@ -249,4 +249,11 @@ void testEmptyConfig() { assertThat(sizer.handleGoAway(GoAwayResponse.getDefaultInstance())).isFalse(); assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isFalse(); } + + @Test + void testGetMaxPendingRpcs() { + PoolStats stats = new PoolStats(); + PoolSizer sizer = new PoolSizer(stats, () -> 0, DEFAULT_CONFIG); + assertThat(sizer.getMaxPendingRpcs()).isEqualTo(MAX_IDLE_SESSIONS * PENDING_RPCS_PER_SESSION); + } } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index 55afab0b79dd..f75c0d768b4e 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -1448,6 +1448,42 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except assertThat(sessionPool.hasSession()).isTrue(); } + @Test + void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception { + com.google.rpc.Status errorStatus = + com.google.rpc.Status.newBuilder() + .setCode(Code.INTERNAL_VALUE) + .setMessage("fake internal error") + .build(); + OpenFakeSessionRequest request = + OpenFakeSessionRequest.newBuilder() + .setStreamError( + StreamError.newBuilder() + .setStatus(errorStatus) + .setDelay(Durations.fromMillis(0)) + .build()) + .build(); + sessionPool.start(request, new Metadata()); + + // Wait until all 20 initial SessionCreationBudget slots have been exhausted and closed + waitForOpenRequestCount(20, Duration.ofSeconds(5)); + + // A new call arriving during the 60s budget lockout must fail immediately instead of queueing + UnaryResponseFuture f = new UnaryResponseFuture<>(); + VRpc rpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + rpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + f); + + ExecutionException ex = + assertThrows(ExecutionException.class, () -> f.get(1, TimeUnit.SECONDS)); + assertThat(ex.getCause()) + .hasMessageThat() + .contains("INTERNAL: Session failed with consecutive failures."); + } + private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException { long deadline = System.currentTimeMillis() + timeout.toMillis(); while (System.currentTimeMillis() < deadline) { From 3f8631ac1140f761c482953dfe9517d341f85342 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 21:01:22 +0000 Subject: [PATCH 2/7] fix(bigtable): fail pendingRpcs on consecutive failures only when pool has no established sessions --- .../data/v2/internal/session/PoolSizer.java | 8 -- .../v2/internal/session/SessionPoolImpl.java | 75 +++++++++---------- .../v2/internal/session/PoolSizerTest.java | 7 -- .../internal/session/SessionPoolImplTest.java | 50 +++++++++++++ 4 files changed, 87 insertions(+), 53 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java index 1a5ebd56b42c..3e9d166d9ae0 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java @@ -100,14 +100,6 @@ public int getScaleDelta() { } } - /** Returns the maximum number of pending vRPCs allowed in the pool queue. */ - int getMaxPendingRpcs() { - if (isDisabled()) { - return 0; - } - return maxIdleSessions * Math.max(1, pendingVRpcsPerSession); - } - /** Returns true if the session should be replaced */ @CheckReturnValue public boolean handleGoAway(GoAwayResponse msg) { diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index 82c78051d9e6..e1ae52630d2c 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -727,6 +727,8 @@ private void onSessionClose( return; } + Status failureStatus = status; + Metadata failureTrailers = trailers; // Handle abnormal close. This can happen if the Session was aborted due to underlying stream // termination if (prevState != SessionState.WAIT_SERVER_CLOSE) { @@ -742,10 +744,6 @@ private void onSessionClose( } else { consecutiveUnimplementedFailures = 0; } - // TODO: decide if max consecutive failures should be capped per client - if (consecutiveFailures >= getMaxConsecutiveFailures(configManager)) { - toBeClosed = popClosableRpcs(); - } } // Budget release for STARTING-phase closes is handled above via sessionsHoldingBudget, @@ -759,20 +757,23 @@ private void onSessionClose( createSession(openParams.withIncrementedAttempts()); } } + + // Only fail pending vRPCs when session creation is failing consecutively AND the pool has no + // established (READY or IN_USE) sessions left to drain the queue. Checking this outside the + // STARTING branch also drains pendingRpcs if the last established session closes (e.g. on + // GOAWAY or keepalive timeout) while consecutiveFailures is already above the threshold. + if (isFailingWithNoEstablishedSessions()) { + failureStatus = lastCloseStatus; + failureTrailers = lastCloseTrailers; + toBeClosed = popClosableRpcs(); + } } finally { poolLock.unlock(); } if (!toBeClosed.isEmpty()) { // vRPCs failing with consecutive failures should fail - VRpcResult result = - VRpcResult.createRejectedError( - Status.fromCode(status.getCode()) - .withDescription( - String.format( - "Session failed with consecutive failures. Most recent server status: %s," - + " metadata: %s.", - status, trailers))); + VRpcResult result = createConsecutiveFailureResult(failureStatus, failureTrailers); for (PendingVRpc vrpc : toBeClosed) { try { vrpc.cancelWithResult(result); @@ -783,6 +784,27 @@ private void onSessionClose( } } + @GuardedBy("poolLock") + private boolean hasEstablishedSession() { + return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0; + } + + @GuardedBy("poolLock") + private boolean isFailingWithNoEstablishedSessions() { + return consecutiveFailures >= getMaxConsecutiveFailures(configManager) + && !hasEstablishedSession(); + } + + private static VRpcResult createConsecutiveFailureResult(Status status, Metadata trailers) { + return VRpcResult.createRejectedError( + Status.fromCode(status.getCode()) + .withDescription( + String.format( + "Session failed with consecutive failures. Most recent server status: %s," + + " metadata: %s.", + status, trailers))); + } + @GuardedBy("poolLock") private void tryDrainPendingRpcs() { while (!pendingRpcs.isEmpty()) { @@ -903,7 +925,7 @@ public boolean hasSession() { return false; } try { - return sessions.getStats().getReadyCount() + sessions.getStats().getInUseCount() > 0; + return hasEstablishedSession(); } finally { poolLock.unlock(); } @@ -978,34 +1000,11 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { }); return; } - if (consecutiveFailures >= getMaxConsecutiveFailures(configManager) - && sessions.getStats().getReadyCount() == 0) { - if (poolSizer.handleNewCall()) { - createSession(openParams); - } - VRpcResult result = - VRpcResult.createRejectedError( - Status.fromCode(lastCloseStatus.getCode()) - .withDescription( - String.format( - "Session failed with consecutive failures. Most recent server status:" - + " %s, metadata: %s.", - lastCloseStatus, lastCloseTrailers))); - cancelWithResult(result); - return; - } - int maxPendingRpcs = poolSizer.getMaxPendingRpcs(); - if (maxPendingRpcs > 0 && pendingRpcs.size() >= maxPendingRpcs) { + if (isFailingWithNoEstablishedSessions()) { if (poolSizer.handleNewCall()) { createSession(openParams); } - VRpcResult result = - VRpcResult.createRejectedError( - Status.RESOURCE_EXHAUSTED.withDescription( - String.format( - "SessionPool pending RPC queue is full (%d/%d)", - pendingRpcs.size(), maxPendingRpcs))); - cancelWithResult(result); + cancelWithResult(createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers)); return; } // Only arm the deadline monitor after we've committed to queueing; otherwise the diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java index 1a693d1c7bfd..a343c7973ce8 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java @@ -249,11 +249,4 @@ void testEmptyConfig() { assertThat(sizer.handleGoAway(GoAwayResponse.getDefaultInstance())).isFalse(); assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isFalse(); } - - @Test - void testGetMaxPendingRpcs() { - PoolStats stats = new PoolStats(); - PoolSizer sizer = new PoolSizer(stats, () -> 0, DEFAULT_CONFIG); - assertThat(sizer.getMaxPendingRpcs()).isEqualTo(MAX_IDLE_SESSIONS * PENDING_RPCS_PER_SESSION); - } } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index f75c0d768b4e..fcd5f63439ee 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -1484,6 +1484,56 @@ void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception { .contains("INTERNAL: Session failed with consecutive failures."); } + @Test + @SuppressWarnings("GuardedBy") + void consecutiveFailuresPreservesPendingRpcsWhilePoolHasInUseSessions() throws Exception { + sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata()); + waitForOpenRequestCount(5, Duration.ofSeconds(5)); + assertPoolServesVRpc(sessionPool, "initial started pool"); + + ReentrantLock poolLock = extractPoolLock(sessionPool); + Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures"); + consecutiveFailuresField.setAccessible(true); + + UnaryResponseFuture checkedOutFuture = new UnaryResponseFuture<>(); + UnaryResponseFuture pendingFuture = new UnaryResponseFuture<>(); + VRpc checkedOutRpc; + poolLock.lock(); + try { + // Mark 4 of the 5 ready sessions closing, then check out the remaining 1 ready session via + // newCall() so readyCount == 0 and inUseCount == 1. + List handles = new ArrayList<>(sessionPool.sessions.getAllSessions()); + for (int i = 1; i < handles.size(); i++) { + handles.get(i).onSessionClosing(); + } + checkedOutRpc = sessionPool.newCall(FakeDescriptor.SCRIPTED); + assertThat(sessionPool.sessions.getStats().getReadyCount()).isEqualTo(0); + assertThat(sessionPool.sessions.getStats().getInUseCount()).isEqualTo(1); + + // Simulate consecutive STARTING failures reaching threshold while inUseCount == 1. + consecutiveFailuresField.setInt(sessionPool, 10); + + // A new PendingVRpc started while inUseCount > 0 must queue instead of failing fast. + VRpc pendingRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + pendingRpc.start( + SessionFakeScriptedRequest.newBuilder().setTag(0).build(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + pendingFuture); + assertThat(pendingFuture.isDone()).isFalse(); + } finally { + poolLock.unlock(); + } + + // Complete the checked-out RPC so its session returns to the pool and drains pendingRpc. + checkedOutRpc.start( + SessionFakeScriptedRequest.newBuilder().setTag(0).build(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + checkedOutFuture); + assertThat(checkedOutFuture.get(5, TimeUnit.SECONDS)).isNotNull(); + assertThat(pendingFuture.get(5, TimeUnit.SECONDS)).isNotNull(); + } + private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException { long deadline = System.currentTimeMillis() + timeout.toMillis(); while (System.currentTimeMillis() < deadline) { From 54bb35c546dcfd9189f3d69a4dafe802f709b0ed Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 21:03:35 +0000 Subject: [PATCH 3/7] fix(bigtable): declare failureStatus and failureTrailers outside try block in onSessionClose --- .../bigtable/data/v2/internal/session/SessionPoolImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index e1ae52630d2c..995fe5666754 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -694,6 +694,8 @@ private void onSessionClose( SessionHandle handle, SessionState prevState, Status status, Metadata trailers) { List> toBeClosed = new ArrayList<>(); + Status failureStatus = status; + Metadata failureTrailers = trailers; poolLock.lock(); try { @@ -727,8 +729,6 @@ private void onSessionClose( return; } - Status failureStatus = status; - Metadata failureTrailers = trailers; // Handle abnormal close. This can happen if the Session was aborted due to underlying stream // termination if (prevState != SessionState.WAIT_SERVER_CLOSE) { From 87a8c361ada2ac343a28694a52248544bde9f6d5 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 21:25:42 +0000 Subject: [PATCH 4/7] fix(bigtable): drain pendingRpcs in afterRelease/configListener and guard threshold=0 --- .../v2/internal/session/SessionPoolImpl.java | 81 ++++++++++++------- .../internal/session/SessionPoolImplTest.java | 21 ++++- 2 files changed, 71 insertions(+), 31 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index 995fe5666754..e257bcb29eff 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -57,6 +57,7 @@ import java.time.Instant; import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Collections; import java.util.Deque; import java.util.HashSet; import java.util.Iterator; @@ -281,6 +282,7 @@ public SessionPoolImpl( clientConfig -> clientConfig.getSessionConfiguration().getSessionPoolConfiguration(), newConfig -> { List> toCancel = new ArrayList<>(); + VRpcResult cancelResult = null; poolLock.lock(); try { budget.updateConfig(newConfig); @@ -292,6 +294,9 @@ public SessionPoolImpl( retryCreateSessionTimeout = null; } toCancel = popClosableRpcs(); + cancelResult = + VRpcResult.createRejectedError( + Status.UNAVAILABLE.withDescription("SessionPool configuration disabled")); CloseSessionRequest closeReq = CloseSessionRequest.newBuilder() .setReason(CloseSessionReason.CLOSE_SESSION_REASON_DOWNSIZE) @@ -302,6 +307,10 @@ public SessionPoolImpl( handle.getSession().close(closeReq); } } else if (poolState == PoolState.STARTED) { + if (isFailingWithNoEstablishedSessions()) { + toCancel = popClosableRpcs(); + cancelResult = createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers); + } while (poolSizer.getScaleDelta() > 0) { if (!createSession(openParams)) { break; @@ -312,18 +321,7 @@ public SessionPoolImpl( poolLock.unlock(); } - if (!toCancel.isEmpty()) { - VRpcResult cancelResult = - VRpcResult.createRejectedError( - Status.UNAVAILABLE.withDescription("SessionPool configuration disabled")); - for (PendingVRpc vrpc : toCancel) { - try { - vrpc.cancelWithResult(cancelResult); - } catch (Throwable t) { - logger.log(Level.WARNING, "Exception dispatching close to op executor", t); - } - } - } + cancelPendingRpcs(toCancel, cancelResult); }); } @@ -630,38 +628,55 @@ private void onSessionReady(SessionHandle handle, OpenSessionResponse ignored) { } private void onVRpcComplete(SessionHandle handle, Duration elapsed, VRpcResult result) { + List> toBeClosed; + VRpcResult closeResult = null; poolLock.lock(); try { handle.onVRpcFinish(elapsed, result); - afterRelease(handle); + toBeClosed = afterRelease(handle); + if (!toBeClosed.isEmpty()) { + closeResult = createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers); + } } finally { poolLock.unlock(); } + cancelPendingRpcs(toBeClosed, closeResult); } // Called when a pending vRPC was drained but cancelled before it attached to a real call. // Mirrors onVRpcComplete; the handle reports no latency because nothing ran on it. private void onPendingVRpcCancelled(SessionHandle handle) { + List> toBeClosed; + VRpcResult closeResult = null; poolLock.lock(); try { handle.onPendingVRpcCancelled(); - afterRelease(handle); + toBeClosed = afterRelease(handle); + if (!toBeClosed.isEmpty()) { + closeResult = createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers); + } } finally { poolLock.unlock(); } + cancelPendingRpcs(toBeClosed, closeResult); } @GuardedBy("poolLock") - private void afterRelease(SessionHandle handle) { + private List> afterRelease(SessionHandle handle) { // pool is shutting down, dont try to drain vrpcs if (poolState != PoolState.STARTED) { - return; + return Collections.emptyList(); } - // Session is shutting down, don't try to drain vrpcs + // Session is shutting down, don't try to drain vrpcs. If this was the last in-use session + // and the pool is in consecutive-failure state, drain and fail any remaining pendingRpcs. if (handle.getSession().getState() != SessionState.READY) { - return; + if (isFailingWithNoEstablishedSessions()) { + return popClosableRpcs(); + } + return Collections.emptyList(); } tryDrainPendingRpcs(); + return Collections.emptyList(); } private void onSessionGoAway(SessionHandle handle, GoAwayResponse msg) { @@ -772,14 +787,21 @@ private void onSessionClose( } if (!toBeClosed.isEmpty()) { - // vRPCs failing with consecutive failures should fail - VRpcResult result = createConsecutiveFailureResult(failureStatus, failureTrailers); - for (PendingVRpc vrpc : toBeClosed) { - try { - vrpc.cancelWithResult(result); - } catch (Throwable t) { - logger.log(Level.WARNING, "Exception dispatching close to op executor", t); - } + cancelPendingRpcs( + toBeClosed, createConsecutiveFailureResult(failureStatus, failureTrailers)); + } + } + + private void cancelPendingRpcs( + List> toCancel, @Nullable VRpcResult cancelResult) { + if (toCancel.isEmpty() || cancelResult == null) { + return; + } + for (PendingVRpc vrpc : toCancel) { + try { + vrpc.cancelWithResult(cancelResult); + } catch (Throwable t) { + logger.log(Level.WARNING, "Exception dispatching close to op executor", t); } } } @@ -791,7 +813,8 @@ private boolean hasEstablishedSession() { @GuardedBy("poolLock") private boolean isFailingWithNoEstablishedSessions() { - return consecutiveFailures >= getMaxConsecutiveFailures(configManager) + return consecutiveFailures > 0 + && consecutiveFailures >= getMaxConsecutiveFailures(configManager) && !hasEstablishedSession(); } @@ -1001,8 +1024,8 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { return; } if (isFailingWithNoEstablishedSessions()) { - if (poolSizer.handleNewCall()) { - createSession(openParams); + if (poolSizer.handleNewCall() || sessions.getStats().getExpectedCapacity() == 0) { + createSession(openParams.withIncrementedAttempts()); } cancelWithResult(createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers)); return; diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index fcd5f63439ee..8e3166c397af 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -1449,6 +1449,7 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except } @Test + @SuppressWarnings("GuardedBy") void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception { com.google.rpc.Status errorStatus = com.google.rpc.Status.newBuilder() @@ -1465,8 +1466,24 @@ void consecutiveFailuresFastFailsDuringBudgetLockout() throws Exception { .build(); sessionPool.start(request, new Metadata()); - // Wait until all 20 initial SessionCreationBudget slots have been exhausted and closed - waitForOpenRequestCount(20, Duration.ofSeconds(5)); + // Wait until all initial SessionCreationBudget slots have been exhausted and closed on the + // client + ReentrantLock poolLock = extractPoolLock(sessionPool); + Field consecutiveFailuresField = SessionPoolImpl.class.getDeclaredField("consecutiveFailures"); + consecutiveFailuresField.setAccessible(true); + long deadline = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < deadline) { + poolLock.lock(); + try { + if (sessionPool.sessions.getAllSessions().isEmpty() + && consecutiveFailuresField.getInt(sessionPool) >= 10) { + break; + } + } finally { + poolLock.unlock(); + } + Thread.sleep(10); + } // A new call arriving during the 60s budget lockout must fail immediately instead of queueing UnaryResponseFuture f = new UnaryResponseFuture<>(); From 9f36aa619e8ae4b7198bd4306aa00292612654cf Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 21:45:30 +0000 Subject: [PATCH 5/7] refactor(bigtable): simplify consecutive failure checks to onSessionClose and PendingVRpc.start --- .../v2/internal/session/SessionPoolImpl.java | 118 +++++++----------- .../internal/session/SessionPoolImplTest.java | 3 + 2 files changed, 50 insertions(+), 71 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index e257bcb29eff..466f04d48c45 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -57,7 +57,6 @@ import java.time.Instant; import java.util.ArrayDeque; import java.util.ArrayList; -import java.util.Collections; import java.util.Deque; import java.util.HashSet; import java.util.Iterator; @@ -146,10 +145,12 @@ private enum PoolState { private int consecutiveFailures = 0; @GuardedBy("poolLock") - private Status lastCloseStatus = Status.UNAVAILABLE; + @Nullable + private Status lastCloseStatus = null; @GuardedBy("poolLock") - private Metadata lastCloseTrailers = new Metadata(); + @Nullable + private Metadata lastCloseTrailers = null; /** * When the client fallback to a non-session AFE session creation will return unimplemented @@ -282,7 +283,6 @@ public SessionPoolImpl( clientConfig -> clientConfig.getSessionConfiguration().getSessionPoolConfiguration(), newConfig -> { List> toCancel = new ArrayList<>(); - VRpcResult cancelResult = null; poolLock.lock(); try { budget.updateConfig(newConfig); @@ -294,9 +294,6 @@ public SessionPoolImpl( retryCreateSessionTimeout = null; } toCancel = popClosableRpcs(); - cancelResult = - VRpcResult.createRejectedError( - Status.UNAVAILABLE.withDescription("SessionPool configuration disabled")); CloseSessionRequest closeReq = CloseSessionRequest.newBuilder() .setReason(CloseSessionReason.CLOSE_SESSION_REASON_DOWNSIZE) @@ -307,10 +304,6 @@ public SessionPoolImpl( handle.getSession().close(closeReq); } } else if (poolState == PoolState.STARTED) { - if (isFailingWithNoEstablishedSessions()) { - toCancel = popClosableRpcs(); - cancelResult = createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers); - } while (poolSizer.getScaleDelta() > 0) { if (!createSession(openParams)) { break; @@ -321,7 +314,18 @@ public SessionPoolImpl( poolLock.unlock(); } - cancelPendingRpcs(toCancel, cancelResult); + if (!toCancel.isEmpty()) { + VRpcResult cancelResult = + VRpcResult.createRejectedError( + Status.UNAVAILABLE.withDescription("SessionPool configuration disabled")); + for (PendingVRpc vrpc : toCancel) { + try { + vrpc.cancelWithResult(cancelResult); + } catch (Throwable t) { + logger.log(Level.WARNING, "Exception dispatching close to op executor", t); + } + } + } }); } @@ -602,6 +606,8 @@ private void onSessionReady(SessionHandle handle, OpenSessionResponse ignored) { new Object[] {handle.getSession().getLogName(), handle.getSession().getState()}); consecutiveFailures = 0; + lastCloseStatus = null; + lastCloseTrailers = null; consecutiveUnimplementedFailures = 0; if (poolState != PoolState.STARTED) { @@ -628,55 +634,38 @@ private void onSessionReady(SessionHandle handle, OpenSessionResponse ignored) { } private void onVRpcComplete(SessionHandle handle, Duration elapsed, VRpcResult result) { - List> toBeClosed; - VRpcResult closeResult = null; poolLock.lock(); try { handle.onVRpcFinish(elapsed, result); - toBeClosed = afterRelease(handle); - if (!toBeClosed.isEmpty()) { - closeResult = createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers); - } + afterRelease(handle); } finally { poolLock.unlock(); } - cancelPendingRpcs(toBeClosed, closeResult); } // Called when a pending vRPC was drained but cancelled before it attached to a real call. // Mirrors onVRpcComplete; the handle reports no latency because nothing ran on it. private void onPendingVRpcCancelled(SessionHandle handle) { - List> toBeClosed; - VRpcResult closeResult = null; poolLock.lock(); try { handle.onPendingVRpcCancelled(); - toBeClosed = afterRelease(handle); - if (!toBeClosed.isEmpty()) { - closeResult = createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers); - } + afterRelease(handle); } finally { poolLock.unlock(); } - cancelPendingRpcs(toBeClosed, closeResult); } @GuardedBy("poolLock") - private List> afterRelease(SessionHandle handle) { + private void afterRelease(SessionHandle handle) { // pool is shutting down, dont try to drain vrpcs if (poolState != PoolState.STARTED) { - return Collections.emptyList(); + return; } - // Session is shutting down, don't try to drain vrpcs. If this was the last in-use session - // and the pool is in consecutive-failure state, drain and fail any remaining pendingRpcs. + // Session is shutting down, don't try to drain vrpcs if (handle.getSession().getState() != SessionState.READY) { - if (isFailingWithNoEstablishedSessions()) { - return popClosableRpcs(); - } - return Collections.emptyList(); + return; } tryDrainPendingRpcs(); - return Collections.emptyList(); } private void onSessionGoAway(SessionHandle handle, GoAwayResponse msg) { @@ -709,8 +698,6 @@ private void onSessionClose( SessionHandle handle, SessionState prevState, Status status, Metadata trailers) { List> toBeClosed = new ArrayList<>(); - Status failureStatus = status; - Metadata failureTrailers = trailers; poolLock.lock(); try { @@ -759,6 +746,10 @@ private void onSessionClose( } else { consecutiveUnimplementedFailures = 0; } + // TODO: decide if max consecutive failures should be capped per client + if (isFailingWithNoEstablishedSessions()) { + toBeClosed = popClosableRpcs(); + } } // Budget release for STARTING-phase closes is handled above via sessionsHoldingBudget, @@ -772,36 +763,19 @@ private void onSessionClose( createSession(openParams.withIncrementedAttempts()); } } - - // Only fail pending vRPCs when session creation is failing consecutively AND the pool has no - // established (READY or IN_USE) sessions left to drain the queue. Checking this outside the - // STARTING branch also drains pendingRpcs if the last established session closes (e.g. on - // GOAWAY or keepalive timeout) while consecutiveFailures is already above the threshold. - if (isFailingWithNoEstablishedSessions()) { - failureStatus = lastCloseStatus; - failureTrailers = lastCloseTrailers; - toBeClosed = popClosableRpcs(); - } } finally { poolLock.unlock(); } if (!toBeClosed.isEmpty()) { - cancelPendingRpcs( - toBeClosed, createConsecutiveFailureResult(failureStatus, failureTrailers)); - } - } - - private void cancelPendingRpcs( - List> toCancel, @Nullable VRpcResult cancelResult) { - if (toCancel.isEmpty() || cancelResult == null) { - return; - } - for (PendingVRpc vrpc : toCancel) { - try { - vrpc.cancelWithResult(cancelResult); - } catch (Throwable t) { - logger.log(Level.WARNING, "Exception dispatching close to op executor", t); + // vRPCs failing with consecutive failures should fail + VRpcResult result = createConsecutiveFailureResult(status, trailers); + for (PendingVRpc vrpc : toBeClosed) { + try { + vrpc.cancelWithResult(result); + } catch (Throwable t) { + logger.log(Level.WARNING, "Exception dispatching close to op executor", t); + } } } } @@ -813,19 +787,21 @@ private boolean hasEstablishedSession() { @GuardedBy("poolLock") private boolean isFailingWithNoEstablishedSessions() { - return consecutiveFailures > 0 - && consecutiveFailures >= getMaxConsecutiveFailures(configManager) + return consecutiveFailures >= getMaxConsecutiveFailures(configManager) + && lastCloseStatus != null && !hasEstablishedSession(); } - private static VRpcResult createConsecutiveFailureResult(Status status, Metadata trailers) { + private static VRpcResult createConsecutiveFailureResult( + Status status, @Nullable Metadata trailers) { + Status errorStatus = + status.isOk() ? Status.UNAVAILABLE : Status.fromCode(status.getCode()); return VRpcResult.createRejectedError( - Status.fromCode(status.getCode()) - .withDescription( - String.format( - "Session failed with consecutive failures. Most recent server status: %s," - + " metadata: %s.", - status, trailers))); + errorStatus.withDescription( + String.format( + "Session failed with consecutive failures. Most recent server status: %s," + + " metadata: %s.", + status, trailers))); } @GuardedBy("poolLock") diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index 8e3166c397af..fc9a0affc14e 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -1528,6 +1528,9 @@ void consecutiveFailuresPreservesPendingRpcsWhilePoolHasInUseSessions() throws E assertThat(sessionPool.sessions.getStats().getInUseCount()).isEqualTo(1); // Simulate consecutive STARTING failures reaching threshold while inUseCount == 1. + Field lastCloseStatusField = SessionPoolImpl.class.getDeclaredField("lastCloseStatus"); + lastCloseStatusField.setAccessible(true); + lastCloseStatusField.set(sessionPool, Status.INTERNAL.withDescription("fake internal error")); consecutiveFailuresField.setInt(sessionPool, 10); // A new PendingVRpc started while inUseCount > 0 must queue instead of failing fast. From 66069b701f24e63a0a3f680c737fc48beed9a446 Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 21:53:30 +0000 Subject: [PATCH 6/7] refactor(bigtable): cancel directly in PendingVRpc.start when failing with no established sessions --- .../bigtable/data/v2/internal/session/SessionPoolImpl.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index 466f04d48c45..7d836e3bbb6f 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -1000,9 +1000,6 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { return; } if (isFailingWithNoEstablishedSessions()) { - if (poolSizer.handleNewCall() || sessions.getStats().getExpectedCapacity() == 0) { - createSession(openParams.withIncrementedAttempts()); - } cancelWithResult(createConsecutiveFailureResult(lastCloseStatus, lastCloseTrailers)); return; } From f7e3c4e2564cc7620a6e7e53e740cf762de3073c Mon Sep 17 00:00:00 2001 From: Mattie Fu Date: Mon, 16 Nov 2026 22:00:41 +0000 Subject: [PATCH 7/7] style(bigtable): format SessionPoolImpl with google-java-format --- .../bigtable/data/v2/internal/session/SessionPoolImpl.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index 7d836e3bbb6f..b1d1e8ede269 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -794,8 +794,7 @@ private boolean isFailingWithNoEstablishedSessions() { private static VRpcResult createConsecutiveFailureResult( Status status, @Nullable Metadata trailers) { - Status errorStatus = - status.isOk() ? Status.UNAVAILABLE : Status.fromCode(status.getCode()); + Status errorStatus = status.isOk() ? Status.UNAVAILABLE : Status.fromCode(status.getCode()); return VRpcResult.createRejectedError( errorStatus.withDescription( String.format(