diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java index 79fc1109fe23..a75d1b7b2ecc 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionList.java @@ -63,7 +63,6 @@ class SessionList { // - starting sessions // - closing sessions private final Set allSessions = new HashSet<>(); - private final Set inUseSessions = new HashSet<>(); // pool level statistics across all the afes private final PoolStats poolStats = new PoolStats(); @@ -119,7 +118,6 @@ public Optional checkoutSession(AfeHandle afeHandle) { handle -> { poolStats.readyCount--; poolStats.inUseCount++; - inUseSessions.add(handle); if (handle.afe.get().sessions.isEmpty()) { afesWithReadySessions.remove(afeHandle); } @@ -221,7 +219,6 @@ private void releaseToPool() { AfeHandle afeHandle = this.afe.get(); poolStats.inUseCount--; - inUseSessions.remove(this); if (!closing && session.getState() == SessionState.READY) { poolStats.readyCount++; diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index c1abfb90232c..8b21a48dd7ba 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -911,7 +911,7 @@ class PendingVRpc implements VRpc realCall; - private BigtableTimer.Timeout deadlineTimeout; + private volatile BigtableTimer.Timeout deadlineTimeout; public PendingVRpc(VRpcDescriptor desc) { this.desc = desc; @@ -922,7 +922,7 @@ public PendingVRpc(VRpcDescriptor desc) { // region. Bounded acquisition on the request hot path; see poolLock. @SuppressWarnings("GuardedBy") public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { - Preconditions.checkState(this.req == null, "request is already started"); + Preconditions.checkState(this.ctx == null, "request is already started"); Preconditions.checkNotNull(req, "request can't be null"); Preconditions.checkNotNull(ctx, "ctx can't be null"); Preconditions.checkNotNull(listener, "listener can't be null"); @@ -940,13 +940,7 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { VRpcResult.createUncommitedError( Status.UNAVAILABLE.withCause( new IllegalStateException("SessionPool lock unavailable (pool wedged)"))); - ctx.getExecutor() - .execute( - () -> { - if (isCancelled) return; - isCancelled = true; - listener.onClose(result); - }); + cancelWithResult(result); return; } try { @@ -954,20 +948,14 @@ public void start(ReqT req, VRpcCallContext ctx, VRpcListener listener) { VRpcResult result = VRpcResult.createUncommitedError( Status.UNAVAILABLE.withCause(new IllegalStateException("SessionPool is closed"))); - ctx.getExecutor().execute(() -> listener.onClose(result)); + cancelWithResult(result); return; } if (poolSizer.isDisabled()) { VRpcResult result = VRpcResult.createRejectedError( Status.UNAVAILABLE.withDescription("SessionPool configuration disabled")); - ctx.getExecutor() - .execute( - () -> { - if (isCancelled) return; - isCancelled = true; - listener.onClose(result); - }); + cancelWithResult(result); return; } // Only arm the deadline monitor after we've committed to queueing; otherwise the @@ -1013,8 +1001,10 @@ public void cancel(@Nullable String message, @Nullable Throwable cause) { // region below. @SuppressWarnings("GuardedBy") private void cancel(Status status, boolean onlyCancelPendingCall) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } // The eager removal is only an optimization to skip a wasted session pick in // tryDrainPendingRpcs. @@ -1037,26 +1027,36 @@ private void cancel(Status status, boolean onlyCancelPendingCall) { () -> { if (isCancelled) return; isCancelled = true; + VRpcListener pendingListener = listener; + req = null; + listener = null; if (realCall != null) { if (!onlyCancelPendingCall) { realCall.cancel(status.getDescription(), status.getCause()); } - } else { - listener.onClose(VRpcResult.createRejectedError(status)); + } else if (pendingListener != null) { + pendingListener.onClose(VRpcResult.createRejectedError(status)); } }); } void cancelWithResult(VRpcResult result) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } ctx.getExecutor() .execute( () -> { if (isCancelled) return; isCancelled = true; - listener.onClose(result); + VRpcListener pendingListener = listener; + req = null; + listener = null; + if (pendingListener != null) { + pendingListener.onClose(result); + } }); } @@ -1077,8 +1077,10 @@ public void requestNext() { } private void drainTo(SessionHandle handle) { - if (deadlineTimeout != null) { - deadlineTimeout.cancel(); + BigtableTimer.Timeout timeout = deadlineTimeout; + deadlineTimeout = null; + if (timeout != null) { + timeout.cancel(); } ctx.getExecutor() .execute( @@ -1087,15 +1089,15 @@ private void drainTo(SessionHandle handle) { SessionPoolImpl.this.onPendingVRpcCancelled(handle); return; } + ReqT pendingReq = req; + VRpcListener pendingListener = listener; + req = null; + listener = null; realCall = newRealCall(desc, handle); - realCall.start(req, ctx, listener); + realCall.start(pendingReq, ctx, pendingListener); }); } - private VRpcListener getListener() { - return listener; - } - private BigtableTimer.Timeout monitorDeadline(Deadline deadline) { // Body runs on backgroundExecutor (off the tick thread). onlyCancelPendingCall=true avoids // racing with a user cancel that already attached a real call. diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java index 55afab0b79dd..911955b67565 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImplTest.java @@ -1448,6 +1448,46 @@ void configUpdateToEmptyClosesSessionsAndScalesBackUpWhenEnabled() throws Except assertThat(sessionPool.hasSession()).isTrue(); } + @Test + void pendingVRpcClearsReferencesAfterDrainAndCancel() throws Exception { + sessionPool.start(OpenFakeSessionRequest.getDefaultInstance(), new Metadata()); + + // Create both PendingVRpcs before any session becomes READY so both return PendingVRpc + UnaryResponseFuture drainedFuture = new UnaryResponseFuture<>(); + VRpc drainedRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + UnaryResponseFuture cancelledFuture = new UnaryResponseFuture<>(); + VRpc cancelledRpc = + sessionPool.newCall(FakeDescriptor.SCRIPTED); + + cancelledRpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + cancelledFuture); + cancelledRpc.cancel("user cancelled", null); + + drainedRpc.start( + SessionFakeScriptedRequest.getDefaultInstance(), + VRpcCallContext.create(Deadline.after(1, TimeUnit.MINUTES), true, vrpcTracer), + drainedFuture); + drainedFuture.get(5, TimeUnit.SECONDS); + + Field reqField = drainedRpc.getClass().getDeclaredField("req"); + Field listenerField = drainedRpc.getClass().getDeclaredField("listener"); + Field timeoutField = drainedRpc.getClass().getDeclaredField("deadlineTimeout"); + reqField.setAccessible(true); + listenerField.setAccessible(true); + timeoutField.setAccessible(true); + + assertThat(reqField.get(drainedRpc)).isNull(); + assertThat(listenerField.get(drainedRpc)).isNull(); + assertThat(timeoutField.get(drainedRpc)).isNull(); + + assertThat(reqField.get(cancelledRpc)).isNull(); + assertThat(listenerField.get(cancelledRpc)).isNull(); + assertThat(timeoutField.get(cancelledRpc)).isNull(); + } + private void waitForOpenRequestCount(int minCount, Duration timeout) throws InterruptedException { long deadline = System.currentTimeMillis() + timeout.toMillis(); while (System.currentTimeMillis() < deadline) {