diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java index 3e9d166d9ae0..4c339626d913 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizer.java @@ -20,6 +20,7 @@ import com.google.bigtable.v2.SessionClientConfiguration; import com.google.cloud.bigtable.data.v2.internal.session.SessionList.PoolStats; import com.google.rpc.Status; +import java.util.function.IntSupplier; import javax.annotation.CheckReturnValue; class PoolSizer { @@ -27,12 +28,13 @@ class PoolSizer { // Fraction of idle sessions to keep in order to manage an increase in requests-in-flight. For // example, a headroom of 50% will keep enough sessions to deal with a 50% increase in QPS or a // 50% increase in latency (since that also increases requests-in-flight). - private volatile float idlesSessionHeadRoom; - private volatile int minIdleSessions; - private volatile int maxIdleSessions; + private float idlesSessionHeadRoom; + private int minIdleSessions; + private int maxIdleSessions; private final PoolStats stats; private final Sized pendingRpcs; + private final IntSupplier consecutiveFailures; // This resolves cold start overprovisioning issue. // Session creation sometimes involves creating new connection and warming the backend. @@ -40,9 +42,10 @@ class PoolSizer { // point reads are expected to complete within 1-4ms. // We assume that it is okay to collect 10 pending calls per starting session so that after // the session is ready it takes 10-40ms to process all pending calls. - private volatile int pendingVRpcsPerSession; + private int pendingVRpcsPerSession; + private int maxConsecutiveFailures; - private volatile boolean disabled; + private boolean disabled; interface Sized { int getSize(); @@ -52,8 +55,17 @@ interface Sized { PoolStats stats, Sized pendingRpcs, SessionClientConfiguration.SessionPoolConfiguration poolConfig) { + this(stats, pendingRpcs, () -> 0, poolConfig); + } + + PoolSizer( + PoolStats stats, + Sized pendingRpcs, + IntSupplier consecutiveFailures, + SessionClientConfiguration.SessionPoolConfiguration poolConfig) { this.stats = stats; this.pendingRpcs = pendingRpcs; + this.consecutiveFailures = consecutiveFailures; updateConfig(poolConfig); } @@ -62,6 +74,7 @@ void updateConfig(SessionClientConfiguration.SessionPoolConfiguration poolConfig this.minIdleSessions = poolConfiguration.getMinSessionCount(); this.maxIdleSessions = poolConfiguration.getMaxSessionCount(); this.pendingVRpcsPerSession = poolConfiguration.getNewSessionQueueLength(); + this.maxConsecutiveFailures = poolConfiguration.getConsecutiveSessionFailureThreshold(); this.disabled = poolConfiguration.equals( SessionClientConfiguration.SessionPoolConfiguration.getDefaultInstance()) @@ -73,12 +86,20 @@ boolean isDisabled() { } public int getScaleDelta() { + if (isDisabled()) { + return 0; + } + int failures = consecutiveFailures.getAsInt(); // Assume each session handles 1 RPC at a time. This should be revisited if sessions get // support for multiplexing. + // Ignore queued pendingRpcs while session establishment is failing so queued requests do not + // drive unbounded session creation. int effectivePending = - pendingVRpcsPerSession > 0 - ? (int) Math.ceil((float) pendingRpcs.getSize() / pendingVRpcsPerSession) - : pendingRpcs.getSize(); + failures > 0 + ? 0 + : (pendingVRpcsPerSession > 0 + ? (int) Math.ceil((float) pendingRpcs.getSize() / pendingVRpcsPerSession) + : pendingRpcs.getSize()); int sessionsInUse = effectivePending + stats.getInUseCount(); int unboundedDesiredIdleSessions = (int) Math.ceil(sessionsInUse * idlesSessionHeadRoom); int desiredIdleSessions = @@ -91,8 +112,8 @@ public int getScaleDelta() { if (desiredCapacity < immediateCapacity) { return desiredCapacity - immediateCapacity; } else if (desiredCapacity > eventualCapacity) { - if (isDisabled()) { - return 0; + if (maxConsecutiveFailures > 0 && failures >= maxConsecutiveFailures) { + return stats.getStartingCount() == 0 ? 1 : 0; } return desiredCapacity - eventualCapacity; } else { diff --git a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java index c1abfb90232c..98e16561808c 100644 --- a/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java +++ b/java-bigtable/google-cloud-bigtable/src/main/java/com/google/cloud/bigtable/data/v2/internal/session/SessionPoolImpl.java @@ -256,6 +256,7 @@ public SessionPoolImpl( new PoolSizer( sessions.getStats(), pendingRpcs::size, + () -> consecutiveFailures, configManager .getClientConfiguration() .getSessionConfiguration() @@ -597,6 +598,7 @@ private void onSessionReady(SessionHandle handle, OpenSessionResponse ignored) { "Session {0} in state {1}", new Object[] {handle.getSession().getLogName(), handle.getSession().getState()}); + boolean wasFailingToOpen = consecutiveFailures > 0; consecutiveFailures = 0; consecutiveUnimplementedFailures = 0; @@ -618,6 +620,14 @@ private void onSessionReady(SessionHandle handle, OpenSessionResponse ignored) { // handle pending rpcs tryDrainPendingRpcs(); + + if (wasFailingToOpen) { + while (poolSizer.getScaleDelta() > 0) { + if (!createSession(openParams)) { + break; + } + } + } } finally { poolLock.unlock(); } diff --git a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java index a343c7973ce8..08f1003224c7 100644 --- a/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java +++ b/java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/data/v2/internal/session/PoolSizerTest.java @@ -33,6 +33,7 @@ class PoolSizerTest { private static final int MIN_IDLE_SESSIONS = 5; private static final int MAX_IDLE_SESSIONS = 400; private static final int PENDING_RPCS_PER_SESSION = 10; + private static final int MAX_CONSECUTIVE_FAILURES = 10; private static final SessionClientConfiguration.SessionPoolConfiguration DEFAULT_CONFIG = SessionClientConfiguration.SessionPoolConfiguration.newBuilder() @@ -40,6 +41,7 @@ class PoolSizerTest { .setMinSessionCount(MIN_IDLE_SESSIONS) .setMaxSessionCount(MAX_IDLE_SESSIONS) .setNewSessionQueueLength(PENDING_RPCS_PER_SESSION) + .setConsecutiveSessionFailureThreshold(MAX_CONSECUTIVE_FAILURES) .build(); @Test @@ -244,9 +246,49 @@ void testEmptyConfig() { assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isFalse(); stats.getTestHelper().setInUseCount(2); - stats.getTestHelper().setExpectedCapacity(0); + stats.getTestHelper().setExpectedCapacity(2); assertThat(sizer.getScaleDelta()).isEqualTo(0); assertThat(sizer.handleGoAway(GoAwayResponse.getDefaultInstance())).isFalse(); assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isFalse(); } + + @Test + void testConsecutiveFailuresThrottlesSessionCreation() { + PoolStats stats = new PoolStats(); + AtomicInteger pendingCount = new AtomicInteger(200); + AtomicInteger consecutiveFailures = new AtomicInteger(0); + PoolSizer sizer = + new PoolSizer(stats, pendingCount::get, consecutiveFailures::get, DEFAULT_CONFIG); + + // While consecutiveFailures > 0, queued pendingRpcs must not drive scale-up beyond + // MIN_IDLE_SESSIONS. + consecutiveFailures.set(1); + stats.getTestHelper().setStartingCount(MIN_IDLE_SESSIONS); + stats.getTestHelper().setExpectedCapacity(MIN_IDLE_SESSIONS); + assertThat(sizer.getScaleDelta()).isEqualTo(0); + assertThat(sizer.handleNewCall()).isFalse(); + + // Once consecutiveFailures >= MAX_CONSECUTIVE_FAILURES, allow at most 1 STARTING probe session. + consecutiveFailures.set(MAX_CONSECUTIVE_FAILURES); + stats.getTestHelper().setStartingCount(1); + stats.getTestHelper().setExpectedCapacity(1); + assertThat(sizer.getScaleDelta()).isEqualTo(0); + assertThat(sizer.handleNewCall()).isFalse(); + assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isFalse(); + + // When the last STARTING session closes (startingCount == 0), allow 1 probe session. + stats.getTestHelper().setStartingCount(0); + stats.getTestHelper().setExpectedCapacity(0); + assertThat(sizer.getScaleDelta()).isEqualTo(1); + assertThat(sizer.handleNewCall()).isTrue(); + assertThat(sizer.handleSessionClose(Status.getDefaultInstance())).isTrue(); + + // Once a session succeeds and resets consecutiveFailures to 0, full scaling resumes. + consecutiveFailures.set(0); + stats.getTestHelper().setReadyCount(1); + stats.getTestHelper().setExpectedCapacity(1); + int effectivePending = (int) Math.ceil((float) pendingCount.get() / PENDING_RPCS_PER_SESSION); + int desiredIdleSessions = (int) Math.ceil(effectivePending * IDLE_SESSION_HEADROOM_RATIO); + assertThat(sizer.getScaleDelta()).isEqualTo(effectivePending + desiredIdleSessions - 1); + } }