Keep port 2010 working through reconnects: notice dead sessions, retry the forward, probe it

src/main/java/com/testingbot/tunnel/App.java

@@ -1554,6 +1554,12 @@ boolean selfTest() {
 
     /** How long between self-test retries, and how many are attempted before giving up. */
     static final long SELF_TEST_RETRY_INTERVAL_MS = 30_000L;
+    /**
+     * The first retry comes quickly. The usual startup failure is a race -- the tunnel server's
+     * proxy not yet routing to a forward registered a moment ago -- that is over within a second,
+     * and waiting the full 30 seconds held every test back by that much.
+     */
+    static final long SELF_TEST_FIRST_RETRY_MS = 2_000L;
     static final int SELF_TEST_RETRY_ATTEMPTS = 10;
 
     private Timer selfTestRetryTimer;
@@ -1614,7 +1620,18 @@ public void run() {
                     cancelSelfTestRetry();
                 }
             }
-        }, selfTestRetryIntervalMs, selfTestRetryIntervalMs);
+        }, Math.min(SELF_TEST_FIRST_RETRY_MS, selfTestRetryIntervalMs), selfTestRetryIntervalMs);
+    }
+
+    /**
+     * Called by the tunnel when it re-established port 2010 on a session that stayed up. Nothing
+     * else re-checks readiness in that case, so the tunnel would otherwise stay not-ready with a
+     * working forward.
+     */
+    public void reverseForwardRestored() {
+        if (!isReady()) {
+            scheduleSelfTestRetry();
+        }
     }
 
     synchronized void cancelSelfTestRetry() {

src/main/java/com/testingbot/tunnel/ForwardProbeHandler.java

@@ -0,0 +1,37 @@
+package com.testingbot.tunnel;
+
+import org.eclipse.jetty.http.HttpHeader;
+import org.eclipse.jetty.http.HttpStatus;
+import org.eclipse.jetty.server.Handler;
+import org.eclipse.jetty.server.Request;
+import org.eclipse.jetty.server.Response;
+import org.eclipse.jetty.util.Callback;
+
+/**
+ * Answers the reverse-forward probe ({@code SSHTunnel.probeReverseForward}) from the local proxy
+ * itself, before anything is proxied or logged.
+ *
+ * <p>The probe enters port 2010 on the tunnel server and must come back out here; echoing its
+ * nonce proves the round trip went through this client's forward and not through a listener some
+ * other, dead session left behind. The host is in {@code .invalid}, which never resolves, so no
+ * real request can match it.
+ */
+public class ForwardProbeHandler extends Handler.Wrapper {
+    public static final String PROBE_HOST = "reverse-forward-probe.testingbot.invalid";
+    public static final String PROBE_HEADER = "X-TestingBot-Probe";
+
+    @Override
+    public boolean handle(Request request, Response response, Callback callback) throws Exception {
+        if (!PROBE_HOST.equalsIgnoreCase(Request.getServerName(request))) {
+            return super.handle(request, response, callback);
+        }
+        String nonce = request.getHeaders().get(PROBE_HEADER);
+        response.setStatus(HttpStatus.NO_CONTENT_204);
+        if (nonce != null) {
+            response.getHeaders().put(PROBE_HEADER, nonce);
+        }
+        response.getHeaders().put(HttpHeader.CONNECTION, "close");
+        callback.succeeded();
+        return true;
+    }
+}

src/main/java/com/testingbot/tunnel/HttpProxy.java

@@ -229,7 +229,11 @@ public void onClosed(org.eclipse.jetty.io.Connection connection) {
         // GracefulHandler tracks in-flight requests so stop() can drain them; it must sit
         // outermost to see every request.
         GracefulHandler gracefulHandler = new GracefulHandler();
-        gracefulHandler.setHandler(logHandler);
+        // Before the log handler: the reverse-forward probe arrives every few seconds and is
+        // not traffic anyone wants to see in the HTTP log or the request metrics.
+        ForwardProbeHandler probeHandler = new ForwardProbeHandler();
+        probeHandler.setHandler(logHandler);
+        gracefulHandler.setHandler(probeHandler);
 
         if (app.isDebugMode()) {
             // Jetty 12 handlers own a Callback that must be completed exactly once. This

src/main/java/ssh/CustomConnectionMonitor.java

@@ -26,6 +26,16 @@ public class CustomConnectionMonitor {
 
     static final long CURRENT_RETRY_DELAY = 5000;
     static final int MAX_RETRIES = 30;
+    /**
+     * The first reconnect goes out after this long, not after the full retry delay: the session
+     * has already been gone for the keepalive's ~15 seconds, and every second more is a second
+     * of failing tests.
+     */
+    static final long FIRST_RETRY_DELAY = 1000;
+    /** Between re-requests of the reverse forward on a session that is up. */
+    static final long FORWARD_RETRY_DELAY = 2000;
+    /** About two minutes of re-requests before the tunnel is rebuilt. */
+    static final int MAX_FORWARD_RETRIES = 60;
 
     private final ReconnectableTunnel tunnel;
     private final ReconnectHost host;
@@ -34,6 +44,7 @@ public class CustomConnectionMonitor {
 
     private final AtomicBoolean retrying = new AtomicBoolean(false);
     private int retryAttempts = 0;
+    private int forwardRetryAttempts = 0;
     private final Object lifecycleLock = new Object();
     private volatile boolean cancelled;
 
@@ -67,7 +78,7 @@ public void connectionLost(Throwable reason) {
             // Only the first loss starts a retry cycle; further reports while one is already in
             // flight would otherwise stack up timers all racing to reconnect the same tunnel.
             if (retrying.compareAndSet(false, true)) {
-                scheduleRetry();
+                scheduleRetry(this::attemptReconnect, Math.min(FIRST_RETRY_DELAY, retryDelayMs));
             }
         }
     }
@@ -77,9 +88,13 @@ private void scheduleRetry() {
     }
 
     private void scheduleRetry(Runnable task) {
+        scheduleRetry(task, retryDelayMs);
+    }
+
+    private void scheduleRetry(Runnable task, long delayMs) {
         synchronized (lifecycleLock) {
             if (!isStopped()) {
-                scheduler.scheduleOnce("Reconnect-" + tunnel.getConnectionId(), task, retryDelayMs);
+                scheduler.scheduleOnce("Reconnect-" + tunnel.getConnectionId(), task, delayMs);
             }
         }
     }
@@ -171,21 +186,48 @@ private void finishReconnect() {
             return;
         }
         tunnel.createPortForwarding();
+        checkReconnected();
+    }
+
+    /** Ready when the session and the forwards are up; otherwise schedules what is missing. */
+    private void checkReconnected() {
         synchronized (lifecycleLock) {
             if (isStopped()) {
                 tunnel.stop();
                 return;
             }
-            if (!tunnel.isAuthenticated() || !tunnel.isForwardingEstablished()) {
+            if (!tunnel.isAuthenticated()) {
                 host.setReady(false);
-                LOG.log(Level.WARNING, "SSH reconnected but port forwarding failed; retrying setup");
+                LOG.log(Level.WARNING, "SSH dropped again while setting up port forwarding; retrying");
                 if (retryAttempts >= MAX_RETRIES) {
                     giveUpAndRebuild();
                 } else {
                     scheduleRetry();
                 }
                 return;
             }
+            if (!tunnel.isForwardingEstablished()) {
+                // The session is up; what failed is the forward. On a tunnel server this is
+                // nearly always 2010 still held by the session this one replaces, which the
+                // server releases once it notices that session is gone. Tearing the healthy
+                // session down and re-dialling it every five seconds -- what this did -- made
+                // each attempt slower and burned the retry budget, so a stale listener that
+                // lived longer than ~3 minutes ended in a rebuild. Ask again on this session,
+                // often, and rebuild only if the port never comes free.
+                host.setReady(false);
+                forwardRetryAttempts += 1;
+                LOG.log(Level.WARNING, String.format(
+                        "[%s] SSH reconnected but port forwarding failed; requesting it again (%d)",
+                        tunnel.getConnectionId(), forwardRetryAttempts));
+                if (forwardRetryAttempts >= MAX_FORWARD_RETRIES) {
+                    forwardRetryAttempts = 0;
+                    giveUpAndRebuild();
+                } else {
+                    scheduleRetry(this::retryForwarding, FORWARD_RETRY_DELAY);
+                }
+                return;
+            }
+            forwardRetryAttempts = 0;
             retrying.set(false);
             scheduler.cancel();
             host.setReady(true);
@@ -196,6 +238,19 @@ private void finishReconnect() {
         }
     }
 
+    /** Re-requests the forwards on the current session; reconnects if the session went away. */
+    private void retryForwarding() {
+        if (isStopped()) {
+            return;
+        }
+        if (!tunnel.isAuthenticated()) {
+            attemptReconnect();
+            return;
+        }
+        tunnel.restartPortForwarding();
+        checkReconnected();
+    }
+
     /**
      * Restarts the local proxy without touching the SSH session.
      *

src/main/java/ssh/ReconnectableTunnel.java

@@ -28,4 +28,9 @@ public interface ReconnectableTunnel {
     default boolean isForwardingEstablished() {
         return false;
     }
+
+    /** Drops whatever forwards are registered and requests them again, on the same session. */
+    default void restartPortForwarding() {
+        createPortForwarding();
+    }
 }