Open several SSH connections to the tunnel server (--ssh-connections, default 4)

src/main/java/com/testingbot/tunnel/App.java

@@ -93,6 +93,10 @@ public class App {
             com.testingbot.tunnel.proxy.NegotiateHosts.none();
     /** Null means "keep the built-in default", so an absent flag changes nothing. */
     private Integer httpDialTimeoutSeconds = null;
+    /** SSH connections to the tunnel server; see {@code --ssh-connections}. */
+    private int sshConnections = DEFAULT_SSH_CONNECTIONS;
+    static final int DEFAULT_SSH_CONNECTIONS = 4;
+    static final int MAX_SSH_CONNECTIONS = 8;
     private Integer httpIdleTimeoutSeconds = null;
     private String controlProxy = null;
     private String controlProxyAuth = null;
@@ -426,6 +430,16 @@ static Options buildOptions() {
         krb5Hosts.setArgName("HOST,HOST");
         options.addOption(krb5Hosts);
 
+        Option sshConnections = new Option(null, "ssh-connections", true,
+            "How many SSH connections to open to the tunnel server, 1-" + MAX_SSH_CONNECTIONS
+            + " (default " + DEFAULT_SSH_CONNECTIONS + "). Browser connections are spread over "
+            + "them, so on a lossy or distant link one lost packet no longer stalls every request "
+            + "and the tunnel is not held to one TCP connection's share of the link; when one "
+            + "connection drops the others keep carrying traffic. Tunnel servers that do not "
+            + "support this get a single connection, as do tunnels with 1.");
+        sshConnections.setArgName("n");
+        options.addOption(sshConnections);
+
         Option dialTimeout = new Option(null, "http-dial-timeout", true,
             "Seconds to wait for a TCP connection to a target or upstream proxy before giving "
             + "up (default 5 for the API, 15 for tunnelled connections). Lower it on a network "
@@ -845,6 +859,19 @@ static void applyOptions(App app, CommandLine commandLine) throws Exception {
         }
 
         app.httpDialTimeoutSeconds = positiveSeconds(commandLine, "http-dial-timeout");
+        if (commandLine.hasOption("ssh-connections")) {
+            String value = commandLine.getOptionValue("ssh-connections").trim();
+            try {
+                int n = Integer.parseInt(value);
+                if (n < 1 || n > MAX_SSH_CONNECTIONS) {
+                    throw new NumberFormatException(value);
+                }
+                app.sshConnections = n;
+            } catch (NumberFormatException invalid) {
+                throw new ParseException("Invalid --ssh-connections '" + value
+                        + "': expected a whole number from 1 to " + MAX_SSH_CONNECTIONS);
+            }
+        }
         app.httpIdleTimeoutSeconds = positiveSeconds(commandLine, "http-idle-timeout");
 
         if (commandLine.hasOption("dns")) {
@@ -1537,6 +1564,11 @@ boolean startProxies() {
             }
         }
 
+        if (tunnel != null) {
+            // Before the self-test: on a server with tunnel-mux, port 2010 is only served once
+            // the server has reached this proxy through one of our forwards.
+            tunnel.settleForwarding();
+        }
         return selfTest();
     }
 
@@ -1590,7 +1622,7 @@ boolean selfTest() {
 
     /** Package-private so a test can drive the retry without a live SSH session. */
     boolean tunnelIsForwarding() {
-        return tunnel != null && tunnel.isForwardingEstablished();
+        return tunnel != null && tunnel.isAnyConnectionForwarding();
     }
 
     /**
@@ -2403,6 +2435,18 @@ static Integer positiveSeconds(CommandLine commandLine, String option) throws Pa
      * up quickly, a tunnelled connection should not -- and one number for both would be worse
      * than either.
      */
+    public int getSshConnections() {
+        return sshConnections;
+    }
+
+    public void setSshConnections(int sshConnections) {
+        this.sshConnections = sshConnections;
+    }
+
+    public boolean isNoProxy() {
+        return noProxy;
+    }
+
     public Integer getHttpDialTimeoutSeconds() {
         return httpDialTimeoutSeconds;
     }

src/main/java/ssh/CustomConnectionMonitor.java

@@ -24,14 +24,19 @@ public class CustomConnectionMonitor {
 
     private static final Logger LOG = Logger.getLogger(CustomConnectionMonitor.class.getName());
 
-    static final long CURRENT_RETRY_DELAY = 5000;
-    static final int MAX_RETRIES = 30;
+    /**
+     * Between reconnect attempts while the network is down: the first one that can get through
+     * should not wait long once it is back (5 s -> 2 s took 2010's return after a 20-second outage
+     * from 25.1 s to 23.6 s). MAX_RETRIES keeps the ~150 s before the tunnel is rebuilt.
+     */
+    static final long CURRENT_RETRY_DELAY = 2000;
+    static final int MAX_RETRIES = 75;
     /**
      * The first reconnect goes out after this long, not after the full retry delay: the session
      * has already been gone for the keepalive's ~15 seconds, and every second more is a second
      * of failing tests.
      */
-    static final long FIRST_RETRY_DELAY = 1000;
+    static final long FIRST_RETRY_DELAY = 250;
     /** Between re-requests of the reverse forward on a session that is up. */
     static final long FORWARD_RETRY_DELAY = 2000;
     /** About two minutes of re-requests before the tunnel is rebuilt. */