From dac3026d20ae2f186fda28c55a987bf539892808 Mon Sep 17 00:00:00 2001 From: Piotr Mlocek Date: Tue, 17 Nov 2026 12:46:46 -0800 Subject: [PATCH] fix(server): find a finished sandbox's supervisor on any replica After a retained sandbox's main process exits, CreateSshSession and ForwardTcp allow access only while the supervisor is connected. The check looked only at the receiving replica's session map, so on a multi-replica gateway a request that landed on another replica was refused with "sandbox is not ready". This broke `sandbox create -- ` when the command finished before the CLI attached. Check the supervisor owner record too, the same way relay opening already routes to the owning replica. Run the host-gateway client as the sandbox's main process again, so the k3s-ha row covers the attach. Signed-off-by: Piotr Mlocek --- crates/openshell-server/src/grpc/sandbox.rs | 26 +++--- .../src/supervisor_session.rs | 80 +++++++++++++++++++ .../host-gateway/tests/host_gateway.rs | 31 ++++--- 3 files changed, 111 insertions(+), 26 deletions(-) diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 72c20a6991..82c92ff28d 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -2882,7 +2882,7 @@ pub(super) async fn handle_forward_tcp( // The main process may finish between minting the SSH token and opening // its transport. Keep the relay reachable until terminal delivery is // finalized so fast commands can attach without a readiness race. - if !sandbox_relay_reachable(state, &sandbox) { + if !sandbox_relay_reachable(state, &sandbox).await? { return Err(Status::failed_precondition("sandbox is not ready")); } @@ -3430,11 +3430,19 @@ pub(super) async fn handle_exec_sandbox_interactive_start( // SSH session handlers // --------------------------------------------------------------------------- -fn sandbox_relay_reachable(state: &ServerState, sandbox: &Sandbox) -> bool { - let phase = SandboxPhase::try_from(sandbox.phase()).ok(); - matches!(phase, Some(SandboxPhase::Ready)) - || (matches!(phase, Some(SandboxPhase::Completed | SandboxPhase::Error)) - && state.supervisor_sessions.has_session(sandbox.object_id())) +/// Whether a relay into the sandbox can open: always when `Ready`, and after +/// the main process exits while its supervisor is connected to any replica. +async fn sandbox_relay_reachable( + state: &Arc, + sandbox: &Sandbox, +) -> Result { + match SandboxPhase::try_from(sandbox.phase()).ok() { + Some(SandboxPhase::Ready) => Ok(true), + Some(SandboxPhase::Completed | SandboxPhase::Error) => { + crate::supervisor_session::supervisor_connected(state, sandbox.object_id()).await + } + _ => Ok(false), + } } pub(super) async fn handle_create_ssh_session( @@ -3453,7 +3461,7 @@ pub(super) async fn handle_create_ssh_session( .await?; let sandbox_id = sandbox.object_id().to_string(); - if !sandbox_relay_reachable(state, &sandbox) { + if !sandbox_relay_reachable(state, &sandbox).await? { return Err(Status::failed_precondition("sandbox is not ready")); } @@ -9660,10 +9668,10 @@ mod tests { .supervisor_sessions .finalize_main_process_exit("sandbox-work") ); - assert!(sandbox_relay_reachable(&state, &sandbox)); + assert!(sandbox_relay_reachable(&state, &sandbox).await.unwrap()); assert!(state.supervisor_sessions.disconnect("sandbox-work")); - assert!(!sandbox_relay_reachable(&state, &sandbox)); + assert!(!sandbox_relay_reachable(&state, &sandbox).await.unwrap()); } #[tokio::test] diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index 9520fc106a..92a8ed6222 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -1825,6 +1825,33 @@ fn owner_is_fresh(owner: &crate::supervisor_owner::OwnerRecord) -> bool { owner.is_fresh(OWNER_TTL) } +/// Whether this sandbox's supervisor is connected to any replica. +/// +/// A peer replica counts only through a fresh owner record. An owner record +/// naming this replica without a local session is a leftover from a lost +/// session. +pub async fn supervisor_connected( + state: &Arc, + sandbox_id: &str, +) -> Result { + if state.supervisor_sessions.has_session(sandbox_id) { + return Ok(true); + } + if state.store.is_single_replica() { + return Ok(false); + } + let owner_index = SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL); + let owner = resolve_owner(state, &owner_index, sandbox_id).await?; + Ok(peer_holds_session(&state.replica_id, owner.as_ref())) +} + +fn peer_holds_session( + self_replica_id: &str, + owner: Option<&crate::supervisor_owner::OwnerRecord>, +) -> bool { + owner.is_some_and(|owner| owner_is_fresh(owner) && owner.owner_replica_id != self_replica_id) +} + /// The replica currently holding this sandbox's supervisor session, if known. pub async fn owner_replica_id(state: &Arc, sandbox_id: &str) -> Option { if state.store.is_single_replica() { @@ -3719,6 +3746,59 @@ mod tests { .expect("complete chart peer TLS materials should configure tonic"); } + #[tokio::test] + async fn supervisor_connected_follows_the_local_session() { + let state = crate::grpc::test_support::test_server_state().await; + let connected = |sandbox_id: &'static str| { + let state = state.clone(); + async move { supervisor_connected(&state, sandbox_id).await.unwrap() } + }; + + assert!(!connected("relay-none").await); + + let (tx, _rx) = mpsc::channel(1); + let (shutdown_tx, _shutdown_rx) = oneshot::channel(); + state.supervisor_sessions.register( + "relay-local".to_string(), + "session".to_string(), + tx, + shutdown_tx, + ); + assert!(connected("relay-local").await); + assert!(state.supervisor_sessions.disconnect("relay-local")); + assert!(!connected("relay-local").await); + + // A single replica holds every session itself, so an owner record + // naming a peer is a leftover and is not read. + assert!(state.store.is_single_replica()); + SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL) + .publish( + "relay-peer", + "session", + "supervisor", + 1, + "peer-replica", + "https://peer", + ) + .await + .unwrap(); + assert!(!connected("relay-peer").await); + } + + #[test] + fn peer_holds_session_only_through_a_fresh_peer_owner_record() { + assert!(peer_holds_session("gw-0", Some(&owner_record("gw-1")))); + assert!(!peer_holds_session("gw-0", None)); + + let mut stale = owner_record("gw-1"); + stale.updated_at_ms = 0; + assert!(!peer_holds_session("gw-0", Some(&stale))); + + // An owner record naming this replica without a local session is a + // leftover from a lost session, not a live supervisor. + assert!(!peer_holds_session("gw-0", Some(&owner_record("gw-0")))); + } + fn sandbox_record(id: &str, name: &str) -> Sandbox { Sandbox { metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { diff --git a/tests/suites/features/host-gateway/tests/host_gateway.rs b/tests/suites/features/host-gateway/tests/host_gateway.rs index 7d285116e4..6c864b0e01 100644 --- a/tests/suites/features/host-gateway/tests/host_gateway.rs +++ b/tests/suites/features/host-gateway/tests/host_gateway.rs @@ -22,7 +22,8 @@ const PYTHON: &str = "python3"; /// Requests `/` from `host.openshell.internal` on the port in `argv[1]` and /// prints the response. The server listens before the sandbox is created, -/// and a Ready sandbox's network is up, so the client doesn't retry. +/// and the network is up before the main process starts, so the client +/// doesn't retry. const CLIENT: &str = r#" import socket, sys @@ -45,25 +46,21 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result let server = HttpRecorder::start(marker.clone()).await?; let image = testkit::python_image(); - // The client runs through `sandbox exec` in a Ready sandbox, not as the - // main process. A gateway with several replicas (k3s-ha) can refuse to - // attach to a main process that already finished with "sandbox is not - // ready" when the request reaches a replica that doesn't hold the - // sandbox's session. - let sandbox = cx - .sandbox("hg") + // The client runs as the main process. It usually finishes before the CLI + // attaches, so on a gateway with several replicas (k3s-ha) this also + // covers attaching to a finished main process through a replica that + // doesn't hold the sandbox's supervisor session. + let client = cx + .cli_sandbox("hg") .image(image.clone()) - .policy(policy(server.port)) - .command(["sh", "-c", "exec sleep infinity"]) - .create_ready() - .await - .context(format!("create a sandbox from {image}"))?; - - let client = sandbox - .exec([PYTHON, "-c", CLIENT, &server.port.to_string()]) + .policy_yaml(policy::to_yaml(&policy(server.port))?) + .command([PYTHON, "-c", CLIENT, &server.port.to_string()]) + .run() .await? .success() - .context("run the client in the sandbox")?; + .context(format!( + "run the client as the main process of a sandbox from {image}" + ))?; testkit::ensure!( client.stdout.contains(&marker), "the client did not get the host server's response:\n{}{}",