Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 17 additions & 9 deletions crates/openshell-server/src/grpc/sandbox.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2882,7 +2882,7 @@ pub(super) async fn handle_forward_tcp(
// The main process may finish between minting the SSH token and opening
// its transport. Keep the relay reachable until terminal delivery is
// finalized so fast commands can attach without a readiness race.
if !sandbox_relay_reachable(state, &sandbox) {
if !sandbox_relay_reachable(state, &sandbox).await? {
return Err(Status::failed_precondition("sandbox is not ready"));
}

Expand Down Expand Up @@ -3430,11 +3430,19 @@ pub(super) async fn handle_exec_sandbox_interactive_start(
// SSH session handlers
// ---------------------------------------------------------------------------

fn sandbox_relay_reachable(state: &ServerState, sandbox: &Sandbox) -> bool {
let phase = SandboxPhase::try_from(sandbox.phase()).ok();
matches!(phase, Some(SandboxPhase::Ready))
|| (matches!(phase, Some(SandboxPhase::Completed | SandboxPhase::Error))
&& state.supervisor_sessions.has_session(sandbox.object_id()))
/// Whether a relay into the sandbox can open: always when `Ready`, and after
/// the main process exits while its supervisor is connected to any replica.
async fn sandbox_relay_reachable(
state: &Arc<ServerState>,
sandbox: &Sandbox,
) -> Result<bool, Status> {
match SandboxPhase::try_from(sandbox.phase()).ok() {
Some(SandboxPhase::Ready) => Ok(true),
Some(SandboxPhase::Completed | SandboxPhase::Error) => {
crate::supervisor_session::supervisor_connected(state, sandbox.object_id()).await
}
_ => Ok(false),
}
}

pub(super) async fn handle_create_ssh_session(
Expand All @@ -3453,7 +3461,7 @@ pub(super) async fn handle_create_ssh_session(
.await?;
let sandbox_id = sandbox.object_id().to_string();

if !sandbox_relay_reachable(state, &sandbox) {
if !sandbox_relay_reachable(state, &sandbox).await? {
return Err(Status::failed_precondition("sandbox is not ready"));
}

Expand Down Expand Up @@ -9660,10 +9668,10 @@ mod tests {
.supervisor_sessions
.finalize_main_process_exit("sandbox-work")
);
assert!(sandbox_relay_reachable(&state, &sandbox));
assert!(sandbox_relay_reachable(&state, &sandbox).await.unwrap());

assert!(state.supervisor_sessions.disconnect("sandbox-work"));
assert!(!sandbox_relay_reachable(&state, &sandbox));
assert!(!sandbox_relay_reachable(&state, &sandbox).await.unwrap());
}

#[tokio::test]
Expand Down
80 changes: 80 additions & 0 deletions crates/openshell-server/src/supervisor_session.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1825,6 +1825,33 @@ fn owner_is_fresh(owner: &crate::supervisor_owner::OwnerRecord) -> bool {
owner.is_fresh(OWNER_TTL)
}

/// Whether this sandbox's supervisor is connected to any replica.
///
/// A peer replica counts only through a fresh owner record. An owner record
/// naming this replica without a local session is a leftover from a lost
/// session.
pub async fn supervisor_connected(
state: &Arc<ServerState>,
sandbox_id: &str,
) -> Result<bool, Status> {
if state.supervisor_sessions.has_session(sandbox_id) {
return Ok(true);
}
if state.store.is_single_replica() {
return Ok(false);
}
let owner_index = SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL);
let owner = resolve_owner(state, &owner_index, sandbox_id).await?;
Ok(peer_holds_session(&state.replica_id, owner.as_ref()))
}

fn peer_holds_session(
self_replica_id: &str,
owner: Option<&crate::supervisor_owner::OwnerRecord>,
) -> bool {
owner.is_some_and(|owner| owner_is_fresh(owner) && owner.owner_replica_id != self_replica_id)
}

/// The replica currently holding this sandbox's supervisor session, if known.
pub async fn owner_replica_id(state: &Arc<ServerState>, sandbox_id: &str) -> Option<String> {
if state.store.is_single_replica() {
Expand Down Expand Up @@ -3719,6 +3746,59 @@ mod tests {
.expect("complete chart peer TLS materials should configure tonic");
}

#[tokio::test]
async fn supervisor_connected_follows_the_local_session() {
let state = crate::grpc::test_support::test_server_state().await;
let connected = |sandbox_id: &'static str| {
let state = state.clone();
async move { supervisor_connected(&state, sandbox_id).await.unwrap() }
};

assert!(!connected("relay-none").await);

let (tx, _rx) = mpsc::channel(1);
let (shutdown_tx, _shutdown_rx) = oneshot::channel();
state.supervisor_sessions.register(
"relay-local".to_string(),
"session".to_string(),
tx,
shutdown_tx,
);
assert!(connected("relay-local").await);
assert!(state.supervisor_sessions.disconnect("relay-local"));
assert!(!connected("relay-local").await);

// A single replica holds every session itself, so an owner record
// naming a peer is a leftover and is not read.
assert!(state.store.is_single_replica());
SupervisorOwnerIndex::new(state.store.clone(), OWNER_TTL)
.publish(
"relay-peer",
"session",
"supervisor",
1,
"peer-replica",
"https://peer",
)
.await
.unwrap();
assert!(!connected("relay-peer").await);
}

#[test]
fn peer_holds_session_only_through_a_fresh_peer_owner_record() {
assert!(peer_holds_session("gw-0", Some(&owner_record("gw-1"))));
assert!(!peer_holds_session("gw-0", None));

let mut stale = owner_record("gw-1");
stale.updated_at_ms = 0;
assert!(!peer_holds_session("gw-0", Some(&stale)));

// An owner record naming this replica without a local session is a
// leftover from a lost session, not a live supervisor.
assert!(!peer_holds_session("gw-0", Some(&owner_record("gw-0"))));
}

fn sandbox_record(id: &str, name: &str) -> Sandbox {
Sandbox {
metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta {
Expand Down
31 changes: 14 additions & 17 deletions tests/suites/features/host-gateway/tests/host_gateway.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ const PYTHON: &str = "python3";

/// Requests `/` from `host.openshell.internal` on the port in `argv[1]` and
/// prints the response. The server listens before the sandbox is created,
/// and a Ready sandbox's network is up, so the client doesn't retry.
/// and the network is up before the main process starts, so the client
/// doesn't retry.
const CLIENT: &str = r#"
import socket, sys
Expand All @@ -45,25 +46,21 @@ async fn python_sandbox_reaches_host_server(cx: &mut Context) -> testkit::Result
let server = HttpRecorder::start(marker.clone()).await?;
let image = testkit::python_image();

// The client runs through `sandbox exec` in a Ready sandbox, not as the
// main process. A gateway with several replicas (k3s-ha) can refuse to
// attach to a main process that already finished with "sandbox is not
// ready" when the request reaches a replica that doesn't hold the
// sandbox's session.
let sandbox = cx
.sandbox("hg")
// The client runs as the main process. It usually finishes before the CLI
// attaches, so on a gateway with several replicas (k3s-ha) this also
// covers attaching to a finished main process through a replica that
// doesn't hold the sandbox's supervisor session.
let client = cx
.cli_sandbox("hg")
.image(image.clone())
.policy(policy(server.port))
.command(["sh", "-c", "exec sleep infinity"])
.create_ready()
.await
.context(format!("create a sandbox from {image}"))?;

let client = sandbox
.exec([PYTHON, "-c", CLIENT, &server.port.to_string()])
.policy_yaml(policy::to_yaml(&policy(server.port))?)
.command([PYTHON, "-c", CLIENT, &server.port.to_string()])
.run()
.await?
.success()
.context("run the client in the sandbox")?;
.context(format!(
"run the client as the main process of a sandbox from {image}"
))?;
testkit::ensure!(
client.stdout.contains(&marker),
"the client did not get the host server's response:\n{}{}",
Expand Down
Loading