From 9c3207051f34bf54e37ddd319841b0dfc8ec5f9e Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:37:43 -0800 Subject: [PATCH 01/15] test(testkit): add compute_driver, driver_config, sandbox logs, and policy set helpers Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 1 + .../conformance/src/credential_gating.rs | 26 +- tests/suites/drivers/podman/Cargo.lock | 1 + tests/suites/features/Cargo.lock | 1 + tests/support/Cargo.lock | 1 + tests/testkit/Cargo.lock | 1 + tests/testkit/README.md | 13 +- tests/testkit/testkit/Cargo.toml | 1 + tests/testkit/testkit/src/context.rs | 264 +++++++++++++----- tests/testkit/testkit/src/lib.rs | 2 + tests/testkit/testkit/src/logs.rs | 175 ++++++++++++ tests/testkit/testkit/src/policy.rs | 43 ++- 12 files changed, 438 insertions(+), 91 deletions(-) create mode 100644 tests/testkit/testkit/src/logs.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 855f0ccac8..90334398f6 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2273,6 +2273,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 2d67926ef0..48bd9eb531 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -13,9 +13,7 @@ use std::time::Duration; use host_server_support::{HttpRecorder, Request, WebSocketRecorder}; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; -use testkit::{ - Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox, YamlFile, -}; +use testkit::{Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; const TEST_HOST: &str = "host.openshell.internal"; const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; @@ -208,12 +206,12 @@ async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> t let sandbox = create_sandbox(cx, "live", unbound.clone(), &[&provider]).await?; wait_for_token(&sandbox, "TOKEN_ABSENT").await?; - set_policy(cx, &sandbox, &bound) + policy::set(&sandbox, &bound) .await .context("bind the provider")?; wait_for_token(&sandbox, "TOKEN_PLACEHOLDER").await?; - set_policy(cx, &sandbox, &unbound) + policy::set(&sandbox, &unbound) .await .context("unbind the provider")?; wait_for_token(&sandbox, "TOKEN_ABSENT").await @@ -393,21 +391,3 @@ async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { Ok(()) } -/// Replace `sandbox`'s policy with `policy` and wait until it is applied. -async fn set_policy(cx: &Context, sandbox: &Sandbox, policy: &proto::SandboxPolicy) -> Result { - let file = YamlFile::new(&policy::to_yaml(policy)?)?; - cx.cli([ - "policy", - "set", - sandbox.name(), - "--policy", - file.path()?, - "--wait", - "--timeout", - "120", - ]) - .await? - .success() - .context("policy set")?; - Ok(()) -} diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index a4c8e1127a..e62158799b 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2261,6 +2261,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index aaada20bb6..c836d8f9ce 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2334,6 +2334,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 6525cfc070..69fecca143 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -2279,6 +2279,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index ccc512b2d5..77f16bea16 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -2252,6 +2252,7 @@ dependencies = [ "openshell-policy", "openshell-sdk", "openshell-testkit-macros", + "prost-types", "quick-junit", "rand 0.9.4", "serde", diff --git a/tests/testkit/README.md b/tests/testkit/README.md index fc8ddce497..744fc75c3d 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -65,10 +65,12 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `cx.sandbox(suffix)` to build a sandbox with a run-unique name, then `create()` it or `create_ready()` to also wait for Ready. Set its image - with `image(..)` and its policy with `policy(..)`. The returned + with `image(..)`, its policy with `policy(..)`, and the template's + driver-keyed `driver_config(json)`. The returned `Sandbox` can wait for ready or finished, stop, start, exec or `poll_exec` until the output matches, read its config and draft policy, set - a setting, read `sandbox get` JSON with `get_json()`, and delete. Its + a setting, read `sandbox get` JSON with `get_json()`, read its supervisor + logs with `logs()` or wait for a line with `logs_until(..)`, and delete. Its `expect_listed`, `expect_absent` and `wait_absent` check list and get. Exec output has `success()`, `failure()` and `expect_stdout()`. - `cx.cli_sandbox(suffix)` for a sandbox only `openshell sandbox create` @@ -80,7 +82,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. create, so a create that fails after the gateway accepted it is still cleaned up. `cx.adopt(name)` returns a `Sandbox` handle for one the CLI created. - `cx.expect_compute_driver(name)` to fail unless the gateway runs only that - compute driver. Driver suites call it first. + compute driver. Driver suites call it first. `cx.compute_driver()` returns + the driver's name, for setup that depends on it, such as a driver config. - `testkit::base_image()` for the image to create sandboxes from or build on: `OPENSHELL_TEST_BASE_IMAGE`, or OpenShell's default sandbox image. - `testkit::python_image()` for sandboxes that run `python3`, curl, git, or @@ -108,7 +111,9 @@ fails. A killed test process skips cleanup, so names are unique to the run. (binaries apply to every endpoint under `name`) that `policy::Endpoint` configures (`.rest(Access::Full)`, `.credential_binding(provider)`, ...). Pass the result to `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` - for the CLI. + for the CLI. `policy::from_yaml` reads a policy file, and + `policy::set(&sandbox, &policy)` replaces a running sandbox's policy and + waits until it is loaded. `testkit::PYTHON_BINARIES` names `python3` in `python_image()` images. - `cx.import_profile(profile)` to import a provider profile and return its `id`, and `cx.create_provider(name, type, credentials)` to create a diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index 7b8ebd90ab..d216c7b824 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -24,6 +24,7 @@ clap = { version = "4", features = ["derive"] } futures = "0.3" libtest-mimic = "0.8" linkme = "0.3" +prost-types = "0.14" quick-junit = "0.8" rand = "0.9" serde = { version = "1", features = ["derive"] } diff --git a/tests/testkit/testkit/src/context.rs b/tests/testkit/testkit/src/context.rs index 1aa4bdd377..8c5a3bd626 100644 --- a/tests/testkit/testkit/src/context.rs +++ b/tests/testkit/testkit/src/context.rs @@ -130,6 +130,7 @@ impl Context { SandboxBuilder { handles: self.handles.clone(), policy: None, + driver_config: None, spec: SandboxSpec { name: Some(self.name(suffix)), ..SandboxSpec::default() @@ -203,6 +204,18 @@ impl Context { /// gateway info reports it. Driver suites check this first, so a suite /// pointed at another driver's gateway fails instead of testing it. pub async fn expect_compute_driver(&self, name: &str) -> Result { + let driver = self.compute_driver().await?; + crate::ensure!( + driver == name, + "the gateway runs compute driver {driver}; expected {name}" + ); + Ok(()) + } + + /// The gateway's compute driver, such as `docker` or `kubernetes`, for + /// tests whose setup depends on it. Fails unless gateway info reports + /// exactly one driver, under the same configured and driver name. + pub async fn compute_driver(&self) -> Result { let info = self .handles .grpc @@ -212,22 +225,7 @@ impl Context { .await .context("get gateway info")? .into_inner(); - let drivers: Vec<(&str, &str)> = info - .compute_drivers - .iter() - .map(|driver| { - let reported = driver - .capabilities - .as_ref() - .map_or("", |capabilities| capabilities.driver_name.as_str()); - (driver.name.as_str(), reported) - }) - .collect(); - crate::ensure!( - matches!(drivers.as_slice(), [(configured, reported)] if *configured == name && *reported == name), - "the gateway reports compute drivers (name, driver name) {drivers:?}; expected only {name}" - ); - Ok(()) + single_compute_driver(&info) } /// Run the cleanup hooks, including deleting the sandboxes the test @@ -238,6 +236,30 @@ impl Context { } } +/// The one compute driver `info` reports, when its configured name and the +/// name the driver reports agree. +fn single_compute_driver(info: &proto::GetGatewayInfoResponse) -> Result { + let drivers: Vec<(&str, &str)> = info + .compute_drivers + .iter() + .map(|driver| { + let reported = driver + .capabilities + .as_ref() + .map_or("", |capabilities| capabilities.driver_name.as_str()); + (driver.name.as_str(), reported) + }) + .collect(); + match drivers.as_slice() { + [(configured, reported)] if configured == reported && !configured.is_empty() => { + Ok((*configured).to_string()) + } + _ => Err(Error::msg(format!( + "the gateway reports compute drivers (name, driver name) {drivers:?}; expected exactly one" + ))), + } +} + /// Register the cleanup hook that deletes sandbox `name`, unless it is /// already tracked. fn track_sandbox(cleanup: &Cleanup, client: &WorkspaceScopedClient, name: &str) { @@ -265,17 +287,19 @@ fn track_sandbox(cleanup: &Cleanup, client: &WorkspaceScopedClient, name: &str) /// The generated gRPC client, for public API the SDK does not wrap yet. #[derive(Clone)] -struct Grpc { +pub(crate) struct Grpc { client: OpenShellClient, workspace: String, } impl Grpc { - async fn client(&self) -> Result { + pub(crate) async fn client(&self) -> Result { Ok(self.client.raw_grpc_fresh().await?) } - fn scope(&self) -> Option { + pub(crate) fn scope( + &self, + ) -> Option { Some(proto::workspace_selector(&self.workspace)) } } @@ -295,6 +319,10 @@ impl Handles { &self.cli } + pub(crate) fn grpc(&self) -> &Grpc { + &self.grpc + } + pub(crate) fn track(&self, name: &str) { track_sandbox(&self.cleanup, &self.client, name); } @@ -319,6 +347,7 @@ impl Handles { pub struct SandboxBuilder { handles: Handles, policy: Option, + driver_config: Option, spec: SandboxSpec, } @@ -352,6 +381,20 @@ impl SandboxBuilder { self } + /// Set the template's `driver_config`, a JSON object keyed by driver + /// name, as `sandbox create --driver-config-json` does. The gateway + /// passes the block for its driver to the driver. + /// + /// ```ignore + /// let driver = cx.compute_driver().await?; + /// cx.sandbox("mount") + /// .driver_config(serde_json::json!({ driver: { "mounts": [] } })) + /// ``` + pub fn driver_config(mut self, config: serde_json::Value) -> Self { + self.driver_config = Some(config); + self + } + /// Full access to the spec for fields without a builder method. pub fn spec(mut self, update: impl FnOnce(&mut SandboxSpec)) -> Self { update(&mut self.spec); @@ -371,36 +414,40 @@ impl SandboxBuilder { pub async fn create(self) -> Result { let name = self.spec.name.clone().unwrap_or_default(); self.handles.track(&name); - let id = match self.policy { - None => { - self.handles - .client - .create_sandbox(self.spec) - .await - .context(format!("create sandbox {name}"))? - .id - } - // The SDK's spec has no policy field, so create through gRPC. - Some(policy) => { - crate::ensure!( - !self.spec.gpu && self.spec.service_exposures.is_empty(), - "testkit doesn't create a sandbox with a policy and a GPU or service exposures" - ); - let request = policy_create_request(self.spec, policy, self.handles.grpc.scope()); - self.handles - .grpc - .client() - .await? - .create_sandbox(request) - .await - .context(format!("create sandbox {name}"))?; - self.handles - .client - .get_sandbox(&name) - .await - .context(format!("get sandbox {name}"))? - .id - } + let id = if self.policy.is_none() && self.driver_config.is_none() { + self.handles + .client + .create_sandbox(self.spec) + .await + .context(format!("create sandbox {name}"))? + .id + } else { + // The SDK's spec has no policy or driver config, so create + // through gRPC. + crate::ensure!( + !self.spec.gpu && self.spec.service_exposures.is_empty(), + "testkit doesn't create a sandbox with a policy or driver config and a GPU or service exposures" + ); + let driver_config = self.driver_config.map(driver_config_struct).transpose()?; + let request = grpc_create_request( + self.spec, + self.policy, + driver_config, + self.handles.grpc.scope(), + ); + self.handles + .grpc + .client() + .await? + .create_sandbox(request) + .await + .context(format!("create sandbox {name}"))?; + self.handles + .client + .get_sandbox(&name) + .await + .context(format!("get sandbox {name}"))? + .id }; Ok(Sandbox { handles: self.handles, @@ -410,15 +457,28 @@ impl SandboxBuilder { } } -/// The gRPC request for a sandbox created from `spec` with `policy`, built -/// as the SDK builds one from `spec` alone. The spec is destructured without -/// `..`, so a field added to the SDK's spec fails to compile here instead of -/// being dropped. [`SandboxBuilder::create`] rejects a GPU or service -/// exposures before calling this. A policy field on the SDK's spec would -/// replace this (#5410). -fn policy_create_request( +/// `config` as the protobuf `Struct` the template carries. It must be a JSON +/// object, keyed by driver name. +fn driver_config_struct(config: serde_json::Value) -> Result { + let serde_json::Value::Object(fields) = config else { + return Err(Error::msg(format!( + "driver config must be a JSON object keyed by driver name; got {config}" + ))); + }; + openshell_core::proto_struct::json_object_to_struct(fields) + .map_err(|error| Error::msg(format!("driver config can't be represented: {error}"))) +} + +/// The gRPC request for a sandbox created from `spec` with `policy` and +/// `driver_config`, built as the SDK builds one from `spec` alone. The spec +/// is destructured without `..`, so a field added to the SDK's spec fails to +/// compile here instead of being dropped. [`SandboxBuilder::create`] rejects +/// a GPU or service exposures before calling this. Policy and driver config +/// fields on the SDK's spec would replace this (#5410). +fn grpc_create_request( spec: SandboxSpec, - policy: proto::SandboxPolicy, + policy: Option, + driver_config: Option, workspace_scope: Option, ) -> proto::CreateSandboxRequest { let SandboxSpec { @@ -439,14 +499,17 @@ fn policy_create_request( workspace_scope, spec: Some(proto::SandboxSpec { environment, - template: image.map(|image| proto::SandboxTemplate { - image, - ..proto::SandboxTemplate::default() + template: (image.is_some() || driver_config.is_some()).then(|| { + proto::SandboxTemplate { + image: image.unwrap_or_default(), + driver_config, + ..proto::SandboxTemplate::default() + } }), providers, command, tty, - policy: Some(policy), + policy, restart_policy: proto::SandboxRestartPolicy::from(restart_policy) as i32, ..proto::SandboxSpec::default() }), @@ -506,6 +569,10 @@ impl Sandbox { &self.name } + pub(crate) fn handles(&self) -> &Handles { + &self.handles + } + pub fn id(&self) -> &str { &self.id } @@ -831,12 +898,83 @@ mod tests { }; let policy = crate::policy::base(); - let spec = policy_create_request(spec, policy.clone(), None) + let spec = grpc_create_request(spec, Some(policy.clone()), None, None) .spec .expect("spec"); - assert_eq!(spec.template.expect("template").image, "python:3"); + let template = spec.template.expect("template"); + assert_eq!(template.image, "python:3"); + assert_eq!(template.driver_config, None); assert_eq!(spec.providers, ["provider"]); assert_eq!(spec.policy, Some(policy)); } + + #[test] + fn grpc_create_request_carries_driver_config_without_an_image() { + let config = driver_config_struct(serde_json::json!({ + "docker": { "mounts": [{ "type": "volume", "source": "data" }] } + })) + .expect("an object converts"); + + let spec = grpc_create_request(SandboxSpec::default(), None, Some(config.clone()), None) + .spec + .expect("spec"); + + let template = spec.template.expect("a template for the driver config"); + assert_eq!(template.image, "", "an empty image selects the default"); + assert_eq!(template.driver_config, Some(config)); + assert_eq!(spec.policy, None); + } + + #[test] + fn grpc_create_request_omits_an_empty_template() { + let spec = grpc_create_request(SandboxSpec::default(), None, None, None) + .spec + .expect("spec"); + assert_eq!(spec.template, None); + } + + #[test] + fn driver_config_must_be_an_object() { + let error = driver_config_struct(serde_json::json!(["docker"])).expect_err("an array"); + assert!( + error.to_string().contains("keyed by driver name"), + "{error}" + ); + } + + fn gateway_info(drivers: &[(&str, &str)]) -> proto::GetGatewayInfoResponse { + proto::GetGatewayInfoResponse { + compute_drivers: drivers + .iter() + .map(|(name, reported)| proto::ComputeDriverInfo { + name: (*name).to_string(), + capabilities: Some(proto::ComputeDriverCapabilities { + driver_name: (*reported).to_string(), + ..proto::ComputeDriverCapabilities::default() + }), + }) + .collect(), + ..proto::GetGatewayInfoResponse::default() + } + } + + #[test] + fn compute_driver_is_the_single_agreeing_driver() { + assert_eq!( + single_compute_driver(&gateway_info(&[("docker", "docker")])).expect("one driver"), + "docker" + ); + for drivers in [ + &[][..], + &[("docker", "podman")][..], + &[("", "")][..], + &[("docker", "docker"), ("podman", "podman")][..], + ] { + assert!( + single_compute_driver(&gateway_info(drivers)).is_err(), + "{drivers:?} should fail" + ); + } + } } diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 7c233ce834..ee1c5d872d 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -34,6 +34,7 @@ mod context; mod error; pub mod gateway; mod image; +mod logs; mod output; pub mod policy; mod poll; @@ -55,6 +56,7 @@ pub use context::{ pub use error::{Error, Result, ResultExt}; pub use gateway::Gateway; pub use image::{PYTHON_BINARIES, base_image, python_image}; +pub use logs::{LOGS_TIMEOUT, Logs}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until, with_deadline}; pub use program::main; diff --git a/tests/testkit/testkit/src/logs.rs b/tests/testkit/testkit/src/logs.rs new file mode 100644 index 0000000000..ce486fcc5b --- /dev/null +++ b/tests/testkit/testkit/src/logs.rs @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A sandbox's supervisor logs, as `openshell logs --source sandbox` shows +//! them. + +use std::fmt; +use std::time::Duration; + +use openshell_sdk::raw::proto; + +use crate::context::Sandbox; +use crate::error::{Result, ResultExt as _}; +use crate::poll::{Poll, poll_until}; + +/// The most lines [`Sandbox::logs`] returns, as `openshell logs -n 500`. +const LINES: u32 = 500; +/// How long [`Sandbox::logs_until`] waits. The supervisor pushes its logs to +/// the gateway in batches, so a line can appear a few seconds after the event. +pub const LOGS_TIMEOUT: Duration = Duration::from_secs(10); + +/// Log lines from a sandbox's supervisor, oldest first. +#[derive(Debug, Clone, Default)] +pub struct Logs { + lines: Vec, +} + +impl Logs { + pub fn new(lines: Vec) -> Self { + Self { lines } + } + + pub fn lines(&self) -> &[proto::SandboxLogLine] { + &self.lines + } + + /// Whether a line contains `needle` in its level, target, message, or a + /// field name or value, as `openshell logs` prints the line. + pub fn contains(&self, needle: &str) -> bool { + self.lines + .iter() + .any(|line| Line(line).to_string().contains(needle)) + } +} + +/// One line per log line, as `openshell logs` prints them. +impl fmt::Display for Logs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + for line in &self.lines { + writeln!(f, "{}", Line(line))?; + } + Ok(()) + } +} + +/// `[.] [] [] [] `, +/// with fields sorted by name. +struct Line<'a>(&'a proto::SandboxLogLine); + +impl fmt::Display for Line<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let line = self.0; + let source = if line.source.is_empty() { + "gateway" + } else { + &line.source + }; + let (secs, millis) = line.event_time.as_ref().map_or((0, 0), |time| { + (time.seconds, time.nanos.clamp(0, 999_999_999) / 1_000_000) + }); + write!( + f, + "[{secs}.{millis:03}] [{source:<7}] [{:<5}] [{}] {}", + line.level, line.target, line.message + )?; + let mut fields: Vec<_> = line.fields.iter().collect(); + fields.sort_by_key(|(key, _)| key.as_str()); + for (key, value) in fields { + write!(f, " {key}={value}")?; + } + Ok(()) + } +} + +impl Sandbox { + /// The last 500 lines the sandbox's supervisor logged, as `openshell logs + /// -n 500 --source sandbox` returns them. + pub async fn logs(&self) -> Result { + let grpc = self.handles().grpc(); + let request = proto::GetSandboxLogsRequest { + sandbox: self.name().to_string(), + workspace_scope: grpc.scope(), + lines: LINES, + sources: vec!["sandbox".to_string()], + ..proto::GetSandboxLogsRequest::default() + }; + let response = grpc + .client() + .await? + .get_sandbox_logs(request) + .await + .context(format!("get logs of sandbox {}", self.name()))? + .into_inner(); + Ok(Logs::new(response.logs)) + } + + /// Read [`Sandbox::logs`] every 250 ms until `ready` accepts them, for up + /// to [`LOGS_TIMEOUT`]. A timeout reports the last logs read. + pub async fn logs_until(&self, what: &str, ready: impl Fn(&Logs) -> bool) -> Result { + poll_until( + &format!("sandbox {} to log {what}", self.name()), + LOGS_TIMEOUT, + Duration::from_millis(250), + async || { + let logs = self.logs().await?; + Ok(if ready(&logs) { + Poll::Ready(logs) + } else { + Poll::Pending(format!("logs:\n{logs}")) + }) + }, + ) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn line(message: &str, source: &str, fields: &[(&str, &str)]) -> proto::SandboxLogLine { + proto::SandboxLogLine { + event_time: Some(prost_types::Timestamp { + seconds: 1234, + nanos: 567_000_000, + }), + level: "INFO".to_string(), + target: "openshell_supervisor".to_string(), + message: message.to_string(), + source: source.to_string(), + fields: fields + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect(), + ..proto::SandboxLogLine::default() + } + } + + #[test] + fn renders_lines_like_the_cli() { + let logs = Logs::new(vec![ + line("hello world", "sandbox", &[]), + line("denied", "", &[("dst_port", "443"), ("action", "deny")]), + ]); + assert_eq!( + logs.to_string(), + "[1234.567] [sandbox] [INFO ] [openshell_supervisor] hello world\n\ + [1234.567] [gateway] [INFO ] [openshell_supervisor] denied action=deny dst_port=443\n" + ); + } + + #[test] + fn contains_searches_message_target_and_fields_per_line() { + let logs = Logs::new(vec![ + line("first", "sandbox", &[("event", "endpoint_mismatch")]), + line("second", "sandbox", &[]), + ]); + assert!(logs.contains("first")); + assert!(logs.contains("openshell_supervisor")); + assert!(logs.contains("event=endpoint_mismatch")); + assert!(logs.contains("endpoint_mismatch")); + assert!(!logs.contains("first\n"), "a needle doesn't span lines"); + assert!(!logs.contains("absent")); + } +} diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index ab7be7cae3..ccaeba71d8 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -3,7 +3,7 @@ //! Sandbox policies for tests: [`base`], read from `base-policy.yaml`, plus //! [`allow`] and the [`Endpoint`] settings for the network access a test -//! needs. +//! needs. [`set`] applies a policy to a running sandbox. //! //! ```ignore //! let mut policy = testkit::policy::base(); @@ -14,7 +14,9 @@ use openshell_sdk::raw::proto; +use crate::context::Sandbox; use crate::error::{Error, Result, ResultExt as _}; +use crate::yaml_file::YamlFile; pub use proto::NetworkAccessPreset as Access; @@ -75,6 +77,38 @@ pub fn to_yaml(policy: &proto::SandboxPolicy) -> Result { .context("serialize the policy") } +/// The policy `yaml` describes, as `sandbox create --policy` reads a file. +/// Use it for a policy kept as a YAML file, such as an example; build one +/// with [`base`] and [`allow`] otherwise. +pub fn from_yaml(yaml: &str) -> Result { + openshell_policy::parse_sandbox_policy(yaml) + .map_err(|error| Error::msg(format!("{error:?}"))) + .context("parse the policy") +} + +/// Replace `sandbox`'s policy with `policy` through `policy set --wait`, and +/// wait up to two minutes until the sandbox has loaded it. +pub async fn set(sandbox: &Sandbox, policy: &proto::SandboxPolicy) -> Result { + let file = YamlFile::new(&to_yaml(policy)?)?; + sandbox + .handles() + .cli() + .run([ + "policy", + "set", + sandbox.name(), + "--policy", + file.path()?, + "--wait", + "--timeout", + "120", + ]) + .await? + .success() + .context(format!("set the policy of sandbox {}", sandbox.name()))?; + Ok(()) +} + /// Settings for an endpoint from [`allow`], named as in policy YAML. pub trait Endpoint { /// `protocol: rest` with `access`. @@ -184,5 +218,12 @@ mod tests { openshell_policy::parse_sandbox_policy(&to_yaml(&policy).expect("serializes")) .expect("the serialized policy parses"); assert_eq!(round_trip, expected); + assert_eq!(from_yaml(&yaml).expect("the YAML parses"), expected); + } + + #[test] + fn from_yaml_reports_an_invalid_policy() { + let error = from_yaml("network_policies: [").expect_err("invalid YAML"); + assert!(error.to_string().contains("parse the policy"), "{error}"); } } From 8dd182cfe63f0b6588a4ad0febd7ad3af04d5c27 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:44:26 -0800 Subject: [PATCH 02/15] test(support): add handler responses, chunked bodies, and raw TCP servers to host-server Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 1 + tests/suites/features/Cargo.lock | 1 + tests/support/Cargo.lock | 1 + tests/support/host-server/Cargo.toml | 1 + tests/support/host-server/src/http.rs | 395 ++++++++++++++++++--- tests/support/host-server/src/lib.rs | 20 +- tests/support/host-server/src/tcp.rs | 174 +++++++++ tests/support/host-server/src/websocket.rs | 112 +++++- 8 files changed, 646 insertions(+), 59 deletions(-) create mode 100644 tests/support/host-server/src/tcp.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index 90334398f6..dbf9114e33 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -1207,6 +1207,7 @@ name = "host-server-support" version = "0.0.0" dependencies = [ "openshell-testkit", + "serde_json", "tokio", "tungstenite", ] diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index c836d8f9ce..d207a62b05 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1229,6 +1229,7 @@ name = "host-server-support" version = "0.0.0" dependencies = [ "openshell-testkit", + "serde_json", "tokio", "tungstenite", ] diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 69fecca143..dafc1d19e3 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -1204,6 +1204,7 @@ name = "host-server-support" version = "0.0.0" dependencies = [ "openshell-testkit", + "serde_json", "tokio", "tungstenite", ] diff --git a/tests/support/host-server/Cargo.toml b/tests/support/host-server/Cargo.toml index e7db63f53d..cf51fd6a4c 100644 --- a/tests/support/host-server/Cargo.toml +++ b/tests/support/host-server/Cargo.toml @@ -13,6 +13,7 @@ publish = false [dependencies] openshell-testkit = { path = "../../testkit/testkit" } +serde_json = "1" tokio = { version = "1.43", features = ["io-util", "net", "rt", "time"] } tungstenite = "0.26" diff --git a/tests/support/host-server/src/http.rs b/tests/support/host-server/src/http.rs index 4d8ff8c812..5aec7932ff 100644 --- a/tests/support/host-server/src/http.rs +++ b/tests/support/host-server/src/http.rs @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use std::fmt::Write as _; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use testkit::{Poll, Result}; @@ -10,7 +12,7 @@ use tokio::task::JoinHandle; use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, find, header}; -/// A request an [`HttpRecorder`] read. +/// A request an [`HttpRecorder`] read. A chunked body arrives decoded. #[derive(Clone, Debug)] pub struct Request { /// The request line and headers. @@ -18,26 +20,166 @@ pub struct Request { pub body: String, } -/// An HTTP server that answers every request with `200` and a fixed body, -/// and records each request it read. It reads a body by `Content-Length` -/// only, not a chunked one. +impl Request { + pub fn method(&self) -> &str { + self.request_line().0 + } + + /// The request target, such as `/path?query`. + pub fn path(&self) -> &str { + self.request_line().1 + } + + /// The value of the first header named `name`, in any case. + pub fn header(&self, name: &str) -> Option { + header(&self.head, name) + } + + fn request_line(&self) -> (&str, &str) { + let line = self.head.lines().next().unwrap_or_default(); + let mut parts = line.split(' '); + ( + parts.next().unwrap_or_default(), + parts.next().unwrap_or_default(), + ) + } +} + +/// A response an [`HttpRecorder`] handler returns. The server adds +/// `Content-Length`, except to an informational response, and `Connection`, +/// unless the handler set one. +#[derive(Clone, Debug)] +pub struct Response { + status: u16, + headers: Vec<(String, String)>, + body: Vec, + keep_alive: bool, +} + +impl Response { + /// A response with `status` and no body, that closes the connection. + pub fn new(status: u16) -> Self { + Self { + status, + headers: Vec::new(), + body: Vec::new(), + keep_alive: false, + } + } + + /// A `200` response with a `text/plain` body. + pub fn text(body: impl Into) -> Self { + Self::new(200) + .header("Content-Type", "text/plain") + .body(body.into()) + } + + /// A `200` response with an `application/json` body. + pub fn json(body: &serde_json::Value) -> Self { + Self::new(200) + .header("Content-Type", "application/json") + .body(body.to_string()) + } + + #[must_use] + pub fn status(mut self, status: u16) -> Self { + self.status = status; + self + } + + #[must_use] + pub fn header(mut self, name: impl Into, value: impl Into) -> Self { + self.headers.push((name.into(), value.into())); + self + } + + #[must_use] + pub fn body(mut self, body: impl Into>) -> Self { + self.body = body.into(); + self + } + + /// Keep the connection open for another request, unless the request + /// asked to close it. + #[must_use] + pub fn keep_alive(mut self) -> Self { + self.keep_alive = true; + self + } + + fn encode(&self, close: bool) -> Vec { + let mut head = format!("HTTP/1.1 {} {}\r\n", self.status, reason(self.status)); + for (name, value) in &self.headers { + let _ = write!(head, "{name}: {value}\r\n"); + } + if self.status >= 200 { + let _ = write!(head, "Content-Length: {}\r\n", self.body.len()); + } + if !self + .headers + .iter() + .any(|(name, _)| name.eq_ignore_ascii_case("connection")) + { + let connection = if close { "close" } else { "keep-alive" }; + let _ = write!(head, "Connection: {connection}\r\n"); + } + head.push_str("\r\n"); + let mut bytes = head.into_bytes(); + bytes.extend_from_slice(&self.body); + bytes + } +} + +fn reason(status: u16) -> &'static str { + match status { + 101 => "Switching Protocols", + 200 => "OK", + 202 => "Accepted", + 204 => "No Content", + 400 => "Bad Request", + 403 => "Forbidden", + 404 => "Not Found", + 405 => "Method Not Allowed", + 418 => "I'm a teapot", + _ => "Status", + } +} + +type Handler = dyn Fn(&Request) -> Response + Send + Sync; + +/// An HTTP server that records each request it read and answers it with a +/// handler's response. A connection stays open for another request while +/// the responses keep it alive and the requests don't close it. pub struct HttpRecorder { pub port: u16, requests: Arc>>, + connections: Arc, task: JoinHandle<()>, } impl HttpRecorder { + /// Answer every request with `200` and `body`. pub async fn start(body: impl Into) -> Result { + let body = body.into(); + Self::serve(move |_| Response::text(body.clone())).await + } + + /// Answer every request with `handler`'s response. + pub async fn serve( + handler: impl Fn(&Request) -> Response + Send + Sync + 'static, + ) -> Result { let (listener, port) = bind().await?; - let body: Arc = body.into().into(); + let handler: Arc = Arc::new(handler); let requests = Arc::new(Mutex::new(Vec::new())); + let connections = Arc::new(AtomicUsize::new(0)); let task_requests = Arc::clone(&requests); + let task_connections = Arc::clone(&connections); let task = tokio::spawn(async move { while let Ok((stream, _)) = listener.accept().await { + task_connections.fetch_add(1, Ordering::AcqRel); tokio::spawn(respond( stream, - Arc::clone(&body), + Arc::clone(&handler), Arc::clone(&task_requests), )); } @@ -45,10 +187,24 @@ impl HttpRecorder { Ok(Self { port, requests, + connections, task, }) } + /// The requests the server has read so far. + pub fn requests(&self) -> Vec { + self.requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + /// The number of connections the server has accepted. + pub fn connections(&self) -> usize { + self.connections.load(Ordering::Acquire) + } + /// The requests the server has read, once there are at least `count`. pub async fn wait_for_requests(&self, count: usize) -> Result> { testkit::poll_until( @@ -59,11 +215,7 @@ impl HttpRecorder { WAIT_TIMEOUT, WAIT_INTERVAL, async || { - let requests = self - .requests - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone(); + let requests = self.requests(); Ok(if requests.len() >= count { Poll::Ready(requests) } else { @@ -81,45 +233,204 @@ impl Drop for HttpRecorder { } } -/// Read one request, headers and `Content-Length` body, record it, and -/// answer with `body`. -async fn respond(mut stream: TcpStream, body: Arc, requests: Arc>>) { +/// Read requests, record each, and answer with `handler` until a response +/// or request closes the connection. +async fn respond(mut stream: TcpStream, handler: Arc, requests: Arc>>) { let _ = stream.set_nodelay(true); - let Some(request) = read_request(&mut stream).await else { - return; - }; - requests - .lock() - .unwrap_or_else(PoisonError::into_inner) - .push(request); - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - let _ = stream.write_all(response.as_bytes()).await; + let mut buffer = Vec::new(); + while let Some(request) = read_request(&mut stream, &mut buffer).await { + let response = handler(&request); + let close = !response.keep_alive + || request + .header("connection") + .is_some_and(|value| value.eq_ignore_ascii_case("close")); + requests + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(request); + if stream.write_all(&response.encode(close)).await.is_err() || close { + break; + } + } let _ = stream.shutdown().await; } -pub(crate) async fn read_request(stream: &mut TcpStream) -> Option { - let mut request = Vec::new(); - let mut buffer = [0_u8; 4096]; +/// Read one request from `stream`, starting with the bytes already in +/// `buffer`, and leave any bytes after it in `buffer`. +pub(crate) async fn read_request(stream: &mut TcpStream, buffer: &mut Vec) -> Option { + let mut chunk = [0_u8; 4096]; loop { - if let Some(end) = find(&request, b"\r\n\r\n") { - let head = String::from_utf8_lossy(&request[..end]).into_owned(); - let length = header(&head, "content-length") - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - if request.len() >= end + 4 + length { - let body = - String::from_utf8_lossy(&request[end + 4..end + 4 + length]).into_owned(); - return Some(Request { head, body }); - } + if let Some((request, length)) = parse_request(buffer) { + buffer.drain(..length); + return Some(request); } - match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut buffer)).await { - Ok(Ok(read)) if read > 0 && request.len() < MAX_REQUEST => { - request.extend_from_slice(&buffer[..read]); + match tokio::time::timeout(READ_TIMEOUT, stream.read(&mut chunk)).await { + Ok(Ok(read)) if read > 0 && buffer.len() < MAX_REQUEST => { + buffer.extend_from_slice(&chunk[..read]); } _ => return None, } } } + +/// A complete request at the start of `bytes`, and its length. +fn parse_request(bytes: &[u8]) -> Option<(Request, usize)> { + let end = find(bytes, b"\r\n\r\n")?; + let head = String::from_utf8_lossy(&bytes[..end]).into_owned(); + let rest = &bytes[end + 4..]; + let chunked = header(&head, "transfer-encoding") + .is_some_and(|value| value.to_ascii_lowercase().contains("chunked")); + let (body, length) = if chunked { + decode_chunked(rest)? + } else { + let length = header(&head, "content-length") + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + (rest.get(..length)?.to_vec(), length) + }; + let body = String::from_utf8_lossy(&body).into_owned(); + Some((Request { head, body }, end + 4 + length)) +} + +/// A complete chunked body at the start of `bytes`, decoded, and its encoded +/// length. +fn decode_chunked(bytes: &[u8]) -> Option<(Vec, usize)> { + let mut body = Vec::new(); + let mut position = 0; + loop { + let line_end = position + find(&bytes[position..], b"\r\n")?; + let size = std::str::from_utf8(&bytes[position..line_end]).ok()?; + let size = size.split(';').next().unwrap_or_default().trim(); + let size = usize::from_str_radix(size, 16).ok()?; + position = line_end + 2; + if size == 0 { + // Skip trailers up to the blank line that ends the body. + loop { + let line_end = position + find(&bytes[position..], b"\r\n")?; + let blank = line_end == position; + position = line_end + 2; + if blank { + return Some((body, position)); + } + } + } + body.extend_from_slice(bytes.get(position..position + size)?); + position += size; + if bytes.get(position..position + 2)? != b"\r\n" { + return None; + } + position += 2; + } +} + +#[cfg(test)] +mod tests { + use super::*; + + async fn connect(port: u16) -> TcpStream { + #[allow( + clippy::disallowed_methods, + reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" + )] + let stream = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + stream.set_nodelay(true).unwrap(); + stream + } + + async fn read_to_end(stream: &mut TcpStream) -> String { + let mut response = Vec::new(); + stream.read_to_end(&mut response).await.unwrap(); + String::from_utf8_lossy(&response).into_owned() + } + + #[test] + fn parse_request_decodes_a_chunked_body_and_leaves_the_next_request() { + let bytes = b"POST /a HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n\ + 3;ext=1\r\nabc\r\nA\r\n0123456789\r\n0\r\nTrailer: x\r\n\r\nGET /b"; + let (request, length) = parse_request(bytes).unwrap(); + assert_eq!(request.method(), "POST"); + assert_eq!(request.path(), "/a"); + assert_eq!(request.body, "abc0123456789"); + assert_eq!(&bytes[length..], b"GET /b"); + } + + #[test] + fn parse_request_waits_for_the_whole_body() { + assert!(parse_request(b"POST / HTTP/1.1\r\nContent-Length: 4\r\n\r\nabc").is_none()); + assert!( + parse_request(b"POST / HTTP/1.1\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n") + .is_none() + ); + let (request, _) = + parse_request(b"POST / HTTP/1.1\r\ncontent-length: 4\r\n\r\nabcd").unwrap(); + assert_eq!(request.body, "abcd"); + assert_eq!(request.header("Content-Length").as_deref(), Some("4")); + } + + #[test] + fn response_encodes_length_and_connection_unless_set() { + let encoded = String::from_utf8(Response::text("ok").encode(true)).unwrap(); + assert_eq!( + encoded, + "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok" + ); + let upgrade = Response::new(101) + .header("Connection", "Upgrade") + .header("Upgrade", "websocket") + .encode(true); + assert_eq!( + String::from_utf8(upgrade).unwrap(), + "HTTP/1.1 101 Switching Protocols\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n" + ); + } + + #[tokio::test] + async fn http_recorder_keeps_a_connection_alive_until_the_request_closes_it() { + let recorder = + HttpRecorder::serve(|request| Response::text(request.path().to_string()).keep_alive()) + .await + .unwrap(); + let mut stream = connect(recorder.port).await; + stream + .write_all( + b"GET /one HTTP/1.1\r\nHost: test\r\n\r\n\ + GET /two HTTP/1.1\r\nHost: test\r\nConnection: close\r\n\r\n", + ) + .await + .unwrap(); + let response = read_to_end(&mut stream).await; + assert!( + response.contains("Connection: keep-alive\r\n\r\n/one"), + "{response}" + ); + assert!( + response.ends_with("Connection: close\r\n\r\n/two"), + "{response}" + ); + let paths: Vec<_> = recorder + .wait_for_requests(2) + .await + .unwrap() + .iter() + .map(|request| request.path().to_string()) + .collect(); + assert_eq!(paths, ["/one", "/two"]); + assert_eq!(recorder.connections(), 1); + } + + #[tokio::test] + async fn http_recorder_closes_after_a_response_that_is_not_kept_alive() { + let recorder = HttpRecorder::start("body").await.unwrap(); + let mut stream = connect(recorder.port).await; + stream + .write_all(b"GET / HTTP/1.1\r\nHost: test\r\n\r\nGET /ignored HTTP/1.1\r\n\r\n") + .await + .unwrap(); + let response = read_to_end(&mut stream).await; + assert!( + response.ends_with("Connection: close\r\n\r\nbody"), + "{response}" + ); + assert_eq!(recorder.requests().len(), 1); + } +} diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 8ac8528570..45647ecf9d 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -9,8 +9,17 @@ //! `host.openshell.internal` to different ones, and stop when dropped. They //! run where the tests run, so a gateway whose sandboxes can't reach the test //! host, such as one on a remote cluster, can't run tests that use them. +//! +//! - [`HttpRecorder`] answers HTTP requests with a fixed body or a handler's +//! [`Response`], and records each parsed [`Request`]. +//! - [`WebSocketRecorder`] completes the WebSocket handshake and records the +//! frames that arrive. +//! - [`TcpServer`] records raw bytes, and echoes them or answers once the +//! client goes idle. +//! - [`free_port`] finds a port nothing listens on. mod http; +mod tcp; mod websocket; use std::time::Duration; @@ -18,8 +27,9 @@ use std::time::Duration; use testkit::{Result, ResultExt as _}; use tokio::net::TcpListener; -pub use http::{HttpRecorder, Request}; -pub use websocket::WebSocketRecorder; +pub use http::{HttpRecorder, Request, Response}; +pub use tcp::TcpServer; +pub use websocket::{TEXT_REPLY, WebSocketRecorder}; const WAIT_TIMEOUT: Duration = Duration::from_secs(10); const WAIT_INTERVAL: Duration = Duration::from_millis(100); @@ -37,6 +47,12 @@ async fn bind() -> Result<(TcpListener, u16)> { Ok((listener, port)) } +/// A port that nothing on the test host listened on when this returned. +pub async fn free_port() -> Result { + let (_listener, port) = bind().await?; + Ok(port) +} + fn find(haystack: &[u8], needle: &[u8]) -> Option { haystack .windows(needle.len()) diff --git a/tests/support/host-server/src/tcp.rs b/tests/support/host-server/src/tcp.rs new file mode 100644 index 0000000000..42c4eaa9b0 --- /dev/null +++ b/tests/support/host-server/src/tcp.rs @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use testkit::{Poll, Result}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::TcpStream; +use tokio::task::JoinHandle; + +use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, find}; + +/// How long a [`TcpServer::reply_when_idle`] connection waits for more bytes +/// before it answers. +const IDLE: Duration = Duration::from_millis(200); + +/// A TCP server that records the bytes every connection sent, in arrival +/// order, for a test that checks raw bytes rather than parsed requests. +pub struct TcpServer { + pub port: u16, + received: Arc>>, + connections: Arc, + task: JoinHandle<()>, +} + +#[derive(Clone)] +enum Mode { + Echo, + ReplyWhenIdle(Arc<[u8]>), +} + +impl TcpServer { + /// Write back every byte a connection sends until it closes. + pub async fn echo() -> Result { + Self::start(Mode::Echo).await + } + + /// Read until a connection is idle for 200 ms or closes, then write + /// `reply` and close it. A test can send several requests at once and + /// see all of them, since the server answers none before the last. + pub async fn reply_when_idle(reply: impl Into>) -> Result { + Self::start(Mode::ReplyWhenIdle(reply.into().into())).await + } + + async fn start(mode: Mode) -> Result { + let (listener, port) = bind().await?; + let received = Arc::new(Mutex::new(Vec::new())); + let connections = Arc::new(AtomicUsize::new(0)); + let task_received = Arc::clone(&received); + let task_connections = Arc::clone(&connections); + let task = tokio::spawn(async move { + while let Ok((stream, _)) = listener.accept().await { + task_connections.fetch_add(1, Ordering::AcqRel); + tokio::spawn(serve(stream, mode.clone(), Arc::clone(&task_received))); + } + }); + Ok(Self { + port, + received, + connections, + task, + }) + } + + /// The bytes the server has read so far. + pub fn received(&self) -> Vec { + self.received + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + /// The number of connections the server has accepted. + pub fn connections(&self) -> usize { + self.connections.load(Ordering::Acquire) + } + + /// The bytes the server has read, once they contain `needle`. + pub async fn wait_for(&self, needle: &[u8]) -> Result> { + testkit::poll_until( + &format!( + "{:?} at the host server on port {}", + String::from_utf8_lossy(needle), + self.port + ), + WAIT_TIMEOUT, + WAIT_INTERVAL, + async || { + let received = self.received(); + Ok(if find(&received, needle).is_some() { + Poll::Ready(received) + } else { + Poll::Pending(format!("{} byte(s)", received.len())) + }) + }, + ) + .await + } +} + +impl Drop for TcpServer { + fn drop(&mut self) { + self.task.abort(); + } +} + +async fn serve(mut stream: TcpStream, mode: Mode, received: Arc>>) { + let _ = stream.set_nodelay(true); + let idle = match mode { + Mode::Echo => READ_TIMEOUT, + Mode::ReplyWhenIdle(_) => IDLE, + }; + let mut buffer = [0_u8; 4096]; + let mut total = 0; + loop { + let read = match tokio::time::timeout(idle, stream.read(&mut buffer)).await { + Ok(Ok(read)) if read > 0 && total < MAX_REQUEST => read, + _ => break, + }; + total += read; + received + .lock() + .unwrap_or_else(PoisonError::into_inner) + .extend_from_slice(&buffer[..read]); + if matches!(mode, Mode::Echo) && stream.write_all(&buffer[..read]).await.is_err() { + return; + } + } + if let Mode::ReplyWhenIdle(reply) = mode { + let _ = stream.write_all(&reply).await; + } + let _ = stream.shutdown().await; +} + +#[cfg(test)] +mod tests { + use super::*; + + async fn connect(port: u16) -> TcpStream { + #[allow( + clippy::disallowed_methods, + reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" + )] + let stream = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + stream.set_nodelay(true).unwrap(); + stream + } + + #[tokio::test] + async fn echo_writes_back_and_records_bytes() { + let server = TcpServer::echo().await.unwrap(); + let mut stream = connect(server.port).await; + stream.write_all(&[0x00, 0xff, b'x']).await.unwrap(); + let mut echoed = [0_u8; 3]; + stream.read_exact(&mut echoed).await.unwrap(); + assert_eq!(echoed, [0x00, 0xff, b'x']); + assert_eq!(server.wait_for(b"x").await.unwrap(), [0x00, 0xff, b'x']); + assert_eq!(server.connections(), 1); + } + + #[tokio::test] + async fn reply_when_idle_records_everything_before_replying() { + let server = TcpServer::reply_when_idle(b"reply".to_vec()).await.unwrap(); + let mut stream = connect(server.port).await; + stream.write_all(b"first ").await.unwrap(); + stream.write_all(b"second").await.unwrap(); + let mut reply = Vec::new(); + stream.read_to_end(&mut reply).await.unwrap(); + assert_eq!(reply, b"reply"); + assert_eq!(server.received(), b"first second"); + } +} diff --git a/tests/support/host-server/src/websocket.rs b/tests/support/host-server/src/websocket.rs index 264dd4b5d3..4f38bdb8f3 100644 --- a/tests/support/host-server/src/websocket.rs +++ b/tests/support/host-server/src/websocket.rs @@ -1,8 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; use testkit::{Poll, Result}; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; @@ -10,21 +10,26 @@ use tokio::net::TcpStream; use tokio::task::JoinHandle; use crate::http::read_request; -use crate::{READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, header}; +use crate::{MAX_REQUEST, READ_TIMEOUT, WAIT_INTERVAL, WAIT_TIMEOUT, bind, header}; /// A WebSocket server that completes the handshake, then reads frames until -/// the connection closes, and records whether any was a binary frame. +/// the connection closes. It records whether any was a binary frame, records +/// each text frame, and answers each text frame with [`TEXT_REPLY`]. pub struct WebSocketRecorder { pub port: u16, state: Arc, task: JoinHandle<()>, } +/// The text frame a [`WebSocketRecorder`] sends after each text frame. +pub const TEXT_REPLY: &str = "received"; + #[derive(Default)] struct WebSocketState { handshake_seen: AtomicBool, binary_seen: AtomicBool, finished: AtomicBool, + texts: Mutex>, } impl WebSocketRecorder { @@ -81,6 +86,15 @@ impl WebSocketRecorder { ) .await } + + /// The text frames the server has read so far. + pub fn texts(&self) -> Vec { + self.state + .texts + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } } impl Drop for WebSocketRecorder { @@ -91,7 +105,7 @@ impl Drop for WebSocketRecorder { async fn accept_websocket(mut stream: TcpStream, state: Arc) { let _ = stream.set_nodelay(true); - let Some(request) = read_request(&mut stream).await else { + let Some(request) = read_request(&mut stream, &mut Vec::new()).await else { return; }; let Some(key) = header(&request.head, "sec-websocket-key") else { @@ -111,8 +125,8 @@ async fn accept_websocket(mut stream: TcpStream, state: Arc) { state.finished.store(true, Ordering::Release); } -/// Read frames until a close frame or the end of the stream, and record a -/// binary frame. +/// Read frames until a close frame or the end of the stream, record a +/// binary frame and each text frame, and answer each text frame. async fn read_frames(stream: &mut TcpStream, state: &WebSocketState) -> std::io::Result<()> { loop { let mut frame_header = [0_u8; 2]; @@ -123,31 +137,77 @@ async fn read_frames(stream: &mut TcpStream, state: &WebSocketState) -> std::io: 127 => stream.read_u64().await?, length => u64::from(length), }; - if opcode == 0x02 { - state.binary_seen.store(true, Ordering::Release); - } - if opcode == 0x08 { - return Ok(()); + match opcode { + 0x01 => { + let text = read_payload(stream, frame_header[1], length).await?; + state + .texts + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(String::from_utf8_lossy(&text).into_owned()); + stream.write_all(&server_text_frame(TEXT_REPLY)).await?; + continue; + } + 0x02 => state.binary_seen.store(true, Ordering::Release), + _ => {} } let mask = if frame_header[1] & 0x80 == 0 { 0 } else { 4 }; + // Read a close frame's payload too, since closing with unread bytes + // resets the connection. let skipped = tokio::io::copy( &mut (&mut *stream).take(mask + length), &mut tokio::io::sink(), ) .await?; - if skipped < mask + length { + if opcode == 0x08 || skipped < mask + length { return Ok(()); } } } +/// Read a text frame's payload, unmasked, up to [`MAX_REQUEST`] bytes. +async fn read_payload( + stream: &mut TcpStream, + second_byte: u8, + length: u64, +) -> std::io::Result> { + let length = usize::try_from(length) + .ok() + .filter(|length| *length <= MAX_REQUEST) + .ok_or_else(|| std::io::Error::other("text frame too large"))?; + let mut mask = [0_u8; 4]; + if second_byte & 0x80 != 0 { + stream.read_exact(&mut mask).await?; + } + let mut payload = vec![0_u8; length]; + stream.read_exact(&mut payload).await?; + for (index, byte) in payload.iter_mut().enumerate() { + *byte ^= mask[index % 4]; + } + Ok(payload) +} + +/// A final, unmasked server text frame. +fn server_text_frame(text: &str) -> Vec { + let mut frame = vec![0x81]; + match text.len() { + length @ 0..=125 => frame.push(u8::try_from(length).unwrap_or_default()), + length => { + frame.push(126); + frame.extend_from_slice(&u16::try_from(length).unwrap_or(u16::MAX).to_be_bytes()); + } + } + frame.extend_from_slice(text.as_bytes()); + frame +} + #[cfg(test)] mod tests { use super::*; use crate::find; /// Complete a handshake with `recorder`, send `frames`, and close. - async fn send_frames(recorder: &WebSocketRecorder, frames: &[(u8, Vec)]) { + async fn send_frames(recorder: &WebSocketRecorder, frames: &[(u8, Vec)]) -> TcpStream { #[allow( clippy::disallowed_methods, reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" @@ -181,6 +241,7 @@ mod tests { stream.write_all(&frame(*opcode, payload)).await.unwrap(); } stream.write_all(&frame(0x08, &[])).await.unwrap(); + stream } /// A final, masked client frame. @@ -202,22 +263,43 @@ mod tests { #[tokio::test] async fn websocket_recorder_sees_a_binary_frame_after_a_long_text_frame() { let recorder = WebSocketRecorder::start().await.unwrap(); - send_frames( + // Hold the connection so the replies to text frames don't reset it. + let _stream = send_frames( &recorder, &[(0x01, vec![b'x'; 300]), (0x02, b"binary".to_vec())], ) .await; assert!(recorder.wait_until_finished().await.unwrap()); + assert_eq!(recorder.texts(), ["x".repeat(300)]); } #[tokio::test] async fn websocket_recorder_sees_no_binary_frame_in_text_frames() { let recorder = WebSocketRecorder::start().await.unwrap(); - send_frames( + // Hold the connection so the replies to text frames don't reset it. + let _stream = send_frames( &recorder, &[(0x01, vec![b'x'; 300]), (0x01, b"text".to_vec())], ) .await; assert!(!recorder.wait_until_finished().await.unwrap()); } + + #[tokio::test] + async fn websocket_recorder_answers_each_text_frame() { + let recorder = WebSocketRecorder::start().await.unwrap(); + let mut stream = send_frames( + &recorder, + &[(0x01, b"one".to_vec()), (0x01, b"two".to_vec())], + ) + .await; + let mut replies = Vec::new(); + stream.read_to_end(&mut replies).await.unwrap(); + assert_eq!( + replies, + [server_text_frame(TEXT_REPLY), server_text_frame(TEXT_REPLY)].concat() + ); + recorder.wait_until_finished().await.unwrap(); + assert_eq!(recorder.texts(), ["one", "two"]); + } } From 0dd040ad689f63fdd692d548cd84e0fb3e86dbb4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:45:45 -0800 Subject: [PATCH 03/15] test(testkit): add policy::base_with for YAML policy sections Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 3 ++- tests/testkit/testkit/src/policy.rs | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 744fc75c3d..0275c0af6f 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -111,7 +111,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. (binaries apply to every endpoint under `name`) that `policy::Endpoint` configures (`.rest(Access::Full)`, `.credential_binding(provider)`, ...). Pass the result to `cx.sandbox(..).policy(policy)`, or `policy::to_yaml` - for the CLI. `policy::from_yaml` reads a policy file, and + for the CLI. `policy::base_with(yaml)` adds YAML sections, such as L7 + rules, to the base policy. `policy::from_yaml` reads a policy file, and `policy::set(&sandbox, &policy)` replaces a running sandbox's policy and waits until it is loaded. `testkit::PYTHON_BINARIES` names `python3` in `python_image()` images. diff --git a/tests/testkit/testkit/src/policy.rs b/tests/testkit/testkit/src/policy.rs index ccaeba71d8..b8765f2dba 100644 --- a/tests/testkit/testkit/src/policy.rs +++ b/tests/testkit/testkit/src/policy.rs @@ -86,6 +86,13 @@ pub fn from_yaml(yaml: &str) -> Result { .context("parse the policy") } +/// [`base`] with the top-level sections in `yaml`, such as +/// `network_policies` or `network_middlewares`, added. Use it for policy +/// settings [`Endpoint`] doesn't cover, such as L7 rules. +pub fn base_with(yaml: &str) -> Result { + from_yaml(&format!("{BASE}{yaml}")) +} + /// Replace `sandbox`'s policy with `policy` through `policy set --wait`, and /// wait up to two minutes until the sandbox has loaded it. pub async fn set(sandbox: &Sandbox, policy: &proto::SandboxPolicy) -> Result { @@ -219,6 +226,11 @@ mod tests { .expect("the serialized policy parses"); assert_eq!(round_trip, expected); assert_eq!(from_yaml(&yaml).expect("the YAML parses"), expected); + assert_eq!( + base_with(yaml.strip_prefix(BASE).expect("starts with the base")) + .expect("the YAML parses"), + expected + ); } #[test] From a0e9efae9902e7b357ea06962da529ce2888acff Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:57:28 -0800 Subject: [PATCH 04/15] test(conformance): port network bypass, listener, and loopback tests Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/lib.rs | 2 + tests/suites/conformance/src/network.rs | 133 ++++++++++++++++++ .../conformance/src/scripts/bypass_client.py | 26 ++++ .../src/scripts/destination_denials_client.py | 21 +++ .../src/scripts/listener_lifetime_probe.py | 81 +++++++++++ .../src/scripts/localhost_client.py | 38 +++++ .../src/scripts/preconnect_owner.py | 21 +++ .../src/scripts/socket_not_inherited.py | 14 ++ tests/suites/conformance/src/support.rs | 79 +++++++++++ 9 files changed, 415 insertions(+) create mode 100644 tests/suites/conformance/src/network.rs create mode 100644 tests/suites/conformance/src/scripts/bypass_client.py create mode 100644 tests/suites/conformance/src/scripts/destination_denials_client.py create mode 100644 tests/suites/conformance/src/scripts/listener_lifetime_probe.py create mode 100644 tests/suites/conformance/src/scripts/localhost_client.py create mode 100644 tests/suites/conformance/src/scripts/preconnect_owner.py create mode 100644 tests/suites/conformance/src/scripts/socket_not_inherited.py create mode 100644 tests/suites/conformance/src/support.rs diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index f0a568679c..4b51903060 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -17,10 +17,12 @@ mod filesystem_policy; mod image; mod labels; mod lifecycle; +mod network; mod persistent_workspace; mod policy_advisor; mod process; mod settings; mod smoke; mod ssh; +mod support; mod templates; diff --git a/tests/suites/conformance/src/network.rs b/tests/suites/conformance/src/network.rs new file mode 100644 index 0000000000..b618509fd6 --- /dev/null +++ b/tests/suites/conformance/src/network.rs @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Sandbox networking: direct connections the sandbox refuses, loopback +//! traffic it leaves alone, and listening sockets whose lifetime it must not +//! extend. +//! +//! Ported from the `bypass_detection`, `listener_lifetime`, and `no_proxy` +//! e2e tests, and the destination denials in `proxy_egress_pipeline`. + +use std::time::Duration; + +use serde_json::json; +use testkit::{Context, ResultExt as _, policy}; + +use crate::support::{ + expect_values, json_result, python_binaries_yaml, python_sandbox, run_python, +}; + +const BYPASS_CLIENT: &str = include_str!("scripts/bypass_client.py"); +const DESTINATION_DENIALS_CLIENT: &str = include_str!("scripts/destination_denials_client.py"); +const LOCALHOST_CLIENT: &str = include_str!("scripts/localhost_client.py"); +const LISTENER_LIFETIME_PROBE: &str = include_str!("scripts/listener_lifetime_probe.py"); +const PRECONNECT_OWNER: &str = include_str!("scripts/preconnect_owner.py"); +const SOCKET_NOT_INHERITED: &str = include_str!("scripts/socket_not_inherited.py"); + +/// A direct TCP connection that bypasses the proxy is denied with a +/// permission error well before the socket's 10-second timeout. +#[testkit::test] +async fn direct_connect_bypass_is_denied_promptly(cx: &mut Context) -> testkit::Result { + let sandbox = python_sandbox(cx, "bypass", policy::base()).await?; + let output = run_python(&sandbox, BYPASS_CLIENT, &[]).await?; + let result = json_result(&output)?; + testkit::ensure!( + result["bypass_result"] == "denied", + "expected the direct connect to be denied; got {result}" + ); + let elapsed = result["elapsed_ms"].as_u64().unwrap_or(u64::MAX); + testkit::ensure!( + elapsed < 8000, + "the denial took {elapsed} ms; expected under 8000" + ); + Ok(()) +} + +/// Connections to cloud metadata, a Kubernetes API port, and an address +/// outside an endpoint's `allowed_ips` fail with `EACCES` even though the +/// policy names them. +#[testkit::test] +async fn reserved_destinations_fail_with_eacces_even_when_allowed( + cx: &mut Context, +) -> testkit::Result { + let policy = policy::base_with(&format!( + r#"network_policies: + destination_denials: + name: destination_denials + endpoints: + - {{ host: 169.254.169.254, port: 80 }} + - {{ host: 127.0.0.1, port: 80 }} + - {{ host: 203.0.113.10, port: 6443 }} + - host: 203.0.113.10 + port: 8080 + allowed_ips: ["198.51.100.0/24"] + binaries: {binaries} +"#, + binaries = python_binaries_yaml(), + ))?; + let sandbox = python_sandbox(cx, "denials", policy).await?; + let output = run_python(&sandbox, DESTINATION_DENIALS_CLIENT, &[]).await?; + expect_values( + &json_result(&output)?, + &[ + ("metadata", json!(13)), + ("control_plane", json!(13)), + ("outside_allowed_ips", json!(13)), + ], + ) +} + +/// A process can serve and fetch HTTP on 127.0.0.1 without proxy variables +/// in its environment. +#[testkit::test] +async fn localhost_is_reachable_without_proxy_environment(cx: &mut Context) -> testkit::Result { + let sandbox = python_sandbox(cx, "local", policy::base()).await?; + let output = run_python(&sandbox, LOCALHOST_CLIENT, &[]).await?; + expect_values( + &json_result(&output)?, + &[ + ("proxy_env_absent", json!(true)), + ("payload", json!({"message": "hello"})), + ], + ) +} + +/// Closing a listener, or exiting while holding one, frees its port: new +/// connections are refused and the port can be bound again. A later exec +/// doesn't inherit the exited child's listener. +#[testkit::test] +async fn closed_or_exited_listener_releases_its_port(cx: &mut Context) -> testkit::Result { + let sandbox = python_sandbox(cx, "listen", policy::base()).await?; + let output = run_python(&sandbox, LISTENER_LIFETIME_PROBE, &[]).await?; + let inode = json_result(&output)?[1].to_string(); + run_python(&sandbox, SOCKET_NOT_INHERITED, &[&inode]) + .await + .context("a later exec inherited the closed listener")?; + Ok(()) +} + +/// An exec doesn't inherit a socket that the sandbox's main process opened +/// without close-on-exec. +#[testkit::test] +async fn exec_does_not_inherit_main_process_socket(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("preconn") + .policy(policy::base()) + .image(testkit::python_image()) + .command(["python3", "-c", PRECONNECT_OWNER]) + .create_ready() + .await?; + let inode = sandbox + .poll_exec( + "the main process to publish its socket inode", + Duration::from_secs(30), + ["cat", "/sandbox/preconnect-inode"], + |output| output.exit_code == 0 && output.stdout.trim().parse::().is_ok(), + ) + .await? + .stdout; + run_python(&sandbox, SOCKET_NOT_INHERITED, &[inode.trim()]) + .await + .context("the exec inherited the main process's socket")?; + Ok(()) +} diff --git a/tests/suites/conformance/src/scripts/bypass_client.py b/tests/suites/conformance/src/scripts/bypass_client.py new file mode 100644 index 0000000000..e3c1a75d51 --- /dev/null +++ b/tests/suites/conformance/src/scripts/bypass_client.py @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Connects straight to an unroutable address, bypassing any proxy, and +# prints the outcome and how long it took. +import json, socket, time + +start = time.monotonic() +result = "unknown" +try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(10) + s.connect(("198.51.100.1", 80)) + result = "connected" + s.close() +except ConnectionRefusedError: + result = "refused" +except PermissionError: + result = "denied" +except socket.timeout: + result = "timeout" +except OSError as e: + result = f"error:{e}" + +elapsed_ms = int((time.monotonic() - start) * 1000) +print(json.dumps({"bypass_result": result, "elapsed_ms": elapsed_ms}), flush=True) diff --git a/tests/suites/conformance/src/scripts/destination_denials_client.py b/tests/suites/conformance/src/scripts/destination_denials_client.py new file mode 100644 index 0000000000..a4ddd6475e --- /dev/null +++ b/tests/suites/conformance/src/scripts/destination_denials_client.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Connects to addresses the policy names but the sandbox must still refuse, +# and prints each connect's errno as JSON, 0 for a connection. +import json +import socket + +targets = { + "metadata": ("169.254.169.254", 80), + "control_plane": ("203.0.113.10", 6443), + "outside_allowed_ips": ("203.0.113.10", 8080), +} +result = {} +for name, target in targets.items(): + try: + with socket.create_connection(target, timeout=10): + result[name] = 0 + except OSError as error: + result[name] = error.errno +print(json.dumps(result, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/listener_lifetime_probe.py b/tests/suites/conformance/src/scripts/listener_lifetime_probe.py new file mode 100644 index 0000000000..e53b2a6875 --- /dev/null +++ b/tests/suites/conformance/src/scripts/listener_lifetime_probe.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Closes listeners and exits a child that owns one, and checks each time that +# the port refuses connections and can be bound again. Prints the child +# listener's address and socket inode as JSON. +import errno, json, os, socket, subprocess, sys + +def assert_closed(address, inode): + # Check before bind can trigger the broker's stale-descriptor collection. + for table in ('/proc/net/tcp', '/proc/net/tcp6'): + for line in open(table): + fields = line.split() + assert not (len(fields) > 9 and fields[3] == '0A' and fields[9] == str(inode)), line + with socket.socket() as client: + client.settimeout(2) + assert client.connect_ex(tuple(address)) == errno.ECONNREFUSED, address + with socket.socket() as replacement: + replacement.bind(tuple(address)) + +def inode(sock): + return int(os.readlink('/proc/self/fd/' + str(sock.fileno()))[8:-1]) + +with socket.socket() as bound: + bound.bind(('127.0.0.1', 0)) + address = bound.getsockname() +with socket.socket() as client: + client.settimeout(2) + assert client.connect_ex(address) == errno.ECONNREFUSED + +for host in ('127.0.0.1', '::1'): + family = socket.AF_INET if host == '127.0.0.1' else socket.AF_INET6 + listener = socket.socket(family) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind((host, 0)) + listener.listen(1) + listener.listen(8) + address, identity = listener.getsockname(), inode(listener) + for operation, expected in ((lambda: listener.bind(address), errno.EINVAL), + (lambda: listener.connect(address), errno.EISCONN)): + try: + operation() + raise AssertionError('operation on listener unexpectedly succeeded') + except OSError as error: + assert error.errno == expected, error + alias = listener.dup() + alias.settimeout(2) + listener.close() + with socket.socket(family) as client: + client.settimeout(2) + client.connect(address) + accepted, _ = alias.accept() + with accepted: + accepted.sendall(b'ok') + assert client.recv(2) == b'ok' + alias.close() + table = '/proc/net/tcp' if family == socket.AF_INET else '/proc/net/tcp6' + for line in open(table): + fields = line.split() + assert not (len(fields) > 9 and fields[3] == '0A' and fields[9] == str(identity)), line + with socket.socket(family) as client: + client.settimeout(2) + assert client.connect_ex(address) == errno.ECONNREFUSED + with socket.socket(family) as replacement: + replacement.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + replacement.bind(address) + +# The child exits while still owning its listener. Its broker must not keep it alive. +child = r''' +import json, os, socket +s = socket.socket() +s.bind(('127.0.0.1', 0)) +s.listen() +inode = int(os.readlink('/proc/self/fd/' + str(s.fileno()))[8:-1]) +print(json.dumps([s.getsockname(), inode]), flush=True) +os._exit(0) +''' +result = subprocess.run([sys.executable, '-c', child], check=True, capture_output=True, text=True, timeout=10) +address, identity = json.loads(result.stdout) +assert_closed(address, identity) +print(json.dumps([address, identity])) diff --git a/tests/suites/conformance/src/scripts/localhost_client.py b/tests/suites/conformance/src/scripts/localhost_client.py new file mode 100644 index 0000000000..3621138742 --- /dev/null +++ b/tests/suites/conformance/src/scripts/localhost_client.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Serves HTTP on 127.0.0.1 and fetches from it, with no proxy variables in +# the environment. +import json +import os +import threading +import urllib.request +from http.server import BaseHTTPRequestHandler, HTTPServer + +for name in ('HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy'): + assert name not in os.environ, f'unexpected proxy environment variable: {name}' + +class Handler(BaseHTTPRequestHandler): + def log_message(self, format, *args): + pass + + def do_GET(self): + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.end_headers() + self.wfile.write(b'{"message":"hello"}') + +server = HTTPServer(('127.0.0.1', 0), Handler) +thread = threading.Thread(target=server.serve_forever, daemon=True) +thread.start() + +try: + with urllib.request.urlopen(f'http://127.0.0.1:{server.server_port}', timeout=10) as response: + print(json.dumps({ + 'proxy_env_absent': True, + 'payload': json.loads(response.read().decode()), + }), flush=True) +finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() diff --git a/tests/suites/conformance/src/scripts/preconnect_owner.py b/tests/suites/conformance/src/scripts/preconnect_owner.py new file mode 100644 index 0000000000..1b17222d69 --- /dev/null +++ b/tests/suites/conformance/src/scripts/preconnect_owner.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Opens an inheritable socket, writes its inode to /sandbox/preconnect-inode, +# and sleeps. It asks libc for the socket, since Python's socket constructor +# requests SOCK_CLOEXEC, which would hide an inheritance bug. +import ctypes +import os +import socket +import time + +libc = ctypes.CDLL(None, use_errno=True) +fd = libc.socket(socket.AF_INET, socket.SOCK_STREAM, 0) +assert fd >= 0, ctypes.get_errno() +assert os.get_inheritable(fd) +inode = int(os.readlink('/proc/self/fd/' + str(fd))[8:-1]) +with open('/sandbox/preconnect-inode.tmp', 'w') as f: + f.write(str(inode)) +os.rename('/sandbox/preconnect-inode.tmp', '/sandbox/preconnect-inode') +while True: + time.sleep(60) diff --git a/tests/suites/conformance/src/scripts/socket_not_inherited.py b/tests/suites/conformance/src/scripts/socket_not_inherited.py new file mode 100644 index 0000000000..488a0d2683 --- /dev/null +++ b/tests/suites/conformance/src/scripts/socket_not_inherited.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Fails if this process holds the socket whose inode is argv[1]. +import os +import sys + +inode = int(sys.argv[1]) +for fd in os.listdir('/proc/self/fd'): + try: + target = os.readlink('/proc/self/fd/' + fd) + except FileNotFoundError: + continue + assert target != 'socket:[' + str(inode) + ']', (fd, target) diff --git a/tests/suites/conformance/src/support.rs b/tests/suites/conformance/src/support.rs new file mode 100644 index 0000000000..dd512371df --- /dev/null +++ b/tests/suites/conformance/src/support.rs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Helpers the network suites share. Not a suite. +//! +//! Their clients are Python scripts in `src/scripts/` that take the server's +//! host and port as arguments, run in a long-running sandbox from +//! [`testkit::python_image`], and print their result as one JSON line. + +use serde_json::Value; +use testkit::sdk::raw::proto; +use testkit::{Context, ExecOutput, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; + +/// The name sandboxes reach the test host's servers by. +pub const TEST_HOST: &str = "host.openshell.internal"; + +/// A long-running sandbox from the Python image with `policy`, Ready. +pub async fn python_sandbox( + cx: &Context, + suffix: &str, + policy: proto::SandboxPolicy, +) -> Result { + cx.sandbox(suffix) + .policy(policy) + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .create_ready() + .await +} + +/// Run `script` with the image's Python and `args`, and fail unless it +/// exits 0. +pub async fn run_python(sandbox: &Sandbox, script: &str, args: &[&str]) -> Result { + let mut command = vec!["python3", "-c", script]; + command.extend_from_slice(args); + sandbox + .exec(command) + .await? + .success() + .context("run the client in the sandbox") +} + +/// The last line of `output`'s stdout that parses as JSON. +pub fn json_result(output: &ExecOutput) -> Result { + output + .stdout + .lines() + .rev() + .find_map(|line| serde_json::from_str(line.trim()).ok()) + .ok_or_else(|| { + testkit::Error::msg(format!( + "no JSON result in the client's output:\n{}\n{}", + output.stdout, output.stderr + )) + }) +} + +/// Fail unless each key in `expected` has its value in `result`. +pub fn expect_values>(result: &Value, expected: &[(K, Value)]) -> Result { + let wrong: Vec = expected + .iter() + .filter(|(key, value)| result[key.as_ref()] != *value) + .map(|(key, value)| { + let key = key.as_ref(); + format!("{key}: expected {value}, got {}", result[key]) + }) + .collect(); + testkit::ensure!(wrong.is_empty(), "{}\nresult: {result}", wrong.join("\n")); + Ok(()) +} + +/// `binaries:` for policy YAML that lets the image's Python connect. +pub fn python_binaries_yaml() -> String { + let paths: Vec = PYTHON_BINARIES + .iter() + .map(|path| format!("{{path: \"{path}\"}}")) + .collect(); + format!("[{}]", paths.join(", ")) +} From 6d1f3b5ce546c60b90f5cb382d5785568642ec33 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:57:37 -0800 Subject: [PATCH 05/15] test(conformance): port REST, GraphQL, JSON-RPC, and MCP L7 policy tests Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/lib.rs | 1 + tests/suites/conformance/src/network_l7.rs | 274 ++++++++++++++++++ .../suites/conformance/src/network_l7/mcp.rs | 221 ++++++++++++++ .../conformance/src/scripts/graphql_client.py | 242 ++++++++++++++++ .../src/scripts/jsonrpc_audit_client.py | 34 +++ .../conformance/src/scripts/jsonrpc_client.py | 197 +++++++++++++ .../conformance/src/scripts/mcp_client.py | 113 ++++++++ .../conformance/src/scripts/rest_client.py | 25 ++ .../src/scripts/rest_upgrade_client.py | 28 ++ 9 files changed, 1135 insertions(+) create mode 100644 tests/suites/conformance/src/network_l7.rs create mode 100644 tests/suites/conformance/src/network_l7/mcp.rs create mode 100644 tests/suites/conformance/src/scripts/graphql_client.py create mode 100644 tests/suites/conformance/src/scripts/jsonrpc_audit_client.py create mode 100644 tests/suites/conformance/src/scripts/jsonrpc_client.py create mode 100644 tests/suites/conformance/src/scripts/mcp_client.py create mode 100644 tests/suites/conformance/src/scripts/rest_client.py create mode 100644 tests/suites/conformance/src/scripts/rest_upgrade_client.py diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 4b51903060..9f36b3c254 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -18,6 +18,7 @@ mod image; mod labels; mod lifecycle; mod network; +mod network_l7; mod persistent_workspace; mod policy_advisor; mod process; diff --git a/tests/suites/conformance/src/network_l7.rs b/tests/suites/conformance/src/network_l7.rs new file mode 100644 index 0000000000..e443eaf151 --- /dev/null +++ b/tests/suites/conformance/src/network_l7.rs @@ -0,0 +1,274 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! L7 network policy: REST, GraphQL, and JSON-RPC rules the sandbox enforces +//! on requests a workload sends straight to a server, without a proxy. +//! +//! Ported from the `forward_proxy_l7_bypass`, `forward_proxy_graphql_l7`, +//! and `forward_proxy_jsonrpc_l7` e2e tests. The upstream servers run on the +//! test host ([`host_server_support`]) and implement none of the protocols: +//! the sandbox parses and enforces them before forwarding, so a server that +//! answers is enough to tell an allowed request from a denied one. + +mod mcp; + +use host_server_support::{HttpRecorder, Request, Response}; +use serde_json::json; +use testkit::{Context, Result, ResultExt as _, policy}; + +use crate::support::{ + TEST_HOST, expect_values, json_result, python_binaries_yaml, python_sandbox, run_python, +}; + +const REST_CLIENT: &str = include_str!("scripts/rest_client.py"); +const REST_UPGRADE_CLIENT: &str = include_str!("scripts/rest_upgrade_client.py"); +const GRAPHQL_CLIENT: &str = include_str!("scripts/graphql_client.py"); +const JSONRPC_CLIENT: &str = include_str!("scripts/jsonrpc_client.py"); +const JSONRPC_AUDIT_CLIENT: &str = include_str!("scripts/jsonrpc_audit_client.py"); + +/// A REST endpoint that allows only `GET /allowed` passes that request and +/// answers a `POST` to the same path with `403` before it reaches upstream. +#[testkit::test] +async fn rest_rules_allow_get_and_deny_post(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(rest_upstream).await?; + let sandbox = python_sandbox(cx, "rest", rest_policy(server.port, "enforce")?).await?; + let output = run_python(&sandbox, REST_CLIENT, &[TEST_HOST, &port(&server)]).await?; + expect_values( + &json_result(&output)?, + &[("get", json!(200)), ("post", json!(403))], + )?; + let methods: Vec = server + .requests() + .iter() + .map(|request| request.method().to_string()) + .collect(); + testkit::ensure!( + methods == ["GET"], + "expected only the GET upstream; got {methods:?}" + ); + Ok(()) +} + +/// A permitted `GET` can't switch a REST endpoint into an uninspected +/// WebSocket tunnel, in enforce or audit mode: the sandbox answers the +/// upgrade with `403` and `unsupported_l7_protocol`. +#[testkit::test] +async fn rest_upgrade_is_refused_in_enforce_and_audit_modes(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(rest_upstream).await?; + for enforcement in ["enforce", "audit"] { + let sandbox = + python_sandbox(cx, enforcement, rest_policy(server.port, enforcement)?).await?; + run_python(&sandbox, REST_UPGRADE_CLIENT, &[TEST_HOST, &port(&server)]) + .await + .context(format!("{enforcement} mode"))? + .expect_stdout("REST_UPGRADE_REFUSED\n")?; + } + Ok(()) +} + +/// GraphQL rules allow listed query and mutation fields and registered +/// persisted queries, and deny unlisted fields, deny-rule mutations, +/// unregistered persisted queries, and a GET with a second, denied query. +/// Bodies sent with Content-Length and chunked both work, through urllib +/// and raw sockets. +#[testkit::test] +async fn graphql_rules_allow_and_deny_operations(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(|request| match request.method() { + "GET" if request.path() == "/" || request.path().starts_with("/graphql?") => ok_json(), + "GET" => ok_json().status(418), + _ => ok_json(), + }) + .await?; + let policy = policy::base_with(&format!( + r"network_policies: + graphql: + name: graphql + endpoints: + - host: {TEST_HOST} + port: {port} + protocol: graphql + enforcement: enforce + persisted_queries: allow_registered + graphql_persisted_queries: + abc123: + operation_type: query + operation_name: Viewer + fields: [viewer] + rules: + - allow: + operation_type: query + fields: [viewer] + - allow: + operation_type: mutation + fields: [createIssue] + deny_rules: + - operation_type: mutation + fields: [deleteRepository] + binaries: {binaries} +", + port = server.port, + binaries = python_binaries_yaml(), + ))?; + let sandbox = python_sandbox(cx, "graphql", policy).await?; + let output = run_python(&sandbox, GRAPHQL_CLIENT, &[TEST_HOST, &port(&server)]).await?; + let mut expected = Vec::new(); + for path in ["forward", "raw"] { + for (key, status) in [ + ("query_allowed", 200), + ("get_query_allowed", 200), + ("duplicate_get_denied", 403), + ("persisted_get_allowed", 200), + ("unregistered_persisted_get_denied", 403), + ("chunked_query_allowed", 200), + ("unlisted_field_denied", 403), + ("mutation_allowed", 200), + ("deny_rule_denied", 403), + ] { + expected.push((format!("{path}_{key}"), json!(status))); + } + } + expect_values(&json_result(&output)?, &expected) +} + +/// JSON-RPC rules allow listed methods whatever their params, deny a +/// deny-rule method, a batch with one denied call, and invalid JSON, through +/// urllib and raw sockets. +#[testkit::test] +async fn jsonrpc_rules_allow_and_deny_methods(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(jsonrpc_upstream).await?; + let policy = policy::base_with(&format!( + r"network_policies: + jsonrpc: + name: jsonrpc + endpoints: + - host: {TEST_HOST} + port: {port} + path: /rpc + protocol: json-rpc + enforcement: enforce + json_rpc: + max_body_bytes: 65536 + rules: + - allow: + method: initialize + - allow: + method: tools/list + - allow: + method: tools/call + deny_rules: + - method: tools/delete + binaries: {binaries} +", + port = server.port, + binaries = python_binaries_yaml(), + ))?; + let sandbox = python_sandbox(cx, "jsonrpc", policy).await?; + let output = run_python(&sandbox, JSONRPC_CLIENT, &[TEST_HOST, &port(&server)]).await?; + let mut expected = Vec::new(); + for path in ["forward", "raw"] { + for (key, status) in [ + ("method_initialize_allowed", 200), + ("method_tools_list_allowed", 200), + ("method_tools_call_allowed", 200), + ("method_tools_call_with_unmatched_params_allowed", 200), + ("method_tools_delete_denied", 403), + ] { + expected.push((format!("{path}_{key}"), json!(status))); + } + } + for (key, status) in [ + ("forward_batch_all_allowed", 200), + ("forward_batch_one_denied", 403), + ("forward_invalid_json_denied", 403), + ] { + expected.push((key.to_string(), json!(status))); + } + expect_values(&json_result(&output)?, &expected) +} + +/// A JSON-RPC endpoint without `enforcement` audits an unlisted method and +/// passes it, but still denies a response frame sent as a request. +#[testkit::test] +async fn jsonrpc_default_audit_mode_still_denies_response_frames( + cx: &mut Context, +) -> testkit::Result { + let server = HttpRecorder::serve(jsonrpc_upstream).await?; + let policy = policy::base_with(&format!( + r"network_policies: + jsonrpc_audit: + name: jsonrpc_audit + endpoints: + - host: {TEST_HOST} + port: {port} + path: /rpc + protocol: json-rpc + json_rpc: + max_body_bytes: 65536 + rules: + - allow: + method: initialize + binaries: {binaries} +", + port = server.port, + binaries = python_binaries_yaml(), + ))?; + let sandbox = python_sandbox(cx, "jsonaudit", policy).await?; + let output = run_python(&sandbox, JSONRPC_AUDIT_CLIENT, &[TEST_HOST, &port(&server)]).await?; + expect_values( + &json_result(&output)?, + &[ + ("forward_unknown_method_audited", json!(200)), + ("forward_response_frame_hard_denied", json!(403)), + ], + ) +} + +/// A REST endpoint on `port` in `enforcement` mode that allows only +/// `GET /allowed`. +fn rest_policy(port: u16, enforcement: &str) -> Result { + policy::base_with(&format!( + r"network_policies: + rest: + name: rest + endpoints: + - host: {TEST_HOST} + port: {port} + protocol: rest + enforcement: {enforcement} + rules: + - allow: + method: GET + path: /allowed + binaries: {binaries} +", + binaries = python_binaries_yaml(), + )) +} + +/// Answers a WebSocket upgrade with `101`, and `GET` or `DELETE` with +/// `200`, so that a refused upgrade shows the sandbox refused it. +fn rest_upstream(request: &Request) -> Response { + match request.method() { + "GET" if request.header("upgrade").is_some() => Response::new(101) + .header("Connection", "Upgrade") + .header("Upgrade", "websocket") + .header("Sec-WebSocket-Accept", "s3pPLMBiTxaQ9kYGzzhZRbK+xOo="), + "GET" | "DELETE" => ok_json(), + _ => Response::new(405), + } +} + +fn jsonrpc_upstream(request: &Request) -> Response { + match request.method() { + "POST" => Response::json(&json!({"jsonrpc": "2.0", "id": 1, "result": {}})), + _ => Response::new(200), + } +} + +fn ok_json() -> Response { + Response::json(&json!({"ok": true})) +} + +fn port(server: &HttpRecorder) -> String { + server.port.to_string() +} diff --git a/tests/suites/conformance/src/network_l7/mcp.rs b/tests/suites/conformance/src/network_l7/mcp.rs new file mode 100644 index 0000000000..3ce7742b1e --- /dev/null +++ b/tests/suites/conformance/src/network_l7/mcp.rs @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! MCP request profiles: the sandbox authorizes tool calls by MCP revision, +//! for the sessionless revision, legacy initialization, and an endpoint that +//! accepts several revisions. +//! +//! Ported from the `mcp_sessionless` e2e test. The fixture on the test host +//! serves initialization, discovery, tools, and a bounded subscription +//! stream, and reports the requests it received in each tool result, so the +//! client can tell a policy denial from a request the fixture accepted. + +use std::sync::{Arc, Mutex, PoisonError}; + +use host_server_support::{HttpRecorder, Request, Response}; +use serde_json::{Value, json}; +use testkit::{Context, Result, ResultExt as _, policy}; + +use crate::support::{TEST_HOST, python_binaries_yaml, python_sandbox, run_python}; + +const MCP_CLIENT: &str = include_str!("../scripts/mcp_client.py"); +const SESSIONLESS: &str = "2026-07-28"; + +/// The sessionless revision: discovery, an allowed and a denied tool, and a +/// subscription stream, with the request metadata the revision requires. +#[testkit::test] +async fn mcp_sessionless_discovery_tools_and_subscription(cx: &mut Context) -> testkit::Result { + run_profile(cx, "mcpsl", &[SESSIONLESS]).await +} + +/// Each legacy revision: initialization, then an allowed and a denied tool. +#[testkit::test] +async fn mcp_legacy_revisions_authorize_tools(cx: &mut Context) -> testkit::Result { + for (suffix, version) in [("mcp0326", "2025-03-26"), ("mcp0618", "2025-06-18")] { + run_profile(cx, suffix, &[version]).await?; + } + Ok(()) +} + +/// An endpoint that accepts a legacy and the sessionless revision authorizes +/// tools under each, against one fixture. +#[testkit::test] +async fn mcp_multi_version_endpoint_authorizes_tools(cx: &mut Context) -> testkit::Result { + run_profile(cx, "mcpmv", &["2025-11-25", SESSIONLESS]).await +} + +/// Start a fresh fixture for `versions`, and run [`MCP_CLIENT`] for them in +/// a sandbox whose policy accepts them. +async fn run_profile(cx: &Context, suffix: &str, versions: &[&str]) -> Result { + let supported: Vec = versions.iter().map(ToString::to_string).collect(); + let received = Arc::new(Mutex::new(Vec::new())); + let server = + HttpRecorder::serve(move |request| fixture(request, &supported, &received)).await?; + + let legacy_rules = if versions.iter().any(|version| *version != SESSIONLESS) { + " - allow:\n method: initialize\n - allow:\n method: notifications/initialized\n" + } else { + "" + }; + let versions_json = serde_json::to_string(versions).context("encode the MCP revisions")?; + let policy = policy::base_with(&format!( + r"network_policies: + mcp: + name: mcp + endpoints: + - host: {TEST_HOST} + port: {port} + path: /mcp + protocol: mcp + enforcement: enforce + mcp: + versions: {versions_json} + max_body_bytes: 65536 + rules: +{legacy_rules} - allow: + method: server/discover + - allow: + method: tools/call + tool: read_status + - allow: + method: subscriptions/listen + deny_rules: + - method: tools/call + tool: read_details + binaries: {binaries} +", + port = server.port, + binaries = python_binaries_yaml(), + ))?; + let sandbox = python_sandbox(cx, suffix, policy).await?; + let output = run_python( + &sandbox, + MCP_CLIENT, + &[TEST_HOST, &server.port.to_string(), &versions_json], + ) + .await + .context(format!("MCP revisions {versions:?}"))?; + for version in versions { + let marker = format!("MCP_OK version={version}"); + testkit::ensure!( + output.stdout.lines().any(|line| line == marker), + "{versions:?}: the client didn't finish {version}:\n{}", + output.stdout + ); + } + Ok(()) +} + +/// The MCP fixture. It records each parsed request as `[version, method, +/// tool]` in `received` before checking it, so rejected requests stay +/// visible in later tool results. +fn fixture(request: &Request, supported: &[String], received: &Mutex>) -> Response { + if request.method() == "GET" { + return Response::new(if request.path() == "/" { 200 } else { 405 }); + } + let Ok(message) = serde_json::from_str::(&request.body) else { + return bad_request("the request body is not JSON"); + }; + let method = message["method"].as_str().unwrap_or_default(); + let params = &message["params"]; + let version = if method == "initialize" { + params["protocolVersion"].as_str().map(ToString::to_string) + } else { + request.header("mcp-protocol-version") + }; + let received_now = { + let mut received = received.lock().unwrap_or_else(PoisonError::into_inner); + received.push(json!([version, method, params.get("name")])); + received.clone() + }; + let Some(version) = version.filter(|version| supported.contains(version)) else { + return bad_request("request revision did not reach the fixture intact"); + }; + if request.path() != "/mcp" { + return bad_request("request revision did not reach the fixture intact"); + } + let sessionless = version == SESSIONLESS; + if sessionless { + let meta = ¶ms["_meta"]; + let intact = request.header("mcp-protocol-version").as_deref() == Some(&version) + && request.header("mcp-method").as_deref() == Some(method) + && meta["io.modelcontextprotocol/protocolVersion"] == version.as_str() + && meta["io.modelcontextprotocol/clientCapabilities"] == json!({}) + && (method != "tools/call" + || request.header("mcp-name").as_deref() == params["name"].as_str()); + if !intact { + return bad_request("request metadata did not reach the fixture intact"); + } + } + + let result = match method { + "initialize" if !sessionless => json!({ + "protocolVersion": version, + "capabilities": {"tools": {}}, + "serverInfo": {"name": "openshell-profile-fixture", "version": "1"}, + }), + "notifications/initialized" if !sessionless => return Response::new(202), + "server/discover" if sessionless => json!({ + "supportedVersions": [version], + "capabilities": {"tools": {"listChanged": true}}, + "ttlMs": 0, + "cacheScope": "private", + "_meta": { + "io.modelcontextprotocol/serverInfo": { + "name": "openshell-sessionless-fixture", "version": "1" + } + }, + }), + // Both tool names work upstream; the sandbox owns the policy denial. + "tools/call" => { + let mut result = json!({ + "content": [{"type": "text", "text": params["name"]}], + "isError": false, + "_meta": {"fixtureRequests": received_now}, + }); + if sessionless { + result["resultType"] = json!("complete"); + } + result + } + "subscriptions/listen" if sessionless => return subscription(&message), + _ => return bad_request("unexpected method for the selected fixture revision"), + }; + Response::json(&json!({"jsonrpc": "2.0", "id": message["id"], "result": result})) +} + +/// A subscription stream that acknowledges, sends one notification, and +/// completes. +fn subscription(message: &Value) -> Response { + let meta = json!({"io.modelcontextprotocol/subscriptionId": message["id"]}); + let events = [ + json!({ + "jsonrpc": "2.0", + "method": "notifications/subscriptions/acknowledged", + "params": {"notifications": message["params"]["notifications"], "_meta": meta}, + }), + json!({ + "jsonrpc": "2.0", + "method": "notifications/tools/list_changed", + "params": {"_meta": meta}, + }), + json!({ + "jsonrpc": "2.0", + "id": message["id"], + "result": {"resultType": "complete", "_meta": meta}, + }), + ]; + let body: String = events + .iter() + .map(|event| format!("event: message\ndata: {event}\n\n")) + .collect(); + Response::new(200) + .header("Content-Type", "text/event-stream") + .body(body) +} + +fn bad_request(reason: &str) -> Response { + Response::new(400) + .header("Content-Type", "text/plain") + .body(reason) +} diff --git a/tests/suites/conformance/src/scripts/graphql_client.py b/tests/suites/conformance/src/scripts/graphql_client.py new file mode 100644 index 0000000000..f8d4c07c6a --- /dev/null +++ b/tests/suites/conformance/src/scripts/graphql_client.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends GraphQL queries, mutations, and persisted queries to HOST:PORT with +# urllib (`forward_*`) and raw sockets (`raw_*`), and prints each status as +# JSON. +import json +import socket +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +DETAILS = {} + +QUERY_VIEWER = "query Viewer { viewer { login } }" +QUERY_REPOSITORY = "query Repo { repository(owner:\"o\", name:\"r\") { id } }" +MUTATION_CREATE = "mutation Create { createIssue(input:{repositoryId:\"r\", title:\"t\", body:\"b\"}) { issue { id } } }" +MUTATION_DELETE = "mutation Delete { deleteRepository(input:{repositoryId:\"r\"}) { clientMutationId } }" + +def forward_status(query): + body = json.dumps({"query": query}).encode() + request = urllib.request.Request( + f"http://{HOST}:{PORT}/graphql", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +def forward_get_status(query): + encoded = urllib.parse.urlencode({"query": query}) + request = urllib.request.Request( + f"http://{HOST}:{PORT}/graphql?{encoded}", + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +def forward_duplicate_get_status(): + safe = urllib.parse.quote_plus(QUERY_VIEWER) + unsafe = urllib.parse.quote_plus(MUTATION_DELETE) + request = urllib.request.Request( + f"http://{HOST}:{PORT}/graphql?query={safe}&query={unsafe}", + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +def forward_persisted_get_status(hash_value): + extensions = json.dumps({"persistedQuery": {"version": 1, "sha256Hash": hash_value}}) + encoded = urllib.parse.urlencode({"operationName": "Viewer", "extensions": extensions}) + request = urllib.request.Request( + f"http://{HOST}:{PORT}/graphql?{encoded}", + method="GET", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +def forward_chunked_status(query): + target = f"{HOST}:{PORT}" + body = json.dumps({"query": query}).encode() + chunk = f"{len(body):x}\r\n".encode() + body + b"\r\n0\r\n\r\n" + + with socket.create_connection((HOST, PORT), timeout=15) as sock: + request = ( + f"POST /graphql HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Content-Type: application/json\r\n" + f"Transfer-Encoding: chunked\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + chunk + sock.sendall(request) + response, body = read_response(sock) + DETAILS["forward_chunked_query_allowed_detail"] = body.decode(errors="replace") + return status_code(response, "forward_chunked_response") + +def read_until(sock, marker): + data = b"" + while marker not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + +def read_response(sock): + response = read_until(sock, b"\r\n\r\n") + headers, _, body = response.partition(b"\r\n\r\n") + content_length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + content_length = int(line.split(b":", 1)[1].strip()) + break + while len(body) < content_length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + return response, body + +def status_code(response, label): + parts = response.split() + if len(parts) < 2: + DETAILS[f"{label}_raw"] = response.decode(errors="replace") + raise RuntimeError(f"{label}: malformed HTTP response: {response!r}") + try: + return int(parts[1]) + except ValueError as error: + DETAILS[f"{label}_raw"] = response.decode(errors="replace") + raise RuntimeError(f"{label}: non-numeric HTTP status: {response!r}") from error + +def raw_http_status(label, request): + last_error = None + for attempt in range(5): + try: + with socket.create_connection((HOST, PORT), timeout=15) as sock: + sock.sendall(request) + sock.shutdown(socket.SHUT_WR) + response = read_until(sock, b"\r\n\r\n") + return status_code(response, f"{label}_response") + except (OSError, RuntimeError) as error: + last_error = error + DETAILS[f"{label}_attempt_{attempt + 1}_error"] = str(error) + time.sleep(0.2) + + raise RuntimeError(f"{label}: failed after 5 attempts: {last_error}") + +def raw_status(query, label): + target = f"{HOST}:{PORT}" + body = json.dumps({"query": query}).encode() + + request = ( + f"POST /graphql HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Content-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + body + return raw_http_status(label, request) + +def raw_get_status(query, label): + target = f"{HOST}:{PORT}" + encoded = urllib.parse.urlencode({"query": query}) + + request = ( + f"GET /graphql?{encoded} HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + return raw_http_status(label, request) + +def raw_duplicate_get_status(): + target = f"{HOST}:{PORT}" + safe = urllib.parse.quote_plus(QUERY_VIEWER) + unsafe = urllib.parse.quote_plus(MUTATION_DELETE) + + request = ( + f"GET /graphql?query={safe}&query={unsafe} HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + return raw_http_status("raw_duplicate_get_denied", request) + +def raw_persisted_get_status(hash_value, label): + target = f"{HOST}:{PORT}" + extensions = json.dumps({"persistedQuery": {"version": 1, "sha256Hash": hash_value}}) + encoded = urllib.parse.urlencode({"operationName": "Viewer", "extensions": extensions}) + + request = ( + f"GET /graphql?{encoded} HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + return raw_http_status(label, request) + +def raw_chunked_status(query): + target = f"{HOST}:{PORT}" + body = json.dumps({"query": query}).encode() + chunk = f"{len(body):x}\r\n".encode() + body + b"\r\n0\r\n\r\n" + + request = ( + f"POST /graphql HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Content-Type: application/json\r\n" + f"Transfer-Encoding: chunked\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + chunk + return raw_http_status("raw_chunked_query_allowed", request) + +results = { + "forward_query_allowed": forward_status(QUERY_VIEWER), + "forward_get_query_allowed": forward_get_status(QUERY_VIEWER), + "forward_duplicate_get_denied": forward_duplicate_get_status(), + "forward_persisted_get_allowed": forward_persisted_get_status("abc123"), + "forward_unregistered_persisted_get_denied": forward_persisted_get_status("missing"), + "forward_chunked_query_allowed": forward_chunked_status(QUERY_VIEWER), + "forward_unlisted_field_denied": forward_status(QUERY_REPOSITORY), + "forward_mutation_allowed": forward_status(MUTATION_CREATE), + "forward_deny_rule_denied": forward_status(MUTATION_DELETE), + "raw_query_allowed": raw_status(QUERY_VIEWER, "raw_query_allowed"), + "raw_get_query_allowed": raw_get_status(QUERY_VIEWER, "raw_get_query_allowed"), + "raw_duplicate_get_denied": raw_duplicate_get_status(), + "raw_persisted_get_allowed": raw_persisted_get_status("abc123", "raw_persisted_get_allowed"), + "raw_unregistered_persisted_get_denied": raw_persisted_get_status("missing", "raw_unregistered_persisted_get_denied"), + "raw_chunked_query_allowed": raw_chunked_status(QUERY_VIEWER), + "raw_unlisted_field_denied": raw_status(QUERY_REPOSITORY, "raw_unlisted_field_denied"), + "raw_mutation_allowed": raw_status(MUTATION_CREATE, "raw_mutation_allowed"), + "raw_deny_rule_denied": raw_status(MUTATION_DELETE, "raw_deny_rule_denied"), +} +results.update(DETAILS) +print(json.dumps(results, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/jsonrpc_audit_client.py b/tests/suites/conformance/src/scripts/jsonrpc_audit_client.py new file mode 100644 index 0000000000..c1145be040 --- /dev/null +++ b/tests/suites/conformance/src/scripts/jsonrpc_audit_client.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends an unlisted JSON-RPC method and a response frame to HOST:PORT/rpc, +# and prints each status as JSON. +import json +import sys +import urllib.error +import urllib.request + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) + +def post_jsonrpc(body): + encoded = json.dumps(body).encode() + request = urllib.request.Request( + f"http://{HOST}:{PORT}/rpc", + data=encoded, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + +results = { + "forward_unknown_method_audited": post_jsonrpc({"jsonrpc": "2.0", "id": 1, "method": "unknown/method"}), + "forward_response_frame_hard_denied": post_jsonrpc({"jsonrpc": "2.0", "id": 1, "result": {}}), +} +print(json.dumps(results, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/jsonrpc_client.py b/tests/suites/conformance/src/scripts/jsonrpc_client.py new file mode 100644 index 0000000000..919009584f --- /dev/null +++ b/tests/suites/conformance/src/scripts/jsonrpc_client.py @@ -0,0 +1,197 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends JSON-RPC calls, batches, and invalid JSON to HOST:PORT/rpc with +# urllib (`forward_*`) and raw sockets (`raw_*`), and prints each status as +# JSON. +import json +import socket +import sys +import time +import urllib.error +import urllib.request + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +DETAILS = { + "debug_target": {"host": HOST, "port": PORT}, +} + +def text(data): + return data.decode(errors="replace") + +def selected_headers(headers): + return { + key.lower(): value + for key, value in headers.items() + if key.lower() in ("content-type", "content-length", "server") + } + +def record_http_error(label, error, request_body): + response_body = error.read() + DETAILS[f"{label}_request"] = request_body + DETAILS[f"{label}_response"] = { + "status": error.code, + "reason": str(error.reason), + "headers": selected_headers(error.headers), + "body": text(response_body), + } + return error.code + +def post_jsonrpc(label, method, params=None, req_id=1): + body = {"jsonrpc": "2.0", "id": req_id, "method": method} + if params is not None: + body["params"] = params + encoded = json.dumps(body).encode() + request = urllib.request.Request( + f"http://{HOST}:{PORT}/rpc", + data=encoded, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + return record_http_error(label, error, body) + +def post_jsonrpc_batch(label, requests): + encoded = json.dumps(requests).encode() + request = urllib.request.Request( + f"http://{HOST}:{PORT}/rpc", + data=encoded, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + return record_http_error(label, error, requests) + +def post_invalid_json(label): + encoded = b"not valid json {" + request = urllib.request.Request( + f"http://{HOST}:{PORT}/rpc", + data=encoded, + headers={"Content-Type": "application/json", "Content-Length": str(len(encoded))}, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + return record_http_error(label, error, text(encoded)) + +def read_until(sock, marker): + data = b"" + while marker not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + +def read_response(sock): + response = read_until(sock, b"\r\n\r\n") + headers, _, body = response.partition(b"\r\n\r\n") + content_length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + content_length = int(line.split(b":", 1)[1].strip()) + break + while len(body) < content_length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + return response, body + +def status_code(response, label): + parts = response.split() + if len(parts) < 2: + DETAILS[f"{label}_raw"] = response.decode(errors="replace") + raise RuntimeError(f"{label}: malformed HTTP response: {response!r}") + try: + return int(parts[1]) + except ValueError as error: + DETAILS[f"{label}_raw"] = response.decode(errors="replace") + raise RuntimeError(f"{label}: non-numeric HTTP status: {response!r}") from error + +def record_raw_response(label, response, body=b""): + code = status_code(response, label) + if code != 200: + DETAILS[f"{label}_raw"] = text(response) + if body: + DETAILS[f"{label}_body"] = text(body) + return code + +def raw_http_status(label, request): + last_error = None + for attempt in range(5): + try: + with socket.create_connection((HOST, PORT), timeout=15) as sock: + sock.sendall(request) + sock.shutdown(socket.SHUT_WR) + response, body = read_response(sock) + return record_raw_response(f"{label}_response", response, body) + except (OSError, RuntimeError) as error: + last_error = error + DETAILS[f"{label}_attempt_{attempt + 1}_error"] = str(error) + time.sleep(0.2) + + raise RuntimeError(f"{label}: failed after 5 attempts: {last_error}") + +def raw_jsonrpc_status(method, params, label): + target = f"{HOST}:{PORT}" + body = {"jsonrpc": "2.0", "id": 1, "method": method} + if params is not None: + body["params"] = params + encoded = json.dumps(body).encode() + request = ( + f"POST /rpc HTTP/1.1\r\n" + f"Host: {target}\r\n" + f"Content-Type: application/json\r\n" + f"Content-Length: {len(encoded)}\r\n" + f"Connection: close\r\n" + f"\r\n" + ).encode() + encoded + return raw_http_status(label, request) + +results = { + # forward proxy — method-only allow rules + "forward_method_initialize_allowed": post_jsonrpc("forward_method_initialize_allowed", "initialize", {"protocolVersion": "2025-11-25", "capabilities": {}}), + "forward_method_tools_list_allowed": post_jsonrpc("forward_method_tools_list_allowed", "tools/list"), + + # forward proxy — method allow/deny rules + "forward_method_tools_call_allowed": post_jsonrpc("forward_method_tools_call_allowed", "tools/call", {"name": "read_status"}), + "forward_method_tools_call_with_unmatched_params_allowed": post_jsonrpc("forward_method_tools_call_with_unmatched_params_allowed", "tools/call", {"name": "blocked_action", "arguments": {"scope": "ignored"}}), + "forward_method_tools_delete_denied": post_jsonrpc("forward_method_tools_delete_denied", "tools/delete", {"name": "purge_cache"}), + + # forward proxy — batch: all requests allowed + "forward_batch_all_allowed": post_jsonrpc_batch("forward_batch_all_allowed", [ + {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}, + {"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "read_status"}}, + ]), + + # forward proxy — batch: one denied request causes full batch denial + "forward_batch_one_denied": post_jsonrpc_batch("forward_batch_one_denied", [ + {"jsonrpc": "2.0", "id": 1, "method": "tools/list"}, + {"jsonrpc": "2.0", "id": 2, "method": "tools/delete", "params": {"name": "purge_cache"}}, + ]), + + # forward proxy — invalid JSON body fails closed before generic rules apply + "forward_invalid_json_denied": post_invalid_json("forward_invalid_json_denied"), + + # raw socket path — representative allowed and denied cases + "raw_method_initialize_allowed": raw_jsonrpc_status("initialize", {"protocolVersion": "2025-11-25", "capabilities": {}}, "raw_method_initialize_allowed"), + "raw_method_tools_list_allowed": raw_jsonrpc_status("tools/list", None, "raw_method_tools_list_allowed"), + "raw_method_tools_call_allowed": raw_jsonrpc_status("tools/call", {"name": "read_status"}, "raw_method_tools_call_allowed"), + "raw_method_tools_call_with_unmatched_params_allowed": raw_jsonrpc_status("tools/call", {"name": "blocked_action", "arguments": {"scope": "ignored"}}, "raw_method_tools_call_with_unmatched_params_allowed"), + "raw_method_tools_delete_denied": raw_jsonrpc_status("tools/delete", {"name": "purge_cache"}, "raw_method_tools_delete_denied"), +} +results.update(DETAILS) +print(json.dumps(results, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/mcp_client.py b/tests/suites/conformance/src/scripts/mcp_client.py new file mode 100644 index 0000000000..7845555f19 --- /dev/null +++ b/tests/suites/conformance/src/scripts/mcp_client.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Runs an MCP client against the fixture at HOST:PORT for each revision in the +# JSON list VERSIONS: discovery or initialization, an allowed and a denied +# tool, and for the sessionless revision, a subscription. Prints +# `MCP_OK version=` after each revision's checks pass. +import json +import sys +import urllib.error +import urllib.request + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +VERSIONS = json.loads(sys.argv[3]) +SESSIONLESS = "2026-07-28" + +# Direct client connections pass through the sandbox's transparent interception. +opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + +def post(request_id, method, params, version): + params = dict(params) + headers = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + } + if method != "initialize": + headers["MCP-Protocol-Version"] = version + if version == SESSIONLESS: + params["_meta"] = { + "io.modelcontextprotocol/protocolVersion": version, + "io.modelcontextprotocol/clientCapabilities": {}, + "io.modelcontextprotocol/clientInfo": {"name": "openshell-e2e", "version": "1"}, + } + headers["Mcp-Method"] = method + if method == "tools/call": + headers["Mcp-Name"] = params["name"] + message = {"jsonrpc": "2.0", "method": method, "params": params} + if request_id is not None: + message["id"] = request_id + request = urllib.request.Request( + f"http://{HOST}:{PORT}/mcp", + data=json.dumps(message).encode(), + headers=headers, + method="POST", + ) + try: + with opener.open(request, timeout=15) as response: + return response.status, response.headers.get_content_type(), response.read() + except urllib.error.HTTPError as error: + return error.code, error.headers.get_content_type(), error.read() + + +expected_receipts = [] +for version in VERSIONS: + if version == SESSIONLESS: + status, content_type, body = post(1, "server/discover", {}, version) + assert status == 200, (version, "discovery", status, body) + assert content_type == "application/json", (version, content_type) + discovery = json.loads(body) + assert discovery["id"] == 1, discovery + assert discovery["result"]["supportedVersions"] == [version], discovery + expected_receipts.append([version, "server/discover", None]) + else: + status, _, body = post(1, "initialize", { + "protocolVersion": version, + "capabilities": {}, + "clientInfo": {"name": "openshell-e2e", "version": "1"}, + }, version) + assert status == 200, (version, "initialize", status, body) + assert json.loads(body)["result"]["protocolVersion"] == version, body + expected_receipts.append([version, "initialize", None]) + status, _, body = post(None, "notifications/initialized", {}, version) + assert status == 202, (version, "initialized", status, body) + expected_receipts.append([version, "notifications/initialized", None]) + + status, _, body = post(2, "tools/call", {"name": "read_status", "arguments": {}}, version) + assert status == 200, (version, "allowed tool", status, body) + tool = json.loads(body) + assert tool["id"] == 2, tool + assert tool["result"]["content"] == [{"type": "text", "text": "read_status"}], tool + expected_receipts.append([version, "tools/call", "read_status"]) + assert tool["result"]["_meta"]["fixtureRequests"] == expected_receipts, tool + + status, _, body = post(3, "tools/call", {"name": "read_details", "arguments": {}}, version) + assert status == 403, (version, "denied tool", status, body) + + # The fixture accepts both tools. A later allowed call proves the denial + # came from the proxy and no denied operation reached the upstream. + status, _, body = post(4, "tools/call", {"name": "read_status", "arguments": {}}, version) + assert status == 200, (version, "receipt tool", status, body) + receipt = json.loads(body) + assert receipt["id"] == 4, receipt + expected_receipts.append([version, "tools/call", "read_status"]) + assert receipt["result"]["_meta"]["fixtureRequests"] == expected_receipts, receipt + + if version == SESSIONLESS: + status, content_type, body = post( + 5, "subscriptions/listen", {"notifications": {"toolsListChanged": True}}, version + ) + assert status == 200, (version, "subscription", status, body) + assert content_type == "text/event-stream", (content_type, body) + events = [json.loads(line[6:]) for line in body.decode().splitlines() if line.startswith("data: ")] + assert len(events) == 3, events + assert events[0]["method"] == "notifications/subscriptions/acknowledged", events + assert events[0]["params"]["notifications"] == {"toolsListChanged": True}, events + assert events[1]["method"] == "notifications/tools/list_changed", events + assert events[1]["params"]["_meta"]["io.modelcontextprotocol/subscriptionId"] == 5, events + assert events[2]["id"] == 5 and events[2]["result"]["resultType"] == "complete", events + expected_receipts.append([version, "subscriptions/listen", None]) + + print(f"MCP_OK version={version}", flush=True) diff --git a/tests/suites/conformance/src/scripts/rest_client.py b/tests/suites/conformance/src/scripts/rest_client.py new file mode 100644 index 0000000000..d59543a760 --- /dev/null +++ b/tests/suites/conformance/src/scripts/rest_client.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Prints the status of GET and POST to /allowed as JSON. +import json +import sys +import urllib.error +import urllib.request + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) + + +def status(method, data=None): + request = urllib.request.Request(f"http://{HOST}:{PORT}/allowed", data=data, method=method) + try: + with urllib.request.urlopen(request, timeout=15) as response: + response.read() + return response.status + except urllib.error.HTTPError as error: + error.read() + return error.code + + +print(json.dumps({"get": status("GET"), "post": status("POST", b"test")}, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/rest_upgrade_client.py b/tests/suites/conformance/src/scripts/rest_upgrade_client.py new file mode 100644 index 0000000000..305742fe46 --- /dev/null +++ b/tests/suites/conformance/src/scripts/rest_upgrade_client.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Asks to upgrade a REST endpoint to WebSocket with an origin-form and an +# absolute-form target, and fails unless the sandbox refuses both. +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) + +for target in ["/allowed", f"http://{HOST}:{PORT}/allowed"]: + s = socket.create_connection((HOST, PORT), timeout=15) + request = ("GET " + target + f" HTTP/1.1\r\nHost: {HOST}:{PORT}\r\n" + "Connection: Upgrade\r\nUpgrade: websocket\r\n" + "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n") + s.sendall(request.encode()) + response = b"" + while True: + chunk = s.recv(4096) + if not chunk: + break + response += chunk + s.close() + assert response.startswith(b"HTTP/1.1 403"), response + assert b"unsupported_l7_protocol" in response, response + assert b"REST endpoints" in response, response +print("REST_UPGRADE_REFUSED") From 332d3616b1907acab6631dbb99b273a7a3e6d71d Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:57:37 -0800 Subject: [PATCH 06/15] test(conformance): port transparent egress pipeline tests Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/lib.rs | 1 + .../suites/conformance/src/network_egress.rs | 439 ++++++++++++++++++ .../conformance/src/scripts/echo_client.py | 23 + .../src/scripts/pipeline_client.py | 28 ++ .../src/scripts/redaction_client.py | 53 +++ .../src/scripts/reload_keep_alive_client.py | 66 +++ .../conformance/src/scripts/status_client.py | 44 ++ .../src/scripts/uninspectable_client.py | 30 ++ 8 files changed, 684 insertions(+) create mode 100644 tests/suites/conformance/src/network_egress.rs create mode 100644 tests/suites/conformance/src/scripts/echo_client.py create mode 100644 tests/suites/conformance/src/scripts/pipeline_client.py create mode 100644 tests/suites/conformance/src/scripts/redaction_client.py create mode 100644 tests/suites/conformance/src/scripts/reload_keep_alive_client.py create mode 100644 tests/suites/conformance/src/scripts/status_client.py create mode 100644 tests/suites/conformance/src/scripts/uninspectable_client.py diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 9f36b3c254..4d837bd3f3 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -18,6 +18,7 @@ mod image; mod labels; mod lifecycle; mod network; +mod network_egress; mod network_l7; mod persistent_workspace; mod policy_advisor; diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs new file mode 100644 index 0000000000..8a63120681 --- /dev/null +++ b/tests/suites/conformance/src/network_egress.rs @@ -0,0 +1,439 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Transparent egress: what the supervisor does with connections a workload +//! makes straight to a destination, without proxy settings. Policy reloads +//! apply to open connections, `tls: skip` relays raw bytes, middleware +//! inspects or blocks traffic, and pipelined requests are authorized one at +//! a time. +//! +//! Ported from the `proxy_egress_pipeline` e2e tests. The upstream servers +//! run on the test host ([`host_server_support`]). + +use std::time::Duration; + +use host_server_support::{HttpRecorder, Request, Response, TcpServer}; +use serde_json::{Value, json}; +use testkit::policy::{self, Endpoint as _}; +use testkit::sdk::raw::proto; +use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; + +use crate::support::{ + TEST_HOST, expect_values, json_result, python_binaries_yaml, python_sandbox, run_python, +}; + +const STATUS_CLIENT: &str = include_str!("scripts/status_client.py"); +const RELOAD_KEEP_ALIVE_CLIENT: &str = include_str!("scripts/reload_keep_alive_client.py"); +const ECHO_CLIENT: &str = include_str!("scripts/echo_client.py"); +const UNINSPECTABLE_CLIENT: &str = include_str!("scripts/uninspectable_client.py"); +const REDACTION_CLIENT: &str = include_str!("scripts/redaction_client.py"); +const PIPELINE_CLIENT: &str = include_str!("scripts/pipeline_client.py"); + +/// Files [`RELOAD_KEEP_ALIVE_CLIENT`] signals through. +const RELOAD_READY: &str = "/tmp/proxy-reload-ready"; +const RELOAD_GO: &str = "/tmp/proxy-reload-go"; +const RELOAD_RESULT: &str = "/tmp/proxy-reload-result"; +const RELOAD_LOG: &str = "/tmp/proxy-reload-client.log"; + +/// The reply [`TcpServer::reply_when_idle`] gives a pipelined request, so the +/// connection stays usable for the next one. +const PIPELINE_REPLY: &[u8] = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok"; + +/// A policy reload that removes access applies to new connections and to a +/// keep-alive connection opened before it: the next request on that +/// connection isn't forwarded. +#[testkit::test] +async fn policy_reload_denies_new_and_open_connections(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(|_| Response::text("ok").keep_alive()).await?; + let allowed = allow_server(server.port); + let sandbox = python_sandbox(cx, "reload", allowed.clone()).await?; + policy::set(&sandbox, &allowed).await?; + + let port = server.port.to_string(); + sandbox + .exec([ + "sh", + "-c", + &format!("nohup python3 -c \"$1\" {TEST_HOST} {port} >{RELOAD_LOG} 2>&1 &"), + "reload-client", + RELOAD_KEEP_ALIVE_CLIENT, + ]) + .await? + .success() + .context("start the keep-alive client")?; + wait_for_file(&sandbox, RELOAD_READY).await?; + expect_statuses(&sandbox, &port, |status| status == 200) + .await + .context("before the reload")?; + + policy::set(&sandbox, &policy::base()).await?; + sandbox + .exec(["touch", RELOAD_GO]) + .await? + .success() + .context("release the keep-alive client")?; + let result: Value = serde_json::from_str(wait_for_file(&sandbox, RELOAD_RESULT).await?.trim()) + .context("parse the keep-alive client's result")?; + testkit::ensure!( + result["failed_closed"] == true, + "the open connection forwarded a request after the reload: {result}" + ); + expect_statuses(&sandbox, &port, |status| status != 200) + .await + .context("after the reload") +} + +/// The gateway rejects a policy update with two endpoints for one host and +/// port, one of them `tls: skip`. The sandbox keeps its policy revisions and +/// still forwards requests under the active policy. +#[testkit::test] +async fn ambiguous_policy_update_is_rejected_and_active_policy_kept( + cx: &mut Context, +) -> testkit::Result { + let server = HttpRecorder::serve(|_| Response::text("ok").keep_alive()).await?; + let allowed = allow_server(server.port); + let sandbox = python_sandbox(cx, "ambig", allowed.clone()).await?; + policy::set(&sandbox, &allowed).await?; + let port = server.port.to_string(); + expect_statuses(&sandbox, &port, |status| status == 200) + .await + .context("before the rejected update")?; + let revisions = policy_revisions(cx, &sandbox).await?; + let connections = server.connections(); + + let mut ambiguous = policy::base(); + policy::allow( + &mut ambiguous, + "terminating", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ); + policy::allow( + &mut ambiguous, + "passthrough", + TEST_HOST, + server.port, + PYTHON_BINARIES, + ) + .tls_skip(); + let file = testkit::YamlFile::new(&policy::to_yaml(&ambiguous)?)?; + let output = cx + .cli([ + "policy", + "set", + sandbox.name(), + "--policy", + file.path()?, + "--wait", + "--timeout", + "120", + ]) + .await? + .failure()?; + testkit::ensure!( + output.report_text().contains("ambiguity validation failed"), + "the rejection doesn't explain the ambiguity:\n{}", + output.report_text() + ); + + let after = policy_revisions(cx, &sandbox).await?; + testkit::ensure!( + after == revisions, + "the rejected policy changed the revisions:\nbefore: {revisions}\nafter: {after}" + ); + expect_statuses(&sandbox, &port, |status| status == 200) + .await + .context("after the rejected update")?; + testkit::ensure!( + server.connections() > connections, + "requests under the active policy didn't reach the server" + ); + Ok(()) +} + +/// An endpoint with `tls: skip` relays bytes that are neither HTTP nor TLS +/// unchanged in both directions. +#[testkit::test] +async fn tls_skip_relays_opaque_bytes(cx: &mut Context) -> testkit::Result { + let server = TcpServer::echo().await?; + let mut policy = policy::base(); + allow(&mut policy, server.port).tls_skip(); + let sandbox = python_sandbox(cx, "tlsskip", policy).await?; + let mut payload = vec![0x00, 0xff, 0x13, 0x37, 0x80, 0x0a]; + payload.extend_from_slice(b"not-http-or-tls"); + payload.extend(0..64_u8); + run_python( + &sandbox, + ECHO_CLIENT, + &[TEST_HOST, &server.port.to_string(), &hex(&payload)], + ) + .await? + .expect_stdout("ECHO_OK\n")?; + Ok(()) +} + +/// A redacting middleware rewrites an API key in each request body before +/// it reaches upstream. +#[testkit::test] +async fn middleware_redacts_request_bodies(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::serve(|request: &Request| { + Response::new(200) + .header("Content-Type", "application/json") + .body(request.body.clone()) + }) + .await?; + let policy = middleware_policy(server.port, "fail_closed", "")?; + let sandbox = python_sandbox(cx, "redact", policy).await?; + let output = run_python( + &sandbox, + REDACTION_CLIENT, + &[TEST_HOST, &server.port.to_string()], + ) + .await?; + let result = json_result(&output)?; + for request in ["first", "second"] { + expect_values(&result[request], &[("api_key", json!("[REDACTED]"))]) + .context(format!("the {request} request"))?; + } + Ok(()) +} + +/// A `fail_closed` middleware blocks traffic it can't inspect before any +/// byte reaches upstream, and the sandbox logs why. +#[testkit::test] +async fn fail_closed_middleware_blocks_uninspectable_traffic(cx: &mut Context) -> testkit::Result { + let server = TcpServer::echo().await?; + let policy = middleware_policy(server.port, "fail_closed", "")?; + let sandbox = python_sandbox(cx, "failclosed", policy).await?; + run_python( + &sandbox, + UNINSPECTABLE_CLIENT, + &[TEST_HOST, &server.port.to_string()], + ) + .await? + .expect_stdout("BLOCKED\n")?; + // Give a forwarded payload time to arrive before checking it didn't. + tokio::time::sleep(Duration::from_millis(200)).await; + let received = server.received(); + testkit::ensure!( + received.is_empty(), + "the payload reached upstream: {:?}", + String::from_utf8_lossy(&received) + ); + sandbox + .logs_until("the middleware denial", |logs| { + logs.contains("openshell.middleware.traffic_uninspectable") + && logs.contains( + "Unsupported tunnel protocol cannot be inspected by required middleware", + ) + }) + .await?; + Ok(()) +} + +/// A `fail_open` middleware on a `tls: skip` endpoint lets traffic it can't +/// inspect through unchanged. +#[testkit::test] +async fn fail_open_middleware_relays_uninspectable_traffic(cx: &mut Context) -> testkit::Result { + let server = TcpServer::echo().await?; + let policy = middleware_policy(server.port, "fail_open", " tls: skip\n")?; + let sandbox = python_sandbox(cx, "failopen", policy).await?; + let mut payload = vec![0x00, 0xff, 0x13, 0x37, 0x80]; + payload.extend_from_slice(b"middleware-bypass"); + run_python( + &sandbox, + ECHO_CLIENT, + &[TEST_HOST, &server.port.to_string(), &hex(&payload)], + ) + .await? + .expect_stdout("ECHO_OK\n")?; + let received = server.received(); + testkit::ensure!( + received == payload, + "upstream received {:?}; expected the payload unchanged", + String::from_utf8_lossy(&received) + ); + Ok(()) +} + +/// A denied request pipelined behind an allowed one is answered with `403` +/// and never reaches upstream, not even as extra bytes after the first +/// request. The relay strips the hop-by-hop `Connection` header. +#[testkit::test] +async fn pipelined_request_is_authorized_separately(cx: &mut Context) -> testkit::Result { + let server = TcpServer::reply_when_idle(PIPELINE_REPLY).await?; + let sandbox = python_sandbox(cx, "pipeline", rest_policy(server.port, "GET")?).await?; + let target = format!("{TEST_HOST}:{}", server.port); + let requests = format!( + "GET /allowed HTTP/1.1\r\nHost: {target}\r\nConnection: keep-alive\r\n\r\n\ + POST /blocked HTTP/1.1\r\nHost: {target}\r\nContent-Length: 0\r\n\r\n" + ); + run_pipeline(&sandbox, server.port, &requests).await?; + let received = String::from_utf8_lossy(&server.received()).into_owned(); + testkit::ensure!( + received.starts_with("GET /allowed HTTP/1.1\r\n") + && !received.to_ascii_lowercase().contains("\r\nconnection:") + && !received.contains("/blocked"), + "expected only the allowed request upstream, without Connection; got:\n{received}" + ); + Ok(()) +} + +/// A denied request pipelined behind an allowed chunked request is +/// authorized on its own: upstream sees the chunked request whole and +/// nothing after it. +#[testkit::test] +async fn chunked_pipelined_request_is_authorized_separately(cx: &mut Context) -> testkit::Result { + let server = TcpServer::reply_when_idle(PIPELINE_REPLY).await?; + let sandbox = python_sandbox(cx, "chunked", rest_policy(server.port, "POST")?).await?; + let target = format!("{TEST_HOST}:{}", server.port); + let requests = format!( + "POST /allowed HTTP/1.1\r\nHost: {target}\r\nTransfer-Encoding: chunked\r\n\ + Connection: keep-alive\r\n\r\n0\r\n\r\n\ + DELETE /blocked HTTP/1.1\r\nHost: {target}\r\nContent-Length: 0\r\n\r\n" + ); + run_pipeline(&sandbox, server.port, &requests).await?; + let received = String::from_utf8_lossy(&server.received()).into_owned(); + testkit::ensure!( + received.starts_with("POST /allowed HTTP/1.1\r\n") + && received.ends_with("0\r\n\r\n") + && !received.contains("/blocked"), + "expected only the allowed chunked request upstream; got:\n{received}" + ); + Ok(()) +} + +/// [`policy::base`] with the image's Python allowed to reach `TEST_HOST` on +/// `port`. +fn allow_server(port: u16) -> proto::SandboxPolicy { + let mut policy = policy::base(); + allow(&mut policy, port); + policy +} + +fn allow(policy: &mut proto::SandboxPolicy, port: u16) -> &mut proto::NetworkEndpoint { + policy::allow(policy, "egress", TEST_HOST, port, PYTHON_BINARIES) +} + +/// An endpoint on `port` with the `openshell/regex` redacting middleware, +/// `on_error`, and the extra endpoint YAML in `endpoint_options`. +fn middleware_policy( + port: u16, + on_error: &str, + endpoint_options: &str, +) -> Result { + policy::base_with(&format!( + r#"network_middlewares: + regex-redactor: + name: Redact API tokens + middleware: openshell/regex + order: 10 + config: + mode: redact + on_error: {on_error} + endpoints: + include: ["{TEST_HOST}"] + exclude: [] +network_policies: + egress: + name: egress + endpoints: + - host: {TEST_HOST} + port: {port} +{endpoint_options} binaries: {binaries} +"#, + binaries = python_binaries_yaml(), + )) +} + +/// A REST endpoint on `port` that allows only `method` on `/allowed`. +fn rest_policy(port: u16, method: &str) -> Result { + policy::base_with(&format!( + r"network_policies: + egress: + name: egress + endpoints: + - host: {TEST_HOST} + port: {port} + protocol: rest + enforcement: enforce + rules: + - allow: + method: {method} + path: /allowed + binaries: {binaries} +", + binaries = python_binaries_yaml(), + )) +} + +/// Send `requests` pipelined in one write, and fail unless the first gets +/// `200` and the second `403`. +async fn run_pipeline(sandbox: &Sandbox, port: u16, requests: &str) -> Result { + let output = run_python( + sandbox, + PIPELINE_CLIENT, + &[TEST_HOST, &port.to_string(), requests], + ) + .await?; + let result = json_result(&output)?; + let statuses: Vec<&str> = result["statuses"] + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .collect(); + testkit::ensure!( + statuses.len() == 2 + && statuses[0].starts_with("HTTP/1.1 200") + && statuses.contains(&"HTTP/1.1 403 Forbidden"), + "expected 200 then 403; got {statuses:?}" + ); + Ok(()) +} + +/// Run [`STATUS_CLIENT`] and fail unless both its requests got a status +/// `expected` accepts. A failed connection counts as status 0. +async fn expect_statuses(sandbox: &Sandbox, port: &str, expected: impl Fn(i64) -> bool) -> Result { + let output = run_python(sandbox, STATUS_CLIENT, &[TEST_HOST, port]).await?; + let result = json_result(&output)?; + for request in ["first", "second"] { + let status = result[request].as_i64().unwrap_or(0); + testkit::ensure!( + expected(status), + "unexpected {request} request outcome: {result}" + ); + } + Ok(()) +} + +/// The sandbox's policy revisions, from `policy list --output json`. +async fn policy_revisions(cx: &Context, sandbox: &Sandbox) -> Result { + cx.cli(["policy", "list", sandbox.name(), "--output", "json"]) + .await? + .success()? + .json() +} + +/// The contents of `path` in the sandbox, once it exists. A timeout reports +/// the keep-alive client's log. +async fn wait_for_file(sandbox: &Sandbox, path: &str) -> Result { + Ok(sandbox + .poll_exec( + &format!("{path} to appear"), + Duration::from_secs(60), + [ + "sh", + "-c", + &format!("cat {path} || {{ cat {RELOAD_LOG}; exit 1; }}"), + ], + |output| output.exit_code == 0, + ) + .await? + .stdout) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} diff --git a/tests/suites/conformance/src/scripts/echo_client.py b/tests/suites/conformance/src/scripts/echo_client.py new file mode 100644 index 0000000000..aea3b9da97 --- /dev/null +++ b/tests/suites/conformance/src/scripts/echo_client.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends the bytes hex-encoded in argv[3] to the echo server at +# argv[1]:argv[2] and fails unless it reads the same bytes back. +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +PAYLOAD = bytes.fromhex(sys.argv[3]) + +with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(PAYLOAD) + echoed = b"" + while len(echoed) < len(PAYLOAD): + chunk = sock.recv(len(PAYLOAD) - len(echoed)) + if not chunk: + break + echoed += chunk +if echoed != PAYLOAD: + raise SystemExit(f"the payload changed in transit: {echoed!r}") +print("ECHO_OK") diff --git a/tests/suites/conformance/src/scripts/pipeline_client.py b/tests/suites/conformance/src/scripts/pipeline_client.py new file mode 100644 index 0000000000..2ed5521bd9 --- /dev/null +++ b/tests/suites/conformance/src/scripts/pipeline_client.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends the pipelined HTTP requests in argv[3] to argv[1]:argv[2] in one +# write, reads until the connection closes, and prints the status lines of +# the responses. +import json +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +REQUESTS = sys.argv[3].encode() + +with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(REQUESTS) + response = b"" + while True: + chunk = sock.recv(4096) + if not chunk: + break + response += chunk +statuses = [ + line.decode(errors="replace") + for line in response.split(b"\r\n") + if line.startswith(b"HTTP/1.1 ") +] +print(json.dumps({"statuses": statuses})) diff --git a/tests/suites/conformance/src/scripts/redaction_client.py b/tests/suites/conformance/src/scripts/redaction_client.py new file mode 100644 index 0000000000..6bb7bc9b99 --- /dev/null +++ b/tests/suites/conformance/src/scripts/redaction_client.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# POSTs a JSON body holding an API key to the body-echo server at +# argv[1]:argv[2] twice, each on its own connection, and prints the bodies +# the server echoed. +import json +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +SECRET = "sk-1234567890abcdef" + + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + raise RuntimeError("incomplete response headers") + data += chunk + headers, body = data.split(b"\r\n\r\n", 1) + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + break + body += chunk + status = int(headers.split(None, 2)[1]) + if status != 200: + raise RuntimeError(f"request failed with HTTP {status}: {body!r}") + return json.loads(body[:length]) + + +def request_once(): + body = json.dumps({"api_key": SECRET}, separators=(",", ":")).encode() + request = ( + "POST /middleware HTTP/1.1\r\n" + f"Host: {HOST}:{PORT}\r\n" + "Content-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\n" + "Connection: close\r\n\r\n" + ).encode() + body + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(request) + return read_response(sock) + + +print(json.dumps({"first": request_once(), "second": request_once()}, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/reload_keep_alive_client.py b/tests/suites/conformance/src/scripts/reload_keep_alive_client.py new file mode 100644 index 0000000000..906cb95019 --- /dev/null +++ b/tests/suites/conformance/src/scripts/reload_keep_alive_client.py @@ -0,0 +1,66 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends a keep-alive GET to argv[1]:argv[2] and touches READY once it +# succeeds. When GO appears, sends a second request on the same connection +# and writes to RESULT whether it was refused. +import json +import os +import socket +import sys +import time + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +READY = "/tmp/proxy-reload-ready" +GO = "/tmp/proxy-reload-go" +RESULT = "/tmp/proxy-reload-result" + + +def read_response(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + return 0 + data += chunk + headers, body = data.split(b"\r\n\r\n", 1) + length = 0 + for line in headers.split(b"\r\n")[1:]: + if line.lower().startswith(b"content-length:"): + length = int(line.split(b":", 1)[1].strip()) + while len(body) < length: + chunk = sock.recv(4096) + if not chunk: + return 0 + body += chunk + return int(headers.split(None, 2)[1]) + + +target = f"{HOST}:{PORT}" +failed_closed = False +second_status = 0 +try: + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall( + f"GET /before-reload HTTP/1.1\r\nHost: {target}\r\nConnection: keep-alive\r\n\r\n".encode() + ) + if read_response(sock) != 200: + raise RuntimeError("the first request was denied") + open(READY, "w").close() + deadline = time.monotonic() + 120 + while not os.path.exists(GO) and time.monotonic() < deadline: + time.sleep(0.1) + if not os.path.exists(GO): + raise RuntimeError("timed out waiting for the policy reload") + try: + sock.sendall( + f"GET /after-reload HTTP/1.1\r\nHost: {target}\r\nConnection: close\r\n\r\n".encode() + ) + second_status = read_response(sock) + except OSError: + second_status = 0 + failed_closed = second_status != 200 +finally: + with open(RESULT, "w") as result: + json.dump({"failed_closed": failed_closed, "second_status": second_status}, result, sort_keys=True) diff --git a/tests/suites/conformance/src/scripts/status_client.py b/tests/suites/conformance/src/scripts/status_client.py new file mode 100644 index 0000000000..1d059a9c37 --- /dev/null +++ b/tests/suites/conformance/src/scripts/status_client.py @@ -0,0 +1,44 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends two GET requests to argv[1]:argv[2], each on its own connection, and +# prints their statuses, or the error a connection failed with. +import json +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) + + +def read_headers(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + + +def status(response): + parts = response.split(None, 2) + return int(parts[1]) if len(parts) > 1 else 0 + + +def request_status(path): + try: + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall( + f"GET {path} HTTP/1.1\r\n" + f"Host: {HOST}:{PORT}\r\nConnection: close\r\n\r\n".encode() + ) + return status(read_headers(sock)) + except OSError as error: + return {"errno": error.errno, "error": repr(error)} + + +print(json.dumps({ + "first": request_status("/first"), + "second": request_status("/second"), +}, sort_keys=True)) diff --git a/tests/suites/conformance/src/scripts/uninspectable_client.py b/tests/suites/conformance/src/scripts/uninspectable_client.py new file mode 100644 index 0000000000..75fa7d65a7 --- /dev/null +++ b/tests/suites/conformance/src/scripts/uninspectable_client.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends bytes that are neither HTTP nor TLS to argv[1]:argv[2] and fails if +# the connection answers with anything but a 403 unsupported_l7_protocol +# denial. A reset or an empty close also counts as blocked. +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +PAYLOAD = bytes([0x00, 0xFF, 0x13, 0x37]) + b"not-http-or-tls" + +with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(PAYLOAD) + denial = b"" + while True: + try: + chunk = sock.recv(4096) + except ConnectionResetError: + break + if not chunk: + break + denial += chunk +if denial and ( + b"HTTP/1.1 403 Forbidden" not in denial + or b"unsupported_l7_protocol" not in denial +): + raise SystemExit(f"missing the middleware denial: {denial!r}") +print("BLOCKED") From 49891dc20acb072f2486dd37c715cb323135d8e8 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:57:37 -0800 Subject: [PATCH 07/15] test(conformance): port credential header, body, and WebSocket text rewrite tests Signed-off-by: Matthew Grossman --- .../conformance/src/credential_gating.rs | 140 +++++++++++++++--- .../src/scripts/credential_rewrite_client.py | 46 ++++++ .../src/scripts/websocket_text_client.py | 74 +++++++++ 3 files changed, 242 insertions(+), 18 deletions(-) create mode 100644 tests/suites/conformance/src/scripts/credential_rewrite_client.py create mode 100644 tests/suites/conformance/src/scripts/websocket_text_client.py diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index 48bd9eb531..f73bc46762 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -4,18 +4,23 @@ //! Credential gating: which endpoints a provider credential may reach, and //! what the supervisor forwards upstream in REST bodies and WebSocket frames. //! -//! Ported from the e2e tests added in NVIDIA/OpenShell#5264. The upstream +//! Ported from the e2e tests added in NVIDIA/OpenShell#5264, the credential +//! test in `proxy_egress_pipeline`, and `websocket_conformance`. The upstream //! servers run on the test host ([`host_server_support`]), so the suite //! requires sandboxes to reach the test host as `host.openshell.internal`. use std::time::Duration; -use host_server_support::{HttpRecorder, Request, WebSocketRecorder}; +use host_server_support::{HttpRecorder, Request, TEXT_REPLY, WebSocketRecorder}; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; use testkit::{Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; -const TEST_HOST: &str = "host.openshell.internal"; +use crate::support::{self, TEST_HOST, python_binaries_yaml}; + +const CREDENTIAL_REWRITE_CLIENT: &str = include_str!("scripts/credential_rewrite_client.py"); +const WEBSOCKET_TEXT_CLIENT: &str = include_str!("scripts/websocket_text_client.py"); + const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; const TEST_SECRET: &str = "e2e-gating-secret-value"; const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; @@ -217,6 +222,84 @@ async fn policy_binding_updates_provider_environment_live(cx: &mut Context) -> t wait_for_token(&sandbox, "TOKEN_ABSENT").await } +/// A provider whose profile declares a REST endpoint with body rewriting +/// gets its placeholder replaced with the secret in both the +/// `Authorization` header and the body of every request, with no +/// placeholder left upstream. +#[testkit::test] +async fn profile_rest_endpoint_rewrites_header_and_body(cx: &mut Context) -> testkit::Result { + let server = HttpRecorder::start("ok").await?; + let sandbox = profile_endpoint_sandbox( + cx, + "rewrite", + &format!( + "{{host: {TEST_HOST}, port: {}, path: /probe, protocol: rest, access: full, \ + enforcement: enforce, request_body_credential_rewrite: true}}", + server.port + ), + ) + .await?; + support::run_python( + &sandbox, + CREDENTIAL_REWRITE_CLIENT, + &[TEST_HOST, &server.port.to_string()], + ) + .await? + .expect_stdout("REWRITE_OK\n")?; + + let requests = server.wait_for_requests(2).await?; + let authorization = format!("Bearer {TEST_SECRET}"); + for request in &requests { + testkit::ensure!( + request.header("authorization").as_deref() == Some(authorization.as_str()) + && request.body.contains(TEST_SECRET) + && !request.head.contains(PLACEHOLDER_PREFIX) + && !request.body.contains(PLACEHOLDER_PREFIX), + "expected the secret in the header and body upstream; got {}", + redact(request) + ); + } + Ok(()) +} + +/// A provider whose profile declares a WebSocket endpoint with +/// `websocket_credential_rewrite` gets its placeholder replaced with the +/// secret in a text frame. +#[testkit::test] +async fn profile_websocket_endpoint_rewrites_text_frames(cx: &mut Context) -> testkit::Result { + let server = WebSocketRecorder::start().await?; + let sandbox = profile_endpoint_sandbox( + cx, + "wstext", + &format!( + "{{host: {TEST_HOST}, port: {}, path: /ws, protocol: websocket, \ + enforcement: enforce, access: read-write, websocket_credential_rewrite: true}}", + server.port + ), + ) + .await?; + support::run_python( + &sandbox, + WEBSOCKET_TEXT_CLIENT, + &[TEST_HOST, &server.port.to_string()], + ) + .await? + .expect_stdout(&format!("{TEXT_REPLY}\n"))?; + + let texts = server.texts(); + testkit::ensure!( + texts.len() == 1 + && texts[0].contains(TEST_SECRET) + && !texts[0].contains(PLACEHOLDER_PREFIX), + "expected one text frame with the secret upstream; got {:?}", + texts + .iter() + .map(|text| text.replace(TEST_SECRET, "")) + .collect::>() + ); + Ok(()) +} + /// Run [`WEBSOCKET_CLIENT`] against `server` in a sandbox with `providers`, /// and return whether its binary frame reached `server`. async fn send_binary_frame( @@ -251,21 +334,43 @@ fn allow(policy: &mut proto::SandboxPolicy, port: u16) -> &mut proto::NetworkEnd /// `endpoints`, its profile declares the REST and WebSocket ports as its /// endpoints; without, a policy must bind the provider explicitly. async fn install_gating_provider(cx: &Context, endpoints: Option<(u16, u16)>) -> Result { - let name = cx.name("cg"); - let endpoints = endpoints.map_or_else(String::new, |(rest, websocket)| { - format!( - "endpoints: - - host: {TEST_HOST} - port: {rest} - protocol: rest - access: full - - host: {TEST_HOST} - port: {websocket} - protocol: websocket - access: read-write -" - ) + let endpoints = endpoints.map_or_else(Vec::new, |(rest, websocket)| { + vec![ + format!("{{host: {TEST_HOST}, port: {rest}, protocol: rest, access: full}}"), + format!( + "{{host: {TEST_HOST}, port: {websocket}, protocol: websocket, access: read-write}}" + ), + ] }); + install_provider_with_endpoints(cx, &endpoints).await +} + +/// A long-running sandbox with a provider from +/// [`install_provider_with_endpoints`] whose profile declares `endpoint`, a +/// YAML flow mapping, and a policy that allows the same endpoint. +async fn profile_endpoint_sandbox(cx: &Context, suffix: &str, endpoint: &str) -> Result { + let provider = install_provider_with_endpoints(cx, &[endpoint.to_string()]).await?; + let policy = policy::base_with(&format!( + "network_policies: + credential_gating: + name: credential_gating + endpoints: [{endpoint}] + binaries: {binaries} +", + binaries = python_binaries_yaml(), + ))?; + create_sandbox(cx, suffix, policy, &[&provider]).await +} + +/// Install a run-unique provider holding `TEST_SECRET` in `TOKEN_ENV`, whose +/// profile declares `endpoints`, each a YAML flow mapping. +async fn install_provider_with_endpoints(cx: &Context, endpoints: &[String]) -> Result { + let name = cx.name("cg"); + let endpoints = if endpoints.is_empty() { + String::new() + } else { + format!("endpoints: [{}]\n", endpoints.join(", ")) + }; let binaries: String = PYTHON_BINARIES .iter() .map(|path| format!(" - path: {path}\n")) @@ -390,4 +495,3 @@ async fn wait_for_token(sandbox: &Sandbox, expected: &str) -> Result { .await?; Ok(()) } - diff --git a/tests/suites/conformance/src/scripts/credential_rewrite_client.py b/tests/suites/conformance/src/scripts/credential_rewrite_client.py new file mode 100644 index 0000000000..b3d77a3ac0 --- /dev/null +++ b/tests/suites/conformance/src/scripts/credential_rewrite_client.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# POSTs the provider's token from E2E_GATING_TOKEN in an Authorization header +# and a JSON body to argv[1]:argv[2]/probe twice, each on its own +# connection, and fails unless both get 200. Never prints the token. +import json +import os +import socket +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +TOKEN = os.environ["E2E_GATING_TOKEN"] + + +def status(sock): + data = b"" + while b"\r\n\r\n" not in data: + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + parts = data.split(None, 2) + return int(parts[1]) if len(parts) > 1 else 0 + + +def request_once(): + body = json.dumps({"credential": TOKEN}, separators=(",", ":")).encode() + request = ( + "POST /probe HTTP/1.1\r\n" + f"Host: {HOST}:{PORT}\r\n" + f"Authorization: Bearer {TOKEN}\r\n" + "Content-Type: application/json\r\n" + f"Content-Length: {len(body)}\r\n" + "Connection: close\r\n\r\n" + ).encode() + body + with socket.create_connection((HOST, PORT), timeout=10) as sock: + sock.sendall(request) + return status(sock) + + +statuses = [request_once(), request_once()] +if statuses != [200, 200]: + raise SystemExit(f"expected two 200 responses; got {statuses}") +print("REWRITE_OK") diff --git a/tests/suites/conformance/src/scripts/websocket_text_client.py b/tests/suites/conformance/src/scripts/websocket_text_client.py new file mode 100644 index 0000000000..a211a83f36 --- /dev/null +++ b/tests/suites/conformance/src/scripts/websocket_text_client.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Upgrades to WebSocket at argv[1]:argv[2], sends the provider's token from +# E2E_GATING_TOKEN in one text frame, and prints the server's text reply. +# Never prints the token. +import base64 +import json +import os +import socket +import struct +import sys + +HOST = sys.argv[1] +PORT = int(sys.argv[2]) +TOKEN = os.environ["E2E_GATING_TOKEN"] + + +def read_exact(sock, size): + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise EOFError("the connection closed mid-frame") + data += chunk + return data + + +def masked_text_frame(text): + data = text.encode() + mask = os.urandom(4) + if len(data) < 126: + header = bytes([0x81, 0x80 | len(data)]) + else: + header = bytes([0x81, 0x80 | 126]) + struct.pack("!H", len(data)) + return header + mask + bytes(byte ^ mask[i % 4] for i, byte in enumerate(data)) + + +def read_text_frame(sock): + _, second = read_exact(sock, 2) + length = second & 0x7F + if length == 126: + length = struct.unpack("!H", read_exact(sock, 2))[0] + elif length == 127: + length = struct.unpack("!Q", read_exact(sock, 8))[0] + mask = read_exact(sock, 4) if second & 0x80 else b"" + payload = read_exact(sock, length) + if mask: + payload = bytes(byte ^ mask[i % 4] for i, byte in enumerate(payload)) + return payload.decode() + + +with socket.create_connection((HOST, PORT), timeout=10) as sock: + key = base64.b64encode(os.urandom(16)).decode() + sock.sendall( + ( + "GET /ws HTTP/1.1\r\n" + f"Host: {HOST}:{PORT}\r\n" + "Upgrade: websocket\r\n" + "Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\n" + "Sec-WebSocket-Version: 13\r\n\r\n" + ).encode() + ) + response = b"" + while b"\r\n\r\n" not in response: + chunk = sock.recv(4096) + if not chunk: + break + response += chunk + if not response.startswith(b"HTTP/1.1 101"): + raise SystemExit(f"upgrade failed: {response!r}") + sock.sendall(masked_text_frame(json.dumps({"authorization": "Bearer " + TOKEN}))) + print(read_text_frame(sock)) From b93916b41a466fb46e040227be667915b36699c6 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:59:09 -0800 Subject: [PATCH 08/15] test(conformance): port static provider credential endpoint binding test Signed-off-by: Matthew Grossman --- .../conformance/src/credential_gating.rs | 114 +++++++++++++++++- .../src/scripts/bound_credential_client.sh | 34 ++++++ 2 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 tests/suites/conformance/src/scripts/bound_credential_client.sh diff --git a/tests/suites/conformance/src/credential_gating.rs b/tests/suites/conformance/src/credential_gating.rs index f73bc46762..308fb49bd4 100644 --- a/tests/suites/conformance/src/credential_gating.rs +++ b/tests/suites/conformance/src/credential_gating.rs @@ -11,7 +11,8 @@ use std::time::Duration; -use host_server_support::{HttpRecorder, Request, TEXT_REPLY, WebSocketRecorder}; +use host_server_support::{HttpRecorder, Request, Response, TEXT_REPLY, WebSocketRecorder}; +use serde_json::json; use testkit::policy::{self, Access, Endpoint as _}; use testkit::sdk::raw::proto; use testkit::{Context, Error, GrpcCode, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; @@ -20,6 +21,8 @@ use crate::support::{self, TEST_HOST, python_binaries_yaml}; const CREDENTIAL_REWRITE_CLIENT: &str = include_str!("scripts/credential_rewrite_client.py"); const WEBSOCKET_TEXT_CLIENT: &str = include_str!("scripts/websocket_text_client.py"); +const BOUND_CREDENTIAL_CLIENT: &str = include_str!("scripts/bound_credential_client.sh"); +const BASH: &str = "/usr/bin/bash"; const TOKEN_ENV: &str = "E2E_GATING_TOKEN"; const TEST_SECRET: &str = "e2e-gating-secret-value"; @@ -300,6 +303,84 @@ async fn profile_websocket_endpoint_rewrites_text_frames(cx: &mut Context) -> te Ok(()) } +/// A provider's credential resolves only at the endpoints its profile +/// declares: another provider's endpoint and an undeclared path on its own +/// endpoint get `403`, and the sandbox logs the mismatch without the secret +/// or its variable name. +#[testkit::test] +async fn profile_endpoints_bind_static_credentials(cx: &mut Context) -> testkit::Result { + const SECRET_A: &str = "e2e-bound-secret"; + const SECRET_B: &str = "e2e-provider-b-secret"; + let authorized = |request: &Request| { + let authorized = request.header("authorization").as_deref() + == Some(format!("Bearer {SECRET_A}").as_str()); + Response::json(&json!({ "authorized": authorized })) + }; + let server_a = HttpRecorder::serve(authorized).await?; + let server_b = HttpRecorder::serve(authorized).await?; + let provider_a = + install_bound_provider(cx, "binda", "BOUND_TOKEN_A", SECRET_A, server_a.port).await?; + let provider_b = + install_bound_provider(cx, "bindb", "BOUND_TOKEN_B", SECRET_B, server_b.port).await?; + + let endpoint = |port| { + format!( + "{{host: {TEST_HOST}, port: {port}, path: /**, protocol: rest, access: full, \ + enforcement: enforce}}" + ) + }; + let policy = policy::base_with(&format!( + "network_policies: + binding: + name: binding + endpoints: [{}, {}] + binaries: [{{path: {BASH}}}] +", + endpoint(server_a.port), + endpoint(server_b.port), + ))?; + // The default image, like the e2e test this replaces, for its bash. + let sandbox = cx + .sandbox("bind") + .policy(policy) + .spec(|spec| spec.providers = vec![provider_a, provider_b]) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let output = sandbox + .exec([ + BASH, + "-c", + BOUND_CREDENTIAL_CLIENT, + "bash", + TEST_HOST, + &server_a.port.to_string(), + &server_b.port.to_string(), + ]) + .await? + .success()?; + testkit::ensure!( + output.stdout.trim() + == r#"ALLOWED={"authorized":true} ENDPOINT_DENIED=403 PATH_DENIED=403"#, + "expected the credential only at its bound endpoint and path; got {}", + output.stdout.replace(SECRET_A, "") + ); + + let logs = sandbox + .logs_until("the endpoint mismatch", |logs| { + logs.contains("openshell.provider_credential.endpoint_mismatch") + && logs.contains("credential_endpoint_mismatch") + }) + .await?; + for leaked in [SECRET_A, SECRET_B, "BOUND_TOKEN_A", "BOUND_TOKEN_B"] { + testkit::ensure!( + !logs.contains(leaked), + "the sandbox logs contain a credential value or variable name" + ); + } + Ok(()) +} + /// Run [`WEBSOCKET_CLIENT`] against `server` in a sandbox with `providers`, /// and return whether its binary frame reached `server`. async fn send_binary_frame( @@ -362,6 +443,37 @@ async fn profile_endpoint_sandbox(cx: &Context, suffix: &str, endpoint: &str) -> create_sandbox(cx, suffix, policy, &[&provider]).await } +/// Install a provider named `cx.name(suffix)` holding `secret` in `env_var`, +/// whose profile binds it to `/allowed/**` on `TEST_HOST` at `port` for +/// [`BASH`]. +async fn install_bound_provider( + cx: &Context, + suffix: &str, + env_var: &str, + secret: &str, + port: u16, +) -> Result { + let name = cx.name(suffix); + let profile = format!( + "id: {name} +display_name: E2E static endpoint binding +category: other +credentials: + - name: bound_token + env_vars: [{env_var}] + required: true + auth_style: bearer + header_name: authorization +endpoints: + - {{host: {TEST_HOST}, port: {port}, path: /allowed/**, protocol: rest, access: full, enforcement: enforce}} +binaries: [{BASH}] +" + ); + cx.install_provider(&name, &profile, &[(env_var, secret)]) + .await?; + Ok(name) +} + /// Install a run-unique provider holding `TEST_SECRET` in `TOKEN_ENV`, whose /// profile declares `endpoints`, each a YAML flow mapping. async fn install_provider_with_endpoints(cx: &Context, endpoints: &[String]) -> Result { diff --git a/tests/suites/conformance/src/scripts/bound_credential_client.sh b/tests/suites/conformance/src/scripts/bound_credential_client.sh new file mode 100644 index 0000000000..8ec43ede27 --- /dev/null +++ b/tests/suites/conformance/src/scripts/bound_credential_client.sh @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Sends provider A's token from BOUND_TOKEN_A to host $1: on port $2 at a +# path its profile binds, on port $3, which provider B's profile binds, and on +# port $2 at a path outside the binding. Prints the first response body and +# the other two statuses. Never prints the token. +host="$1" port_a="$2" port_b="$3" + +http_request() { + local port="$1" path="$2" status_line line + HTTP_STATUS= HTTP_BODY= + if ! exec 3<>"/dev/tcp/$host/$port"; then + HTTP_STATUS=connect-denied + return 1 + fi + printf 'GET %s HTTP/1.1\r\nHost: %s:%s\r\nAuthorization: Bearer %s\r\nConnection: close\r\n\r\n' \ + "$path" "$host" "$port" "$BOUND_TOKEN_A" >&3 + IFS= read -r status_line <&3 || return 1 + status_line="${status_line%$'\r'}" + HTTP_STATUS="${status_line#* }" + HTTP_STATUS="${HTTP_STATUS%% *}" + while IFS= read -r line <&3; do + line="${line%$'\r'}" + [[ -z "$line" ]] && break + done + while IFS= read -r line <&3 || [[ -n "$line" ]]; do HTTP_BODY+="$line"; done + exec 3>&- 3<&- +} + +http_request "$port_a" /allowed/check; allowed="$HTTP_BODY" +http_request "$port_b" /allowed/check || true; endpoint_denied="$HTTP_STATUS" +http_request "$port_a" /other/check; path_denied="$HTTP_STATUS" +printf 'ALLOWED=%s ENDPOINT_DENIED=%s PATH_DENIED=%s\n' "$allowed" "$endpoint_denied" "$path_denied" From a7b142c3f747ee33867777a91fee9a748afdc0ea Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:03:42 -0800 Subject: [PATCH 09/15] test(conformance): port live policy update tests Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/lib.rs | 1 + .../suites/conformance/src/network_egress.rs | 11 +- tests/suites/conformance/src/policy_update.rs | 364 ++++++++++++++++++ tests/suites/conformance/src/support.rs | 10 +- 4 files changed, 376 insertions(+), 10 deletions(-) create mode 100644 tests/suites/conformance/src/policy_update.rs diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index 4d837bd3f3..e1ce94e6ab 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -22,6 +22,7 @@ mod network_egress; mod network_l7; mod persistent_workspace; mod policy_advisor; +mod policy_update; mod process; mod settings; mod smoke; diff --git a/tests/suites/conformance/src/network_egress.rs b/tests/suites/conformance/src/network_egress.rs index 8a63120681..f29d8fa63a 100644 --- a/tests/suites/conformance/src/network_egress.rs +++ b/tests/suites/conformance/src/network_egress.rs @@ -19,7 +19,8 @@ use testkit::sdk::raw::proto; use testkit::{Context, PYTHON_BINARIES, Result, ResultExt as _, Sandbox}; use crate::support::{ - TEST_HOST, expect_values, json_result, python_binaries_yaml, python_sandbox, run_python, + TEST_HOST, expect_values, json_result, policy_revisions, python_binaries_yaml, python_sandbox, + run_python, }; const STATUS_CLIENT: &str = include_str!("scripts/status_client.py"); @@ -408,14 +409,6 @@ async fn expect_statuses(sandbox: &Sandbox, port: &str, expected: impl Fn(i64) - Ok(()) } -/// The sandbox's policy revisions, from `policy list --output json`. -async fn policy_revisions(cx: &Context, sandbox: &Sandbox) -> Result { - cx.cli(["policy", "list", sandbox.name(), "--output", "json"]) - .await? - .success()? - .json() -} - /// The contents of `path` in the sandbox, once it exists. A timeout reports /// the keep-alive client's log. async fn wait_for_file(sandbox: &Sandbox, path: &str) -> Result { diff --git a/tests/suites/conformance/src/policy_update.rs b/tests/suites/conformance/src/policy_update.rs new file mode 100644 index 0000000000..a55efa1d58 --- /dev/null +++ b/tests/suites/conformance/src/policy_update.rs @@ -0,0 +1,364 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Live policy updates through `policy set` and `policy update`: revisions +//! and versions, idempotent updates, scoped rule changes, and the revision a +//! supervisor writes when it fills in a sparse initial policy. +//! +//! Ported from the `live_policy_update` e2e tests. Revisions are read from +//! `policy get` and `policy list` with `--output json`. + +use std::time::Duration; + +use serde_json::Value; +use testkit::policy; +use testkit::sdk::raw::proto; +use testkit::{Context, Result, Sandbox}; + +use crate::support::policy_revisions; + +/// A network-only policy the supervisor fills in with the filesystem paths it +/// needs at startup, which makes a second revision. +const SPARSE_POLICY: &str = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../../examples/policy-advisor/sandbox-policy.yaml" +)); + +/// How long a revision may take to be loaded. +const LOAD_TIMEOUT: Duration = Duration::from_mins(2); + +/// Each new policy makes a newer version with a different hash, and setting +/// the same policy again makes no new version. Starts from a policy without +/// network rules, adds one rule, then a second. Every revision is listed, and +/// none is left pending. +#[testkit::test] +async fn policy_set_versions_and_ignores_repeats(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("polset") + .policy(policy::base()) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let initial = settled(cx, &sandbox).await?; + + let mut one_rule = policy::base(); + policy::allow(&mut one_rule, "rule_0", "api.anthropic.com", 443, &["/**"]); + let mut two_rules = one_rule.clone(); + policy::allow(&mut two_rules, "rule_1", "example.com", 443, &["/**"]); + + let mut previous = initial.clone(); + let mut versions = vec![version(&initial)?]; + for (step, next) in [("one rule", &one_rule), ("two rules", &two_rules)] { + policy::set(&sandbox, next).await?; + let updated = settled(cx, &sandbox).await?; + testkit::ensure!( + version(&updated)? > version(&previous)?, + "setting {step} didn't make a newer version: before {previous}, after {updated}" + ); + testkit::ensure!( + updated["hash"] != previous["hash"], + "setting {step} didn't change the hash: {}", + updated["hash"] + ); + + policy::set(&sandbox, next).await?; + let repeated = settled(cx, &sandbox).await?; + for field in ["version", "hash"] { + testkit::ensure!( + repeated[field] == updated[field], + "setting {step} again changed the {field}: {} became {}", + updated[field], + repeated[field] + ); + } + versions.push(version(&updated)?); + previous = updated; + } + + let revisions = policy_revisions(cx, &sandbox).await?; + let listed: Vec = revisions["revisions"] + .as_array() + .into_iter() + .flatten() + .filter_map(|revision| revision["version"].as_u64()) + .collect(); + let missing: Vec<_> = versions.iter().filter(|v| !listed.contains(v)).collect(); + testkit::ensure!( + missing.is_empty(), + "policy list is missing versions {missing:?}: {revisions}" + ); + Ok(()) +} + +/// `policy update` with an explicit rule and binaries changes only the +/// endpoint they select, even when another rule has the same host and ports. +/// An update whose ports or binaries match the rule only in part is rejected +/// without a new revision. +#[testkit::test] +async fn policy_update_changes_only_the_selected_endpoint(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .sandbox("polscope") + .policy(scoped_policy()?) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let before = settled(cx, &sandbox).await?; + let rules_before = &before["policy"]["network_policies"]; + testkit::ensure!( + rules_before.is_object(), + "policy get has no network policies: {before}" + ); + + // Each partial update matches one of ports and binaries fully, so a check + // that needs both to mismatch can't pass. + for (target, binaries) in [ + ( + "api.example.com:443:POST:/admin", + &["/usr/bin/curl", "/usr/bin/python3"][..], + ), + ( + "api.example.com:443,8443:POST:/admin", + &["/usr/bin/curl"][..], + ), + ] { + let mut args = vec![ + "policy", + "update", + sandbox.name(), + "--rule-name", + "internal_api", + "--add-allow", + target, + ]; + for binary in binaries { + args.extend(["--binary", binary]); + } + cx.cli(args).await?.failure()?; + let unchanged = policy_get(cx, &sandbox).await?; + for field in ["version", "hash", "policy"] { + testkit::ensure!( + unchanged[field] == before[field], + "rejected update {target} with {binaries:?} changed the {field}" + ); + } + } + + update( + cx, + &sandbox, + "--add-allow", + "api.example.com:443,8443:POST:/admin", + ) + .await?; + let allowed = policy_get(cx, &sandbox).await?; + testkit::ensure!( + version(&allowed)? > version(&before)? && allowed["hash"] != before["hash"], + "the allow didn't make a new revision: before {before}, after {allowed}" + ); + let rules = &allowed["policy"]["network_policies"]; + testkit::ensure!( + rules["sibling"] == rules_before["sibling"], + "the allow changed the sibling rule: {}", + rules["sibling"] + ); + testkit::ensure!( + rules["internal_api"]["endpoints"][1] == rules_before["internal_api"]["endpoints"][1], + "the allow changed the other endpoint: {}", + rules["internal_api"]["endpoints"][1] + ); + testkit::ensure!( + has_rule(&rules["internal_api"]["endpoints"][0]["rules"], |rule| { + rule["allow"]["method"] == "POST" && rule["allow"]["path"] == "/admin" + }), + "the endpoint has no POST /admin allow rule: {}", + rules["internal_api"]["endpoints"][0] + ); + + update( + cx, + &sandbox, + "--add-deny", + "api.example.com:443,8443:POST:/admin/private", + ) + .await?; + let denied = policy_get(cx, &sandbox).await?; + testkit::ensure!( + version(&denied)? > version(&allowed)?, + "the deny didn't make a new revision: before {allowed}, after {denied}" + ); + let rules = &denied["policy"]["network_policies"]; + testkit::ensure!( + rules["sibling"] == rules_before["sibling"], + "the deny changed the sibling rule: {}", + rules["sibling"] + ); + testkit::ensure!( + has_rule( + &rules["internal_api"]["endpoints"][0]["deny_rules"], + |rule| { rule["method"] == "POST" && rule["path"] == "/admin/private" } + ), + "the endpoint has no POST /admin/private deny rule: {}", + rules["internal_api"]["endpoints"][0] + ); + Ok(()) +} + +/// A sandbox created with a sparse policy gets a second revision with the +/// paths the supervisor added, and the supervisor reports it loaded: the +/// first is superseded and neither stays pending. Regression for #2159. +#[testkit::test] +async fn sparse_initial_policy_revision_is_loaded(cx: &mut Context) -> testkit::Result { + let sandbox = cx + .cli_sandbox("sparse") + .policy_yaml(SPARSE_POLICY) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + testkit::poll_until( + "the filled-in revision 2 to be loaded", + LOAD_TIMEOUT, + Duration::from_secs(2), + async || { + let active = policy_get(cx, &sandbox).await?; + let revisions = policy_revisions(cx, &sandbox).await?; + let statuses = statuses(&revisions); + let expected = [(1_u64, "superseded".to_string()), (2, "loaded".to_string())]; + Ok( + if active["version"] == 2 && statuses.as_slice() == expected { + testkit::Poll::Ready(()) + } else { + testkit::Poll::Pending(format!( + "active version {}, revisions {statuses:?}", + active["version"] + )) + }, + ) + }, + ) + .await +} + +/// [`policy::base`] with two rules that share a host and ports: `internal_api` +/// with a second endpoint, and `sibling` for other binaries. +fn scoped_policy() -> Result { + policy::base_with( + "network_policies: + internal_api: + name: internal_api + binaries: [{path: /usr/bin/curl}, {path: /usr/bin/python3}] + endpoints: + - {host: api.example.com, ports: [443, 8443], protocol: rest, access: read-only} + - {host: other.example.com, port: 443, protocol: rest, access: read-only} + sibling: + name: sibling + binaries: [{path: /usr/bin/wget}] + endpoints: + - {host: api.example.com, ports: [443, 8443], protocol: rest, access: read-only} +", + ) +} + +/// Add `rule` to the `internal_api` rule for curl and Python with +/// `policy update --wait`. +async fn update(cx: &Context, sandbox: &Sandbox, flag: &str, rule: &str) -> Result { + cx.cli([ + "policy", + "update", + sandbox.name(), + "--rule-name", + "internal_api", + "--binary", + "/usr/bin/curl", + "--binary", + "/usr/bin/python3", + flag, + rule, + "--wait", + ]) + .await? + .success()?; + Ok(()) +} + +/// The sandbox's active policy, from `policy get --full --output json`. +async fn policy_get(cx: &Context, sandbox: &Sandbox) -> Result { + let active: Value = cx + .cli([ + "policy", + "get", + sandbox.name(), + "--full", + "--output", + "json", + ]) + .await? + .success()? + .json()?; + version(&active)?; + testkit::ensure!( + active["hash"].as_str().is_some_and(|hash| !hash.is_empty()), + "policy get has no hash: {active}" + ); + Ok(active) +} + +/// The active policy once its revision is loaded and no revision is pending. +/// Fails if a revision failed to load. +async fn settled(cx: &Context, sandbox: &Sandbox) -> Result { + testkit::poll_until( + &format!("the policy of sandbox {} to be loaded", sandbox.name()), + LOAD_TIMEOUT, + Duration::from_secs(1), + async || { + let active = policy_get(cx, sandbox).await?; + let revisions = policy_revisions(cx, sandbox).await?; + let statuses = statuses(&revisions); + testkit::ensure!( + statuses.iter().all(|(_, status)| status != "failed"), + "a policy revision failed to load: {revisions}" + ); + let active_version = version(&active)?; + let loaded = statuses + .iter() + .any(|(version, status)| *version == active_version && status == "loaded"); + let pending = statuses.iter().any(|(_, status)| status == "pending"); + Ok(if loaded && !pending { + testkit::Poll::Ready(active) + } else { + testkit::Poll::Pending(format!( + "active version {active_version}, revisions {statuses:?}" + )) + }) + }, + ) + .await +} + +/// Each revision's version and status, oldest first. +fn statuses(revisions: &Value) -> Vec<(u64, String)> { + let mut statuses: Vec<_> = revisions["revisions"] + .as_array() + .into_iter() + .flatten() + .map(|revision| { + ( + revision["version"].as_u64().unwrap_or_default(), + revision["status"].as_str().unwrap_or_default().to_string(), + ) + }) + .collect(); + statuses.sort(); + statuses +} + +fn version(policy: &Value) -> Result { + policy["version"] + .as_u64() + .ok_or_else(|| testkit::Error::msg(format!("policy get has no version: {policy}"))) +} + +fn has_rule(rules: &Value, matches: impl Fn(&Value) -> bool) -> bool { + rules + .as_array() + .is_some_and(|rules| rules.iter().any(matches)) +} diff --git a/tests/suites/conformance/src/support.rs b/tests/suites/conformance/src/support.rs index dd512371df..5b80d1e5a6 100644 --- a/tests/suites/conformance/src/support.rs +++ b/tests/suites/conformance/src/support.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Helpers the network suites share. Not a suite. +//! Helpers the network and policy suites share. Not a suite. //! //! Their clients are Python scripts in `src/scripts/` that take the server's //! host and port as arguments, run in a long-running sandbox from @@ -69,6 +69,14 @@ pub fn expect_values>(result: &Value, expected: &[(K, Value)]) -> Ok(()) } +/// The sandbox's policy revisions, from `policy list --output json`. +pub async fn policy_revisions(cx: &Context, sandbox: &Sandbox) -> Result { + cx.cli(["policy", "list", sandbox.name(), "--output", "json"]) + .await? + .success()? + .json() +} + /// `binaries:` for policy YAML that lets the image's Python connect. pub fn python_binaries_yaml() -> String { let paths: Vec = PYTHON_BINARIES From 4d109ea4ea69ab3900b45c01aae6070ed422ae41 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:35:32 -0800 Subject: [PATCH 10/15] test(support): add a loopback TCP exchange client to host-server Signed-off-by: Matthew Grossman --- tests/support/host-server/src/lib.rs | 5 ++-- tests/support/host-server/src/tcp.rs | 38 +++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/tests/support/host-server/src/lib.rs b/tests/support/host-server/src/lib.rs index 45647ecf9d..611fa647f7 100644 --- a/tests/support/host-server/src/lib.rs +++ b/tests/support/host-server/src/lib.rs @@ -16,7 +16,8 @@ //! frames that arrive. //! - [`TcpServer`] records raw bytes, and echoes them or answers once the //! client goes idle. -//! - [`free_port`] finds a port nothing listens on. +//! - [`free_port`] finds a port nothing listens on, and [`exchange`] sends +//! bytes to a loopback port and reads the reply. mod http; mod tcp; @@ -28,7 +29,7 @@ use testkit::{Result, ResultExt as _}; use tokio::net::TcpListener; pub use http::{HttpRecorder, Request, Response}; -pub use tcp::TcpServer; +pub use tcp::{TcpServer, exchange}; pub use websocket::{TEXT_REPLY, WebSocketRecorder}; const WAIT_TIMEOUT: Duration = Duration::from_secs(10); diff --git a/tests/support/host-server/src/tcp.rs b/tests/support/host-server/src/tcp.rs index 42c4eaa9b0..1b6d566895 100644 --- a/tests/support/host-server/src/tcp.rs +++ b/tests/support/host-server/src/tcp.rs @@ -5,7 +5,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, PoisonError}; use std::time::Duration; -use testkit::{Poll, Result}; +use testkit::{Poll, Result, ResultExt as _}; use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use tokio::net::TcpStream; use tokio::task::JoinHandle; @@ -100,6 +100,35 @@ impl TcpServer { } } +/// Connect to `port` on the test host's loopback, send `request`, and return +/// what arrives until the other end closes. For a port the test reaches from +/// the host, such as one `openshell forward` listens on. +pub async fn exchange(port: u16, request: &[u8]) -> Result> { + #[allow( + clippy::disallowed_methods, + reason = "standalone test support has no openshell-core dependency and enables TCP_NODELAY below" + )] + let mut stream = TcpStream::connect(("127.0.0.1", port)) + .await + .context(format!("connect to 127.0.0.1:{port}"))?; + stream.set_nodelay(true).context("enable TCP_NODELAY")?; + stream + .write_all(request) + .await + .context(format!("send to 127.0.0.1:{port}"))?; + let mut reply = Vec::new(); + tokio::time::timeout(READ_TIMEOUT, stream.read_to_end(&mut reply)) + .await + .map_err(|_| { + testkit::Error::msg(format!( + "127.0.0.1:{port} didn't close within {READ_TIMEOUT:?}; read {} byte(s)", + reply.len() + )) + })? + .context(format!("read from 127.0.0.1:{port}"))?; + Ok(reply) +} + impl Drop for TcpServer { fn drop(&mut self) { self.task.abort(); @@ -171,4 +200,11 @@ mod tests { assert_eq!(reply, b"reply"); assert_eq!(server.received(), b"first second"); } + + #[tokio::test] + async fn exchange_returns_the_reply_until_close() { + let server = TcpServer::reply_when_idle(b"reply".to_vec()).await.unwrap(); + assert_eq!(exchange(server.port, b"hello").await.unwrap(), b"reply"); + assert_eq!(server.received(), b"hello"); + } } From f9f647918fe62d57139c65c2e45f2340ff2d8b22 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:37:06 -0800 Subject: [PATCH 11/15] test(conformance): port the port forward test Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/lib.rs | 1 + tests/suites/conformance/src/port_forward.rs | 90 +++++++++++++++++++ .../conformance/src/scripts/echo_server.py | 19 ++++ 3 files changed, 110 insertions(+) create mode 100644 tests/suites/conformance/src/port_forward.rs create mode 100644 tests/suites/conformance/src/scripts/echo_server.py diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index e1ce94e6ab..0a1ed08203 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -23,6 +23,7 @@ mod network_l7; mod persistent_workspace; mod policy_advisor; mod policy_update; +mod port_forward; mod process; mod settings; mod smoke; diff --git a/tests/suites/conformance/src/port_forward.rs b/tests/suites/conformance/src/port_forward.rs new file mode 100644 index 0000000000..999a7988a2 --- /dev/null +++ b/tests/suites/conformance/src/port_forward.rs @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Port forwarding with `openshell forward`. +//! +//! Ported from the `port_forward` e2e test. + +use std::time::Duration; + +use testkit::{Context, policy}; + +const ECHO_SERVER: &str = include_str!("scripts/echo_server.py"); + +/// How long `forward start` and the first echo through it may take. +const FORWARD_TIMEOUT: Duration = Duration::from_secs(60); + +/// `forward start -d` forwards a local port to a server on the sandbox's +/// loopback, and `forward stop` ends it. +#[testkit::test] +async fn forward_reaches_sandbox_loopback_server(cx: &mut Context) -> testkit::Result { + let local_port = host_server_support::free_port().await?; + let port = local_port.to_string(); + let sandbox = cx + .sandbox("fwd") + .policy(policy::base()) + .image(testkit::python_image()) + .command(["python3", "-c", ECHO_SERVER, &port]) + .create_ready() + .await?; + let name = sandbox.name().to_string(); + + // The forward's ssh keeps the CLI's stdout and stderr open, so wait for + // the CLI to exit rather than for its output to end. + let mut start = cx.cli_spawn(["forward", "start", "-d", &port, &name])?; + let code = start.wait(FORWARD_TIMEOUT).await?; + if code != 0 { + let output = start.output(FORWARD_TIMEOUT).await?; + return Err(testkit::Error::msg(format!( + "`{}` exited {code}:\n{}", + output.command, + output.report_text() + ))); + } + // Stops the forward if the test fails before it does. Its outcome is + // ignored, since the forward is already gone when the test passes. + let cli = cx.cli_handle(); + let (stop_port, stop_name) = (port.clone(), name.clone()); + cx.on_cleanup(format!("stop forward {port}"), async move || { + cli.run(["forward", "stop", &stop_port, &stop_name]).await?; + Ok(()) + }); + + testkit::poll_until( + &format!("an echo through forwarded port {port}"), + FORWARD_TIMEOUT, + Duration::from_secs(1), + async || { + Ok( + match host_server_support::exchange(local_port, b"hello-forward\n").await { + Ok(reply) if reply == b"echo:hello-forward\n" => testkit::Poll::Ready(()), + Ok(reply) => testkit::Poll::Pending(format!( + "reply {:?}", + String::from_utf8_lossy(&reply) + )), + Err(error) => testkit::Poll::Pending(error.to_string()), + }, + ) + }, + ) + .await?; + + cx.cli(["forward", "stop", &port, &name]).await?.success()?; + testkit::poll_until( + &format!("forwarded port {port} to close"), + FORWARD_TIMEOUT, + Duration::from_secs(1), + async || { + Ok( + match host_server_support::exchange(local_port, b"after-stop\n").await { + Err(_) => testkit::Poll::Ready(()), + Ok(reply) => testkit::Poll::Pending(format!( + "still forwarding; reply {:?}", + String::from_utf8_lossy(&reply) + )), + }, + ) + }, + ) + .await +} diff --git a/tests/suites/conformance/src/scripts/echo_server.py b/tests/suites/conformance/src/scripts/echo_server.py new file mode 100644 index 0000000000..9933a17333 --- /dev/null +++ b/tests/suites/conformance/src/scripts/echo_server.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# A TCP server on 127.0.0.1:argv[1] that answers each connection's first +# read with b"echo:" and the bytes read, then closes it. + +import socket +import sys + +server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +server.bind(("127.0.0.1", int(sys.argv[1]))) +server.listen(8) +while True: + conn, _ = server.accept() + with conn: + data = conn.recv(4096) + if data: + conn.sendall(b"echo:" + data) From 4d53f1e7ad3dd4b7ac7ee6d0d4c8076e7f2661f4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:39:04 -0800 Subject: [PATCH 12/15] test(features): port the IP literal egress test to routable-host-ip Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 4 + tests/suites/features/Cargo.lock | 9 ++ tests/suites/features/Cargo.toml | 2 +- .../features/routable-host-ip/Cargo.toml | 17 ++++ .../tests/routable_host_ip.rs | 96 +++++++++++++++++++ 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 tests/suites/features/routable-host-ip/Cargo.toml create mode 100644 tests/suites/features/routable-host-ip/tests/routable_host_ip.rs diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 361737cb7a..c51f6ae520 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -99,6 +99,10 @@ jobs: {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"routable-host-ip","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"routable-host-ip","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"routable-host-ip","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"routable-host-ip","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"connect-ssh-supervision","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index d207a62b05..dcec7c2a8e 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2928,6 +2928,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "routable-host-ip" +version = "0.0.0" +dependencies = [ + "host-server-support", + "openshell-testkit", + "serde_json", +] + [[package]] name = "rustc-demangle" version = "0.1.27" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index 78f4dd6277..d404806f50 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["cdi", "connect-ssh-supervision", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] +members = ["cdi", "connect-ssh-supervision", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image", "routable-host-ip"] diff --git a/tests/suites/features/routable-host-ip/Cargo.toml b/tests/suites/features/routable-host-ip/Cargo.toml new file mode 100644 index 0000000000..0645be32e8 --- /dev/null +++ b/tests/suites/features/routable-host-ip/Cargo.toml @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "routable-host-ip" +description = "Features suite: sandboxes reach the test host by an IP literal policy endpoint" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +host-server-support = { path = "../../../support/host-server" } +openshell-testkit = { path = "../../../testkit/testkit" } +serde_json = "1" diff --git a/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs new file mode 100644 index 0000000000..613fe4d0a2 --- /dev/null +++ b/tests/suites/features/routable-host-ip/tests/routable_host_ip.rs @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! A policy endpoint whose host is an IP literal reaches that address +//! transparently, whether `allowed_ips` names it or not. +//! +//! The servers run on the test host, and the sandbox reaches them by the +//! address `host.openshell.internal` resolves to. Not conformance, because +//! that address must be one the supervisor lets an IP literal reach: rootless +//! Podman's is link-local, which is always blocked. +//! +//! Ported from the `proxy_egress_pipeline` e2e test of IP literals, which +//! used two support containers on the e2e network instead. + +use host_server_support::HttpRecorder; +use serde_json::Value; +use testkit::policy; +use testkit::{Context, PYTHON_BINARIES, ResultExt as _}; + +/// Sends two GET requests to `argv[1]:argv[2]` and prints their statuses as +/// one JSON line. +const STATUS_CLIENT: &str = include_str!("../../../conformance/src/scripts/status_client.py"); + +const RESOLVE: &str = "import socket; print(socket.gethostbyname('host.openshell.internal'))"; + +/// Requests reach a server by an IP literal endpoint with `allowed_ips` for +/// that address, and another by an IP literal endpoint without it. +#[testkit::test] +async fn ip_literal_endpoints_reach_the_host(cx: &mut Context) -> testkit::Result { + let explicit = HttpRecorder::start("explicit").await?; + let implicit = HttpRecorder::start("implicit").await?; + let sandbox = cx + .sandbox("hostip") + .policy(policy::base()) + .image(testkit::python_image()) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let ip = sandbox + .exec(["python3", "-c", RESOLVE]) + .await? + .success() + .context("resolve host.openshell.internal in the sandbox")? + .stdout + .trim() + .to_string(); + + let mut ip_policy = policy::base(); + policy::allow( + &mut ip_policy, + "explicit", + &ip, + explicit.port, + PYTHON_BINARIES, + ) + .allowed_ips = vec![format!("{ip}/32")]; + policy::allow( + &mut ip_policy, + "implicit", + &ip, + implicit.port, + PYTHON_BINARIES, + ); + policy::set(&sandbox, &ip_policy).await?; + + for (mode, server) in [("explicit allowed_ips", &explicit), ("implicit", &implicit)] { + let output = sandbox + .exec([ + "python3", + "-c", + STATUS_CLIENT, + &ip, + &server.port.to_string(), + ]) + .await? + .success() + .context(format!("run the {mode} client"))?; + let result: Value = output + .stdout + .lines() + .rev() + .find_map(|line| serde_json::from_str(line.trim()).ok()) + .ok_or_else(|| { + testkit::Error::msg(format!( + "no JSON result from the {mode} client:\n{}\n{}", + output.stdout, output.stderr + )) + })?; + testkit::ensure!( + result["first"] == 200 && result["second"] == 200, + "{mode} requests to {ip}:{} failed: {result}", + server.port + ); + } + Ok(()) +} From a625a6dd785c3449cbb9190f58039b380cbb9873 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:39:52 -0800 Subject: [PATCH 13/15] test(tmachine): run conformance against a gateway in config push mode Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 1 + .../ansible/playbooks/config-push.yaml | 34 +++++++++++++++++++ tests/tmachine/config.nix | 6 ++++ 3 files changed, 41 insertions(+) create mode 100644 tests/tmachine/ansible/playbooks/config-push.yaml diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index c51f6ae520..2192e9795c 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -82,6 +82,7 @@ jobs: # workspace of its own. rows='[ {"environment":"ubuntu-docker-rootful","installer":"deb","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful-config-push","installer":"deb","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-docker-rootful","installer":"deb-external-docker-driver","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04","external_driver":true}, {"environment":"fedora-podman-rootful","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/tests/tmachine/ansible/playbooks/config-push.yaml b/tests/tmachine/ansible/playbooks/config-push.yaml new file mode 100644 index 0000000000..68e2478283 --- /dev/null +++ b/tests/tmachine/ansible/playbooks/config-push.yaml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Make the gateway push configuration to supervisors over their sessions +# (`config_delivery_mode = "push"`) instead of having them poll for it. Every +# installer merges /etc/openshell-test/gateway-overrides.toml into the +# gateway's configuration (see openshell_gateway/tasks/merge-settings.yaml). +--- +- name: Gateway configuration push for tests + hosts: all + gather_facts: false + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Create the test settings directory + become: true + ansible.builtin.file: + path: /etc/openshell-test + state: directory + owner: root + group: root + mode: "0755" + + - name: Push configuration to supervisors + become: true + ansible.builtin.copy: + dest: /etc/openshell-test/gateway-overrides.toml + content: | + [openshell.gateway] + config_delivery_mode = "push" + owner: root + group: root + mode: "0644" diff --git a/tests/tmachine/config.nix b/tests/tmachine/config.nix index 5a41618bf4..9b11a8cce1 100644 --- a/tests/tmachine/config.nix +++ b/tests/tmachine/config.nix @@ -112,6 +112,12 @@ let bases = dockerBases ++ podmanBases; setup.playbooks = [ "ansible/playbooks/cdi.yaml" ]; }; + # A gateway that pushes configuration to supervisors, for conformance in + # push delivery mode. + config-push = { + bases = dockerBases; + setup.playbooks = [ "ansible/playbooks/config-push.yaml" ]; + }; }; featureEnvironments = pkgs.lib.concatMap ( feature: map (base: withSetup base feature features.${feature}.setup) features.${feature}.bases From 82be802ee86c151ee981c0deaf536ce3e621b784 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:44:46 -0800 Subject: [PATCH 14/15] test(e2e): remove legacy network, L7, policy update, and port forward tests These tests now run in the conformance suite and the routable-host-ip feature package. Signed-off-by: Matthew Grossman --- e2e/rust/Cargo.toml | 25 - e2e/rust/e2e-podman.sh | 6 - e2e/rust/src/harness/sandbox.rs | 20 - e2e/rust/tests/bypass_detection.rs | 79 - e2e/rust/tests/forward_proxy_graphql_l7.rs | 415 ----- e2e/rust/tests/forward_proxy_jsonrpc_l7.rs | 498 ------ e2e/rust/tests/forward_proxy_l7_bypass.rs | 239 --- e2e/rust/tests/host_gateway_alias.rs | 341 +--- e2e/rust/tests/listener_lifetime.rs | 154 -- e2e/rust/tests/live_policy_update.rs | 704 -------- e2e/rust/tests/mcp_sessionless.rs | 381 ----- e2e/rust/tests/no_proxy.rs | 59 - e2e/rust/tests/port_forward.rs | 140 -- e2e/rust/tests/proxy_egress_pipeline.rs | 1683 -------------------- e2e/rust/tests/websocket_conformance.rs | 517 ------ tasks/test.toml | 9 +- tests/tmachine/artifacts.nix | 4 - 17 files changed, 2 insertions(+), 5272 deletions(-) delete mode 100644 e2e/rust/tests/bypass_detection.rs delete mode 100644 e2e/rust/tests/forward_proxy_graphql_l7.rs delete mode 100644 e2e/rust/tests/forward_proxy_jsonrpc_l7.rs delete mode 100644 e2e/rust/tests/forward_proxy_l7_bypass.rs delete mode 100644 e2e/rust/tests/listener_lifetime.rs delete mode 100644 e2e/rust/tests/live_policy_update.rs delete mode 100644 e2e/rust/tests/mcp_sessionless.rs delete mode 100644 e2e/rust/tests/no_proxy.rs delete mode 100644 e2e/rust/tests/port_forward.rs delete mode 100644 e2e/rust/tests/proxy_egress_pipeline.rs delete mode 100644 e2e/rust/tests/websocket_conformance.rs diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 1e0e822236..2827199ea8 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -189,11 +189,6 @@ name = "credential_drivers" path = "tests/credential_drivers.rs" required-features = ["e2e-kubernetes-credential-drivers"] -[[test]] -name = "websocket_conformance" -path = "tests/websocket_conformance.rs" -required-features = ["e2e-host-gateway"] - [[test]] name = "user_namespaces" path = "tests/user_namespaces.rs" @@ -204,21 +199,6 @@ name = "host_gateway_alias" path = "tests/host_gateway_alias.rs" required-features = ["e2e-host-gateway"] -[[test]] -name = "forward_proxy_l7_bypass" -path = "tests/forward_proxy_l7_bypass.rs" -required-features = ["e2e-host-gateway"] - -[[test]] -name = "forward_proxy_graphql_l7" -path = "tests/forward_proxy_graphql_l7.rs" -required-features = ["e2e-host-gateway"] - -[[test]] -name = "forward_proxy_jsonrpc_l7" -path = "tests/forward_proxy_jsonrpc_l7.rs" -required-features = ["e2e-host-gateway"] - [[test]] name = "workspace_lifecycle" path = "tests/workspace_lifecycle.rs" @@ -229,11 +209,6 @@ name = "supervisor_middleware_http_hooks" path = "tests/supervisor_middleware_http_hooks.rs" required-features = ["e2e-docker"] -[[test]] -name = "proxy_egress_pipeline" -path = "tests/proxy_egress_pipeline.rs" -required-features = ["e2e-host-gateway"] - [[test]] name = "workspace_namespace_managed" path = "tests/workspace_namespace_managed.rs" diff --git a/e2e/rust/e2e-podman.sh b/e2e/rust/e2e-podman.sh index d5a54741b7..4ec68f9672 100755 --- a/e2e/rust/e2e-podman.sh +++ b/e2e/rust/e2e-podman.sh @@ -22,17 +22,11 @@ source "${ROOT}/e2e/support/conformance.sh" # unfiltered e2e:podman task remains available while the omitted targets are # stabilized and can be added here. PODMAN_CI_TESTS=( - bypass_detection ephemeral_cleanup driver_config_volume edge_tunnel_e2e - forward_proxy_graphql_l7 - forward_proxy_jsonrpc_l7 - forward_proxy_l7_bypass host_gateway_alias - live_policy_update local_driver_token_restart - no_proxy ocsf_jsonl_conformance otlp_relay podman_corporate_proxy diff --git a/e2e/rust/src/harness/sandbox.rs b/e2e/rust/src/harness/sandbox.rs index 9f57a29daa..bf71208f98 100644 --- a/e2e/rust/src/harness/sandbox.rs +++ b/e2e/rust/src/harness/sandbox.rs @@ -647,26 +647,6 @@ impl SandboxGuard { Ok(combined) } - /// Spawn `openshell forward start` as a background process. - /// - /// Returns the child process handle. The caller is responsible for killing - /// it (or it will be killed on drop since `kill_on_drop(true)` is set). - /// - /// # Errors - /// - /// Returns an error if the process cannot be spawned. - pub fn spawn_forward(&self, port: u16) -> Result { - let mut cmd = openshell_cmd(); - cmd.arg("forward") - .arg("start") - .arg(port.to_string()) - .arg(&self.name); - cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); - - cmd.spawn() - .map_err(|e| format!("failed to spawn port forward: {e}")) - } - /// Delete the sandbox explicitly. /// /// Also kills the background child process if one exists. This is called diff --git a/e2e/rust/tests/bypass_detection.rs b/e2e/rust/tests/bypass_detection.rs deleted file mode 100644 index 569e3a60d6..0000000000 --- a/e2e/rust/tests/bypass_detection.rs +++ /dev/null @@ -1,79 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Verify that direct TCP bypass attempts fail promptly at the sandbox -//! syscall boundary instead of reaching the runtime's external network. -//! -//! This test is implementation-agnostic — it validates the observable -//! behavior rather than a particular packet-filter implementation. - -#![cfg(feature = "e2e")] - -use openshell_e2e::harness::sandbox::SandboxGuard; - -/// Python script that attempts a raw TCP connect bypassing the proxy. -/// -/// `socket.connect()` does not honor proxy environment variables. The script -/// reports the outcome and wall-clock time so the test can assert that the -/// sandbox's seccomp mediation blocks it before the outer fence is needed. -/// -/// Target 198.51.100.1 is RFC 5737 TEST-NET-2 — documentation-only address -/// space that will never route. -fn bypass_attempt_script() -> &'static str { - r#" -import json, socket, time - -start = time.monotonic() -result = "unknown" -try: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.settimeout(10) - s.connect(("198.51.100.1", 80)) - result = "connected" - s.close() -except ConnectionRefusedError: - result = "refused" -except PermissionError: - result = "denied" -except socket.timeout: - result = "timeout" -except OSError as e: - result = f"error:{e}" - -elapsed_ms = int((time.monotonic() - start) * 1000) -print(json.dumps({"bypass_result": result, "elapsed_ms": elapsed_ms}), flush=True) -"# -} - -/// A direct TCP connection bypassing supervision should be denied without -/// waiting for the socket's network timeout. -#[tokio::test] -async fn bypass_attempt_is_rejected_fast() { - let guard = SandboxGuard::create(&["--", "python3", "-c", bypass_attempt_script()]) - .await - .expect("sandbox create"); - - let json_line = guard - .create_output - .lines() - .find(|l| l.contains("bypass_result")) - .unwrap_or_else(|| panic!("no bypass_result JSON in output:\n{}", guard.create_output)); - - let parsed: serde_json::Value = serde_json::from_str(json_line.trim()) - .unwrap_or_else(|e| panic!("failed to parse JSON '{json_line}': {e}")); - - let result = parsed["bypass_result"].as_str().unwrap(); - let elapsed_ms = parsed["elapsed_ms"].as_u64().unwrap(); - - assert_eq!( - result, "denied", - "expected seccomp mediation to deny the direct connect, got '{result}' after {elapsed_ms}ms.\n\ - Full output:\n{}", - guard.create_output - ); - - assert!( - elapsed_ms < 8000, - "bypass rejection took {elapsed_ms}ms — expected < 8000ms." - ); -} diff --git a/e2e/rust/tests/forward_proxy_graphql_l7.rs b/e2e/rust/tests/forward_proxy_graphql_l7.rs deleted file mode 100644 index 5c25556cfa..0000000000 --- a/e2e/rust/tests/forward_proxy_graphql_l7.rs +++ /dev/null @@ -1,415 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! E2E tests for GraphQL L7 inspection through transparent interception. -//! -//! The upstream server deliberately does not implement GraphQL. `OpenShell` -//! parses and enforces GraphQL before forwarding, so any HTTP server that -//! accepts POST /graphql is enough to prove allowed requests reach upstream -//! and denied requests are stopped by the sandbox proxy. - -#![cfg(feature = "e2e")] - -use std::io::Write; - -use openshell_e2e::harness::container::ContainerHttpServer; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const TEST_SERVER_ALIAS: &str = "graphql-l7.openshell.test"; - -async fn start_test_server() -> Result { - let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer - -class Handler(BaseHTTPRequestHandler): - def read_chunked(self): - while True: - size_line = self.rfile.readline() - if not size_line: - return - size = int(size_line.split(b";", 1)[0].strip(), 16) - if size == 0: - while self.rfile.readline().strip(): - pass - return - self.rfile.read(size) - self.rfile.read(2) - def do_GET(self): - if self.path == "/" or self.path.startswith("/graphql?"): - self.send_response(200) - else: - self.send_response(418) - self.end_headers() - self.wfile.write(b'{"ok":true}') - def do_POST(self): - if self.headers.get("Transfer-Encoding", "").lower() == "chunked": - self.read_chunked() - else: - _ = self.rfile.read(int(self.headers.get("Content-Length", "0"))) - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(b'{"ok":true}') - def log_message(self, format, *args): - pass - -HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() -"#; - - ContainerHttpServer::start_python(TEST_SERVER_ALIAS, script).await -} - -fn write_graphql_policy(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - test_graphql_l7: - name: test_graphql_l7 - endpoints: - - host: {host} - port: {port} - protocol: graphql - enforcement: enforce - persisted_queries: allow_registered - graphql_persisted_queries: - abc123: - operation_type: query - operation_name: Viewer - fields: [viewer] - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - rules: - - allow: - operation_type: query - fields: [viewer] - - allow: - operation_type: mutation - fields: [createIssue] - deny_rules: - - operation_type: mutation - fields: [deleteRepository] - binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn graphql_l7_enforces_high_level_and_raw_transparent_paths() { - let server = start_test_server().await.expect("start test server"); - let policy = write_graphql_policy(&server.host, server.port).expect("write custom policy"); - let policy_path = policy - .path() - .to_str() - .expect("temp policy path should be utf-8") - .to_string(); - - let script = format!( - r#" -import json -import socket -import time -import urllib.error -import urllib.parse -import urllib.request - -HOST = {host:?} -PORT = {port} -DETAILS = {{}} - -QUERY_VIEWER = "query Viewer {{ viewer {{ login }} }}" -QUERY_REPOSITORY = "query Repo {{ repository(owner:\"o\", name:\"r\") {{ id }} }}" -MUTATION_CREATE = "mutation Create {{ createIssue(input:{{repositoryId:\"r\", title:\"t\", body:\"b\"}}) {{ issue {{ id }} }} }}" -MUTATION_DELETE = "mutation Delete {{ deleteRepository(input:{{repositoryId:\"r\"}}) {{ clientMutationId }} }}" - -def forward_status(query): - body = json.dumps({{"query": query}}).encode() - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/graphql", - data=body, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -def forward_get_status(query): - encoded = urllib.parse.urlencode({{"query": query}}) - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/graphql?{{encoded}}", - method="GET", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -def forward_duplicate_get_status(): - safe = urllib.parse.quote_plus(QUERY_VIEWER) - unsafe = urllib.parse.quote_plus(MUTATION_DELETE) - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/graphql?query={{safe}}&query={{unsafe}}", - method="GET", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -def forward_persisted_get_status(hash_value): - extensions = json.dumps({{"persistedQuery": {{"version": 1, "sha256Hash": hash_value}}}}) - encoded = urllib.parse.urlencode({{"operationName": "Viewer", "extensions": extensions}}) - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/graphql?{{encoded}}", - method="GET", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -def forward_chunked_status(query): - target = f"{{HOST}}:{{PORT}}" - body = json.dumps({{"query": query}}).encode() - chunk = f"{{len(body):x}}\r\n".encode() + body + b"\r\n0\r\n\r\n" - - with socket.create_connection((HOST, PORT), timeout=15) as sock: - request = ( - f"POST /graphql HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Content-Type: application/json\r\n" - f"Transfer-Encoding: chunked\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + chunk - sock.sendall(request) - response, body = read_response(sock) - DETAILS["forward_chunked_query_allowed_detail"] = body.decode(errors="replace") - return status_code(response, "forward_chunked_response") - -def read_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def read_response(sock): - response = read_until(sock, b"\r\n\r\n") - headers, _, body = response.partition(b"\r\n\r\n") - content_length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - content_length = int(line.split(b":", 1)[1].strip()) - break - while len(body) < content_length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - return response, body - -def status_code(response, label): - parts = response.split() - if len(parts) < 2: - DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{{label}}: malformed HTTP response: {{response!r}}") - try: - return int(parts[1]) - except ValueError as error: - DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{{label}}: non-numeric HTTP status: {{response!r}}") from error - -def raw_http_status(label, request): - last_error = None - for attempt in range(5): - try: - with socket.create_connection((HOST, PORT), timeout=15) as sock: - sock.sendall(request) - sock.shutdown(socket.SHUT_WR) - response = read_until(sock, b"\r\n\r\n") - return status_code(response, f"{{label}}_response") - except (OSError, RuntimeError) as error: - last_error = error - DETAILS[f"{{label}}_attempt_{{attempt + 1}}_error"] = str(error) - time.sleep(0.2) - - raise RuntimeError(f"{{label}}: failed after 5 attempts: {{last_error}}") - -def raw_status(query, label): - target = f"{{HOST}}:{{PORT}}" - body = json.dumps({{"query": query}}).encode() - - request = ( - f"POST /graphql HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Content-Type: application/json\r\n" - f"Content-Length: {{len(body)}}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + body - return raw_http_status(label, request) - -def raw_get_status(query, label): - target = f"{{HOST}}:{{PORT}}" - encoded = urllib.parse.urlencode({{"query": query}}) - - request = ( - f"GET /graphql?{{encoded}} HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() - return raw_http_status(label, request) - -def raw_duplicate_get_status(): - target = f"{{HOST}}:{{PORT}}" - safe = urllib.parse.quote_plus(QUERY_VIEWER) - unsafe = urllib.parse.quote_plus(MUTATION_DELETE) - - request = ( - f"GET /graphql?query={{safe}}&query={{unsafe}} HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() - return raw_http_status("raw_duplicate_get_denied", request) - -def raw_persisted_get_status(hash_value, label): - target = f"{{HOST}}:{{PORT}}" - extensions = json.dumps({{"persistedQuery": {{"version": 1, "sha256Hash": hash_value}}}}) - encoded = urllib.parse.urlencode({{"operationName": "Viewer", "extensions": extensions}}) - - request = ( - f"GET /graphql?{{encoded}} HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() - return raw_http_status(label, request) - -def raw_chunked_status(query): - target = f"{{HOST}}:{{PORT}}" - body = json.dumps({{"query": query}}).encode() - chunk = f"{{len(body):x}}\r\n".encode() + body + b"\r\n0\r\n\r\n" - - request = ( - f"POST /graphql HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Content-Type: application/json\r\n" - f"Transfer-Encoding: chunked\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + chunk - return raw_http_status("raw_chunked_query_allowed", request) - -results = {{ - "forward_query_allowed": forward_status(QUERY_VIEWER), - "forward_get_query_allowed": forward_get_status(QUERY_VIEWER), - "forward_duplicate_get_denied": forward_duplicate_get_status(), - "forward_persisted_get_allowed": forward_persisted_get_status("abc123"), - "forward_unregistered_persisted_get_denied": forward_persisted_get_status("missing"), - "forward_chunked_query_allowed": forward_chunked_status(QUERY_VIEWER), - "forward_unlisted_field_denied": forward_status(QUERY_REPOSITORY), - "forward_mutation_allowed": forward_status(MUTATION_CREATE), - "forward_deny_rule_denied": forward_status(MUTATION_DELETE), - "raw_query_allowed": raw_status(QUERY_VIEWER, "raw_query_allowed"), - "raw_get_query_allowed": raw_get_status(QUERY_VIEWER, "raw_get_query_allowed"), - "raw_duplicate_get_denied": raw_duplicate_get_status(), - "raw_persisted_get_allowed": raw_persisted_get_status("abc123", "raw_persisted_get_allowed"), - "raw_unregistered_persisted_get_denied": raw_persisted_get_status("missing", "raw_unregistered_persisted_get_denied"), - "raw_chunked_query_allowed": raw_chunked_status(QUERY_VIEWER), - "raw_unlisted_field_denied": raw_status(QUERY_REPOSITORY, "raw_unlisted_field_denied"), - "raw_mutation_allowed": raw_status(MUTATION_CREATE, "raw_mutation_allowed"), - "raw_deny_rule_denied": raw_status(MUTATION_DELETE, "raw_deny_rule_denied"), -}} -results.update(DETAILS) -print(json.dumps(results, sort_keys=True)) -"#, - host = server.host, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - - for (key, expected) in [ - ("forward_query_allowed", 200), - ("forward_get_query_allowed", 200), - ("forward_duplicate_get_denied", 403), - ("forward_persisted_get_allowed", 200), - ("forward_unregistered_persisted_get_denied", 403), - ("forward_chunked_query_allowed", 200), - ("forward_unlisted_field_denied", 403), - ("forward_mutation_allowed", 200), - ("forward_deny_rule_denied", 403), - ("raw_query_allowed", 200), - ("raw_get_query_allowed", 200), - ("raw_duplicate_get_denied", 403), - ("raw_persisted_get_allowed", 200), - ("raw_unregistered_persisted_get_denied", 403), - ("raw_chunked_query_allowed", 200), - ("raw_unlisted_field_denied", 403), - ("raw_mutation_allowed", 200), - ("raw_deny_rule_denied", 403), - ] { - let expected_fragment = format!(r#""{key}": {expected}"#); - assert!( - guard.create_output.contains(&expected_fragment), - "expected {key}={expected}, got:\n{}", - guard.create_output - ); - } -} diff --git a/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs b/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs deleted file mode 100644 index b46dac1313..0000000000 --- a/e2e/rust/tests/forward_proxy_jsonrpc_l7.rs +++ /dev/null @@ -1,498 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! E2E tests for JSON-RPC L7 inspection through transparent interception. -//! -//! The upstream server deliberately does not implement JSON-RPC. `OpenShell` -//! parses and enforces JSON-RPC before forwarding, so any HTTP server that -//! accepts POST /rpc is enough to prove allowed requests reach upstream -//! and denied requests are stopped by the sandbox proxy. - -#![cfg(feature = "e2e")] - -use std::io::Write; - -use openshell_e2e::harness::container::ContainerHttpServer; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const RULES_TEST_SERVER_ALIAS: &str = "jsonrpc-l7-rules.openshell.test"; -const AUDIT_TEST_SERVER_ALIAS: &str = "jsonrpc-l7-audit.openshell.test"; - -async fn start_test_server(alias: &str) -> Result { - let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer - -class Handler(BaseHTTPRequestHandler): - def read_body(self): - if self.headers.get("Transfer-Encoding", "").lower() == "chunked": - data = b"" - while True: - size_line = self.rfile.readline() - if not size_line: - break - size = int(size_line.split(b";", 1)[0].strip(), 16) - if size == 0: - while self.rfile.readline().strip(): - pass - break - data += self.rfile.read(size) - self.rfile.read(2) - return data - return self.rfile.read(int(self.headers.get("Content-Length", "0"))) - - def do_GET(self): - self.send_response(200) - self.end_headers() - - def do_POST(self): - self.read_body() - self.send_response(200) - self.send_header("Content-Type", "application/json") - self.end_headers() - self.wfile.write(b'{"jsonrpc":"2.0","id":1,"result":{}}') - - def log_message(self, format, *args): - pass - -HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() -"#; - - ContainerHttpServer::start_python(alias, script).await -} - -fn write_jsonrpc_policy(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - test_jsonrpc_l7: - name: test_jsonrpc_l7 - endpoints: - - host: {host} - port: {port} - path: /rpc - protocol: json-rpc - enforcement: enforce - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - json_rpc: - max_body_bytes: 65536 - rules: - - allow: - method: initialize - - allow: - method: tools/list - - allow: - method: tools/call - deny_rules: - - method: tools/delete - binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -fn write_jsonrpc_default_audit_policy(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - test_jsonrpc_l7_audit: - name: test_jsonrpc_l7_audit - endpoints: - - host: {host} - port: {port} - path: /rpc - protocol: json-rpc - allowed_ips: - - "10.0.0.0/8" - - "100.64.0.0/10" - - "172.0.0.0/8" - - "198.18.0.0/15" - - "192.168.0.0/16" - - "fc00::/7" - json_rpc: - max_body_bytes: 65536 - rules: - - allow: - method: initialize - binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn jsonrpc_l7_enforces_high_level_and_raw_transparent_paths() { - let server = start_test_server(RULES_TEST_SERVER_ALIAS) - .await - .expect("start test server"); - let policy = write_jsonrpc_policy(&server.host, server.port).expect("write custom policy"); - let policy_path = policy - .path() - .to_str() - .expect("temp policy path should be utf-8") - .to_string(); - - let script = format!( - r#" -import json -import socket -import time -import urllib.error -import urllib.request - -HOST = {host:?} -PORT = {port} -DETAILS = {{ - "debug_target": {{"host": HOST, "port": PORT}}, -}} - -def text(data): - return data.decode(errors="replace") - -def selected_headers(headers): - return {{ - key.lower(): value - for key, value in headers.items() - if key.lower() in ("content-type", "content-length", "server") - }} - -def record_http_error(label, error, request_body): - response_body = error.read() - DETAILS[f"{{label}}_request"] = request_body - DETAILS[f"{{label}}_response"] = {{ - "status": error.code, - "reason": str(error.reason), - "headers": selected_headers(error.headers), - "body": text(response_body), - }} - return error.code - -def post_jsonrpc(label, method, params=None, req_id=1): - body = {{"jsonrpc": "2.0", "id": req_id, "method": method}} - if params is not None: - body["params"] = params - encoded = json.dumps(body).encode() - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/rpc", - data=encoded, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - return record_http_error(label, error, body) - -def post_jsonrpc_batch(label, requests): - encoded = json.dumps(requests).encode() - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/rpc", - data=encoded, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - return record_http_error(label, error, requests) - -def post_invalid_json(label): - encoded = b"not valid json {{" - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/rpc", - data=encoded, - headers={{"Content-Type": "application/json", "Content-Length": str(len(encoded))}}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - return record_http_error(label, error, text(encoded)) - -def read_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def read_response(sock): - response = read_until(sock, b"\r\n\r\n") - headers, _, body = response.partition(b"\r\n\r\n") - content_length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - content_length = int(line.split(b":", 1)[1].strip()) - break - while len(body) < content_length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - return response, body - -def status_code(response, label): - parts = response.split() - if len(parts) < 2: - DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{{label}}: malformed HTTP response: {{response!r}}") - try: - return int(parts[1]) - except ValueError as error: - DETAILS[f"{{label}}_raw"] = response.decode(errors="replace") - raise RuntimeError(f"{{label}}: non-numeric HTTP status: {{response!r}}") from error - -def record_raw_response(label, response, body=b""): - code = status_code(response, label) - if code != 200: - DETAILS[f"{{label}}_raw"] = text(response) - if body: - DETAILS[f"{{label}}_body"] = text(body) - return code - -def raw_http_status(label, request): - last_error = None - for attempt in range(5): - try: - with socket.create_connection((HOST, PORT), timeout=15) as sock: - sock.sendall(request) - sock.shutdown(socket.SHUT_WR) - response, body = read_response(sock) - return record_raw_response(f"{{label}}_response", response, body) - except (OSError, RuntimeError) as error: - last_error = error - DETAILS[f"{{label}}_attempt_{{attempt + 1}}_error"] = str(error) - time.sleep(0.2) - - raise RuntimeError(f"{{label}}: failed after 5 attempts: {{last_error}}") - -def raw_jsonrpc_status(method, params, label): - target = f"{{HOST}}:{{PORT}}" - body = {{"jsonrpc": "2.0", "id": 1, "method": method}} - if params is not None: - body["params"] = params - encoded = json.dumps(body).encode() - request = ( - f"POST /rpc HTTP/1.1\r\n" - f"Host: {{target}}\r\n" - f"Content-Type: application/json\r\n" - f"Content-Length: {{len(encoded)}}\r\n" - f"Connection: close\r\n" - f"\r\n" - ).encode() + encoded - return raw_http_status(label, request) - -results = {{ - # forward proxy — method-only allow rules - "forward_method_initialize_allowed": post_jsonrpc("forward_method_initialize_allowed", "initialize", {{"protocolVersion": "2025-11-25", "capabilities": {{}}}}), - "forward_method_tools_list_allowed": post_jsonrpc("forward_method_tools_list_allowed", "tools/list"), - - # forward proxy — method allow/deny rules - "forward_method_tools_call_allowed": post_jsonrpc("forward_method_tools_call_allowed", "tools/call", {{"name": "read_status"}}), - "forward_method_tools_call_with_unmatched_params_allowed": post_jsonrpc("forward_method_tools_call_with_unmatched_params_allowed", "tools/call", {{"name": "blocked_action", "arguments": {{"scope": "ignored"}}}}), - "forward_method_tools_delete_denied": post_jsonrpc("forward_method_tools_delete_denied", "tools/delete", {{"name": "purge_cache"}}), - - # forward proxy — batch: all requests allowed - "forward_batch_all_allowed": post_jsonrpc_batch("forward_batch_all_allowed", [ - {{"jsonrpc": "2.0", "id": 1, "method": "tools/list"}}, - {{"jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {{"name": "read_status"}}}}, - ]), - - # forward proxy — batch: one denied request causes full batch denial - "forward_batch_one_denied": post_jsonrpc_batch("forward_batch_one_denied", [ - {{"jsonrpc": "2.0", "id": 1, "method": "tools/list"}}, - {{"jsonrpc": "2.0", "id": 2, "method": "tools/delete", "params": {{"name": "purge_cache"}}}}, - ]), - - # forward proxy — invalid JSON body fails closed before generic rules apply - "forward_invalid_json_denied": post_invalid_json("forward_invalid_json_denied"), - - # raw socket path — representative allowed and denied cases - "raw_method_initialize_allowed": raw_jsonrpc_status("initialize", {{"protocolVersion": "2025-11-25", "capabilities": {{}}}}, "raw_method_initialize_allowed"), - "raw_method_tools_list_allowed": raw_jsonrpc_status("tools/list", None, "raw_method_tools_list_allowed"), - "raw_method_tools_call_allowed": raw_jsonrpc_status("tools/call", {{"name": "read_status"}}, "raw_method_tools_call_allowed"), - "raw_method_tools_call_with_unmatched_params_allowed": raw_jsonrpc_status("tools/call", {{"name": "blocked_action", "arguments": {{"scope": "ignored"}}}}, "raw_method_tools_call_with_unmatched_params_allowed"), - "raw_method_tools_delete_denied": raw_jsonrpc_status("tools/delete", {{"name": "purge_cache"}}, "raw_method_tools_delete_denied"), -}} -results.update(DETAILS) -print(json.dumps(results, sort_keys=True)) -"#, - host = server.host, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - - for (key, expected) in [ - // forward proxy — allowed - ("forward_method_initialize_allowed", 200), - ("forward_method_tools_list_allowed", 200), - ("forward_method_tools_call_allowed", 200), - ( - "forward_method_tools_call_with_unmatched_params_allowed", - 200, - ), - // forward proxy — method denied - ("forward_method_tools_delete_denied", 403), - // forward proxy — batch - ("forward_batch_all_allowed", 200), - ("forward_batch_one_denied", 403), - // forward proxy — parse error - ("forward_invalid_json_denied", 403), - // raw socket path — allowed - ("raw_method_initialize_allowed", 200), - ("raw_method_tools_list_allowed", 200), - ("raw_method_tools_call_allowed", 200), - ("raw_method_tools_call_with_unmatched_params_allowed", 200), - // raw socket path — method denied - ("raw_method_tools_delete_denied", 403), - ] { - let expected_fragment = format!(r#""{key}": {expected}"#); - assert!( - guard.create_output.contains(&expected_fragment), - "expected {key}={expected}, got:\n{}", - guard.create_output - ); - } -} - -#[tokio::test] -async fn jsonrpc_forward_proxy_hard_denies_response_frames_in_default_audit_mode() { - let server = start_test_server(AUDIT_TEST_SERVER_ALIAS) - .await - .expect("start test server"); - let policy = - write_jsonrpc_default_audit_policy(&server.host, server.port).expect("write custom policy"); - let policy_path = policy - .path() - .to_str() - .expect("temp policy path should be utf-8") - .to_string(); - - let script = format!( - r#" -import json -import urllib.error -import urllib.request - -HOST = {host:?} -PORT = {port} - -def post_jsonrpc(body): - encoded = json.dumps(body).encode() - request = urllib.request.Request( - f"http://{{HOST}}:{{PORT}}/rpc", - data=encoded, - headers={{"Content-Type": "application/json"}}, - method="POST", - ) - try: - with urllib.request.urlopen(request, timeout=15) as response: - response.read() - return response.status - except urllib.error.HTTPError as error: - error.read() - return error.code - -results = {{ - "forward_unknown_method_audited": post_jsonrpc({{"jsonrpc": "2.0", "id": 1, "method": "unknown/method"}}), - "forward_response_frame_hard_denied": post_jsonrpc({{"jsonrpc": "2.0", "id": 1, "result": {{}}}}), -}} -print(json.dumps(results, sort_keys=True)) -"#, - host = server.host, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - - for (key, expected) in [ - ("forward_unknown_method_audited", 200), - ("forward_response_frame_hard_denied", 403), - ] { - let expected_fragment = format!(r#""{key}": {expected}"#); - assert!( - guard.create_output.contains(&expected_fragment), - "expected {key}={expected}, got:\n{}", - guard.create_output - ); - } -} diff --git a/e2e/rust/tests/forward_proxy_l7_bypass.rs b/e2e/rust/tests/forward_proxy_l7_bypass.rs deleted file mode 100644 index f274f20e93..0000000000 --- a/e2e/rust/tests/forward_proxy_l7_bypass.rs +++ /dev/null @@ -1,239 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Regression tests: the forward proxy path must evaluate L7 rules for -//! endpoints that have them configured. Allowed requests (e.g. GET on a -//! read-only endpoint) should succeed; denied requests (e.g. POST) should -//! receive 403. - -#![cfg(feature = "e2e")] - -use std::io::Write; - -use openshell_e2e::harness::container::ContainerHttpServer; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -async fn start_test_server(alias: &str) -> Result { - let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - if self.headers.get("Upgrade"): - self.send_response(101) - self.send_header("Connection", "Upgrade") - self.send_header("Upgrade", "websocket") - self.send_header("Sec-WebSocket-Accept", "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=") - self.end_headers() - return - self.send_response(200) - self.end_headers() - self.wfile.write(b'{"ok":true}') - def do_DELETE(self): - self.send_response(200) - self.end_headers() - self.wfile.write(b'{"ok":true}') - def log_message(self, format, *args): - pass - -HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() -"#; - - ContainerHttpServer::start_python(alias, script).await -} - -fn write_policy_with_l7_rules(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - test_l7: - name: test_l7 - endpoints: - - host: {host} - port: {port} - protocol: rest - enforcement: enforce - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - rules: - - allow: - method: GET - path: /allowed - binaries: - - path: /usr/bin/curl - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -/// GET /allowed should succeed — the L7 policy explicitly allows it. -#[tokio::test] -async fn forward_proxy_allows_l7_permitted_request() { - let server = start_test_server("rest-l7-allow.openshell.test") - .await - .expect("start test server"); - let policy = - write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); - let policy_path = policy - .path() - .to_str() - .expect("temp policy path should be utf-8") - .to_string(); - - let script = format!( - r#" -import urllib.request, urllib.error, json, sys -url = "http://{host}:{port}/allowed" -try: - resp = urllib.request.urlopen(url, timeout=15) - last = {{"status": resp.status, "error": None}} -except urllib.error.HTTPError as e: - last = {{"status": e.code, "error": str(e)}} -except Exception as e: - last = {{"status": -1, "error": str(e)}} -print(json.dumps(last)) -"#, - host = server.host, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - - // L7 policy allows GET /allowed — should succeed. - assert!( - guard.create_output.contains("\"status\": 200"), - "expected 200 for L7-allowed GET, got:\n{}", - guard.create_output - ); -} - -/// POST /allowed should be denied — the L7 policy only allows GET. -#[tokio::test] -async fn forward_proxy_denies_l7_blocked_request() { - let server = start_test_server("rest-l7-deny.openshell.test") - .await - .expect("start test server"); - let policy = - write_policy_with_l7_rules(&server.host, server.port).expect("write custom policy"); - let policy_path = policy - .path() - .to_str() - .expect("temp policy path should be utf-8") - .to_string(); - - let script = format!( - r#" -import urllib.request, urllib.error, json, sys -url = "http://{host}:{port}/allowed" -req = urllib.request.Request(url, data=b"test", method="POST") -try: - resp = urllib.request.urlopen(req, timeout=15) - print(json.dumps({{"status": resp.status, "error": None}})) -except urllib.error.HTTPError as e: - print(json.dumps({{"status": e.code, "error": str(e)}})) -except Exception as e: - print(json.dumps({{"status": -1, "error": str(e)}})) -"#, - host = server.host, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - - // L7 policy denies POST — should return 403. - assert!( - guard.create_output.contains("\"status\": 403"), - "expected 403 for L7-denied POST, got:\n{}", - guard.create_output - ); -} - -/// A permitted GET cannot switch a REST endpoint into an uninspected tunnel. -#[tokio::test] -async fn rest_upgrade_is_refused_in_enforce_and_audit_modes() { - let server = start_test_server("rest-upgrade.openshell.test") - .await - .expect("start backend"); - for enforcement in ["enforce", "audit"] { - let source = write_policy_with_l7_rules(&server.host, server.port).unwrap(); - let policy = std::fs::read_to_string(source.path()).unwrap().replace( - "enforcement: enforce", - &format!("enforcement: {enforcement}"), - ); - let mut file = NamedTempFile::new().unwrap(); - file.write_all(policy.as_bytes()).unwrap(); - file.flush().unwrap(); - let policy_path = file.path().to_str().unwrap(); - let script = format!( - r#" -import socket -for target in ["/allowed", "http://{host}:{port}/allowed"]: - s = socket.create_connection(("{host}", {port}), timeout=15) - request = ("GET " + target + " HTTP/1.1\r\nHost: {host}:{port}\r\n" - "Connection: Upgrade\r\nUpgrade: websocket\r\n" - "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nSec-WebSocket-Version: 13\r\n\r\n") - s.sendall(request.encode()) - response = b"" - while True: - chunk = s.recv(4096) - if not chunk: - break - response += chunk - s.close() - assert response.startswith(b"HTTP/1.1 403"), response - assert b"unsupported_l7_protocol" in response, response - assert b"REST endpoints" in response, response -print("REST_UPGRADE_REFUSED") -"#, - host = server.host, - port = server.port - ); - let guard = - SandboxGuard::create(&["--policy", policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - assert!( - guard.create_output.contains("REST_UPGRADE_REFUSED"), - "{enforcement}: {}", - guard.create_output - ); - } -} diff --git a/e2e/rust/tests/host_gateway_alias.rs b/e2e/rust/tests/host_gateway_alias.rs index e3f8cb1c9f..98f6ec8159 100644 --- a/e2e/rust/tests/host_gateway_alias.rs +++ b/e2e/rust/tests/host_gateway_alias.rs @@ -4,76 +4,14 @@ #![cfg(feature = "e2e")] use std::io::Write; -use std::process::Stdio; -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::container::is_e2e_driver; use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::{Builder as TempFileBuilder, NamedTempFile}; +use tempfile::NamedTempFile; use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; use tokio::net::TcpListener; use tokio::task::JoinHandle; -const BINDING_PROVIDER_A_NAME: &str = "e2e-static-endpoint-binding-provider-a"; -const BINDING_PROVIDER_B_NAME: &str = "e2e-static-endpoint-binding-provider-b"; -const BINDING_PROFILE_A_ID: &str = "e2e-static-endpoint-binding-a"; -const BINDING_PROFILE_B_ID: &str = "e2e-static-endpoint-binding-b"; - -async fn run_cli(args: &[&str]) -> Result { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .map_err(|e| format!("failed to spawn openshell {}: {e}", args.join(" ")))?; - - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - if !output.status.success() { - return Err(format!( - "openshell {} failed (exit {:?}):\n{combined}", - args.join(" "), - output.status.code() - )); - } - - Ok(combined) -} - -async fn wait_for_sandbox_logs( - sandbox_name: &str, - expected: impl Fn(&str) -> bool, -) -> Result { - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); - - loop { - let logs = run_cli(&[ - "logs", - sandbox_name, - "-n", - "500", - "--since", - "2m", - "--source", - "sandbox", - ]) - .await?; - if expected(&logs) { - return Ok(logs); - } - if tokio::time::Instant::now() >= deadline { - return Err(format!( - "timed out waiting for expected sandbox logs:\n{logs}" - )); - } - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } -} - struct HostServer { port: u16, task: JoinHandle<()>, @@ -81,13 +19,6 @@ struct HostServer { impl HostServer { async fn start(response_body: &str) -> Result { - Self::start_with_auth_check(response_body, None).await - } - - async fn start_with_auth_check( - response_body: &str, - expected_authorization: Option<&str>, - ) -> Result { let listener = TcpListener::bind(("0.0.0.0", 0)) .await .map_err(|e| format!("bind host test server: {e}"))?; @@ -96,14 +27,12 @@ impl HostServer { .map_err(|e| format!("read host test server address: {e}"))? .port(); let response_body = response_body.as_bytes().to_vec(); - let expected_authorization = expected_authorization.map(str::to_string); let task = tokio::spawn(async move { loop { let Ok((mut stream, _)) = listener.accept().await else { break; }; let body = response_body.clone(); - let expected_authorization = expected_authorization.clone(); tokio::spawn(async move { let mut request = Vec::new(); let mut buf = [0_u8; 1024]; @@ -119,17 +48,6 @@ impl HostServer { break; } } - - let body = expected_authorization.map_or(body, |expected| { - let request = String::from_utf8_lossy(&request); - format!( - r#"{{"authorized":{}}}"#, - request.lines().any(|line| { - line.eq_ignore_ascii_case(&format!("Authorization: {expected}")) - }) - ) - .into_bytes() - }); let response = format!( "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len() @@ -147,114 +65,11 @@ impl HostServer { } } -fn write_binding_profile( - id: &str, - display_name: &str, - env_var: &str, - host: &str, - port: u16, -) -> Result { - let mut file = TempFileBuilder::new() - .suffix(".yaml") - .tempfile() - .map_err(|e| format!("create provider profile: {e}"))?; - let profile = format!( - r#"id: {id} -display_name: {display_name} -category: other -credentials: - - name: bound_token - env_vars: [{env_var}] - required: true - auth_style: bearer - header_name: authorization -endpoints: - - host: {host} - port: {port} - path: /allowed/** - protocol: rest - access: full - enforcement: enforce -binaries: [/usr/bin/bash] -"# - ); - file.write_all(profile.as_bytes()) - .map_err(|e| format!("write provider profile: {e}"))?; - file.flush() - .map_err(|e| format!("flush provider profile: {e}"))?; - Ok(file) -} - -fn write_binding_policy(port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create binding policy: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/bin, /usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - binding_test: - name: binding_test - endpoints: - - host: host.openshell.internal - port: {port} - path: /** - protocol: rest - access: full - enforcement: enforce - - host: host.docker.internal - port: {port} - path: /** - protocol: rest - access: full - enforcement: enforce - binaries: - - path: /usr/bin/bash -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write binding policy: {e}"))?; - file.flush() - .map_err(|e| format!("flush binding policy: {e}"))?; - Ok(file) -} - impl Drop for HostServer { fn drop(&mut self) { self.task.abort(); } } - -async fn delete_provider(name: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("provider") - .arg("delete") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -async fn delete_provider_profile(id: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("profile") - .arg("delete") - .arg(id) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - fn write_policy(port: u16) -> Result { let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; let policy = format!( @@ -412,157 +227,3 @@ printf 'RESPONSE_EOF\n' sandbox.cleanup().await; } } - -#[tokio::test] -async fn static_provider_credentials_are_bound_to_profile_endpoints() { - let server = HostServer::start_with_auth_check("", Some("Bearer e2e-bound-secret")) - .await - .expect("start credential echo server"); - let profile_a = write_binding_profile( - BINDING_PROFILE_A_ID, - "E2E static endpoint binding A", - "BOUND_TOKEN_A", - "host.openshell.internal", - server.port, - ) - .expect("write provider A binding profile"); - let profile_b = write_binding_profile( - BINDING_PROFILE_B_ID, - "E2E static endpoint binding B", - "BOUND_TOKEN_B", - "host.docker.internal", - server.port, - ) - .expect("write provider B binding profile"); - let policy = write_binding_policy(server.port).expect("write binding policy"); - let profile_a_path = profile_a.path().to_string_lossy().into_owned(); - let profile_b_path = profile_b.path().to_string_lossy().into_owned(); - let policy_path = policy.path().to_string_lossy().into_owned(); - - delete_provider(BINDING_PROVIDER_A_NAME).await; - delete_provider(BINDING_PROVIDER_B_NAME).await; - delete_provider_profile(BINDING_PROFILE_A_ID).await; - delete_provider_profile(BINDING_PROFILE_B_ID).await; - run_cli(&["profile", "import", "--file", &profile_a_path]) - .await - .expect("import provider A endpoint-binding profile"); - run_cli(&["profile", "import", "--file", &profile_b_path]) - .await - .expect("import provider B endpoint-binding profile"); - run_cli(&[ - "provider", - "create", - "--name", - BINDING_PROVIDER_A_NAME, - "--type", - BINDING_PROFILE_A_ID, - "--credential", - "BOUND_TOKEN_A=e2e-bound-secret", - ]) - .await - .expect("create endpoint-bound provider A"); - run_cli(&[ - "provider", - "create", - "--name", - BINDING_PROVIDER_B_NAME, - "--type", - BINDING_PROFILE_B_ID, - "--credential", - "BOUND_TOKEN_B=e2e-provider-b-secret", - ]) - .await - .expect("create endpoint-bound provider B"); - - let command = format!( - r#" -http_request() {{ - local host="$1" path="$2" status_line line - HTTP_STATUS= HTTP_BODY= - if ! exec 3<>"/dev/tcp/$host/{port}"; then - HTTP_STATUS=connect-denied - return 1 - fi - printf 'GET %s HTTP/1.1\r\nHost: %s:{port}\r\nAuthorization: Bearer %s\r\nConnection: close\r\n\r\n' "$path" "$host" "$BOUND_TOKEN_A" >&3 - IFS= read -r status_line <&3 || return 1 - status_line="${{status_line%$'\r'}}" - HTTP_STATUS="${{status_line#* }}" - HTTP_STATUS="${{HTTP_STATUS%% *}}" - while IFS= read -r line <&3; do - line="${{line%$'\r'}}" - [[ -z "$line" ]] && break - done - while IFS= read -r line <&3 || [[ -n "$line" ]]; do HTTP_BODY+="$line"; done - exec 3>&- 3<&- -}} -http_request host.openshell.internal /allowed/check; allowed="$HTTP_BODY" -http_request host.docker.internal /allowed/check || true; host_denied="$HTTP_STATUS" -http_request host.openshell.internal /other/check; path_denied="$HTTP_STATUS" -printf 'ALLOWED=%s HOST_DENIED=%s PATH_DENIED=%s\n' "$allowed" "$host_denied" "$path_denied" -"#, - port = server.port, - ); - let mut guard = SandboxGuard::create(&[ - "--policy", - &policy_path, - "--provider", - BINDING_PROVIDER_A_NAME, - "--provider", - BINDING_PROVIDER_B_NAME, - "--no-auto-providers", - "--", - "/usr/bin/bash", - "-c", - &command, - ]) - .await - .expect("run endpoint-binding requests"); - - let logs = wait_for_sandbox_logs(&guard.name, |logs| { - logs.contains("openshell.provider_credential.endpoint_mismatch") - && logs.contains("credential_endpoint_mismatch") - }) - .await - .expect("fetch endpoint mismatch logs"); - assert!( - guard - .create_output - .contains(r#"ALLOWED={"authorized":true}"#), - "credential should resolve at the bound endpoint:\n{}\nlogs:\n{logs}", - guard.create_output, - ); - let expected_host_denial = if is_e2e_driver("podman") { - "HOST_DENIED=connect-denied" - } else { - "HOST_DENIED=403" - }; - assert!( - guard.create_output.contains(expected_host_denial), - "same placeholder must be denied at an unbound host:\n{}", - guard.create_output - ); - assert!( - guard.create_output.contains("PATH_DENIED=403"), - "same placeholder must be denied at an unbound path on its bound host:\n{}", - guard.create_output - ); - - assert!( - logs.contains("openshell.provider_credential.endpoint_mismatch") - && logs.contains("credential_endpoint_mismatch"), - "OCSF logs should explain the endpoint-binding denial without secret material:\n{logs}" - ); - assert!( - !logs.contains("e2e-bound-secret") - && !logs.contains("e2e-provider-b-secret") - && !logs.contains("BOUND_TOKEN_A") - && !logs.contains("BOUND_TOKEN_B"), - "OCSF logs must not contain credential values or environment keys:\n{logs}" - ); - - guard.cleanup().await; - delete_provider(BINDING_PROVIDER_A_NAME).await; - delete_provider(BINDING_PROVIDER_B_NAME).await; - delete_provider_profile(BINDING_PROFILE_A_ID).await; - delete_provider_profile(BINDING_PROFILE_B_ID).await; -} diff --git a/e2e/rust/tests/listener_lifetime.rs b/e2e/rust/tests/listener_lifetime.rs deleted file mode 100644 index ec58a0dcd2..0000000000 --- a/e2e/rust/tests/listener_lifetime.rs +++ /dev/null @@ -1,154 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -use openshell_e2e::harness::sandbox::SandboxGuard; - -const LIFETIME_PROBE: &str = r" -import errno, json, os, socket, subprocess, sys - -def assert_closed(address, inode): - # Check before bind can trigger the broker's stale-descriptor collection. - for table in ('/proc/net/tcp', '/proc/net/tcp6'): - for line in open(table): - fields = line.split() - assert not (len(fields) > 9 and fields[3] == '0A' and fields[9] == str(inode)), line - with socket.socket() as client: - client.settimeout(2) - assert client.connect_ex(tuple(address)) == errno.ECONNREFUSED, address - with socket.socket() as replacement: - replacement.bind(tuple(address)) - -def inode(sock): - return int(os.readlink('/proc/self/fd/' + str(sock.fileno()))[8:-1]) - -with socket.socket() as bound: - bound.bind(('127.0.0.1', 0)) - address = bound.getsockname() -with socket.socket() as client: - client.settimeout(2) - assert client.connect_ex(address) == errno.ECONNREFUSED - -for host in ('127.0.0.1', '::1'): - family = socket.AF_INET if host == '127.0.0.1' else socket.AF_INET6 - listener = socket.socket(family) - listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - listener.bind((host, 0)) - listener.listen(1) - listener.listen(8) - address, identity = listener.getsockname(), inode(listener) - for operation, expected in ((lambda: listener.bind(address), errno.EINVAL), - (lambda: listener.connect(address), errno.EISCONN)): - try: - operation() - raise AssertionError('operation on listener unexpectedly succeeded') - except OSError as error: - assert error.errno == expected, error - alias = listener.dup() - alias.settimeout(2) - listener.close() - with socket.socket(family) as client: - client.settimeout(2) - client.connect(address) - accepted, _ = alias.accept() - with accepted: - accepted.sendall(b'ok') - assert client.recv(2) == b'ok' - alias.close() - table = '/proc/net/tcp' if family == socket.AF_INET else '/proc/net/tcp6' - for line in open(table): - fields = line.split() - assert not (len(fields) > 9 and fields[3] == '0A' and fields[9] == str(identity)), line - with socket.socket(family) as client: - client.settimeout(2) - assert client.connect_ex(address) == errno.ECONNREFUSED - with socket.socket(family) as replacement: - replacement.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - replacement.bind(address) - -# The child exits while still owning its listener. Its broker must not keep it alive. -child = r''' -import json, os, socket -s = socket.socket() -s.bind(('127.0.0.1', 0)) -s.listen() -inode = int(os.readlink('/proc/self/fd/' + str(s.fileno()))[8:-1]) -print(json.dumps([s.getsockname(), inode]), flush=True) -os._exit(0) -''' -result = subprocess.run([sys.executable, '-c', child], check=True, capture_output=True, text=True, timeout=10) -address, identity = json.loads(result.stdout) -assert_closed(address, identity) -print(json.dumps([address, identity])) -"; - -#[tokio::test] -async fn closing_or_exiting_listener_refuses_connections_and_releases_port() { - let mut sandbox = SandboxGuard::create(&[]).await.expect("create sandbox"); - let output = sandbox - .exec(&["python3", "-c", LIFETIME_PROBE]) - .await - .expect("listener lifetime probe"); - let observer = r" -import json, os, sys -_, inode = json.loads(sys.argv[1]) -for fd in os.listdir('/proc/self/fd'): - try: - target = os.readlink('/proc/self/fd/' + fd) - except FileNotFoundError: - continue - assert target != 'socket:[' + str(inode) + ']', (fd, target) -"; - sandbox - .exec(&["python3", "-c", observer, output.trim()]) - .await - .expect("subsequent exec must not inherit closed listener"); - sandbox.cleanup().await; -} - -#[tokio::test] -async fn unrelated_exec_does_not_inherit_broker_preconnect_socket() { - // Request a non-CLOEXEC socket explicitly; Python's socket constructor - // otherwise masks broker inheritance bugs by requesting SOCK_CLOEXEC. - let owner = r" -import ctypes, json, os, socket, time -libc = ctypes.CDLL(None, use_errno=True) -fd = libc.socket(socket.AF_INET, socket.SOCK_STREAM, 0) -assert fd >= 0, ctypes.get_errno() -assert os.get_inheritable(fd) -inode = int(os.readlink('/proc/self/fd/' + str(fd))[8:-1]) -with open('/sandbox/preconnect-inode.json', 'w') as f: - json.dump(inode, f) -while True: - time.sleep(60) -"; - let mut sandbox = SandboxGuard::create_detached_main(&["python3", "-c", owner]) - .await - .expect("create socket owner"); - let observer = r" -import json, os, time -path = '/sandbox/preconnect-inode.json' -for _ in range(100): - try: - with open(path) as f: - inode = json.load(f) - break - except (FileNotFoundError, json.JSONDecodeError): - time.sleep(0.1) -else: - raise AssertionError('socket owner did not publish its inode') -for fd in os.listdir('/proc/self/fd'): - try: - target = os.readlink('/proc/self/fd/' + fd) - except FileNotFoundError: - continue - assert target != 'socket:[' + str(inode) + ']', (fd, target) -print('broker socket was not inherited') -"; - sandbox - .exec(&["python3", "-c", observer]) - .await - .expect("unrelated exec inherited broker descriptor"); - sandbox.cleanup().await; -} diff --git a/e2e/rust/tests/live_policy_update.rs b/e2e/rust/tests/live_policy_update.rs deleted file mode 100644 index b6a35c8898..0000000000 --- a/e2e/rust/tests/live_policy_update.rs +++ /dev/null @@ -1,704 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! E2E tests for live policy updates on a running sandbox. -//! -//! Covers the full round-trip: -//! - Create sandbox with policy A -//! - Verify initial policy version via `policy get` -//! - Push same policy A again -> no version bump (idempotent) -//! - Push different policy B -> new version, `--wait` for sandbox to load it -//! - Verify policy history via `policy list` -//! -//! These tests replace the Python e2e tests `test_live_policy_update_and_logs` -//! and `test_live_policy_update_from_empty_network_policies`, which were flaky -//! due to hard-coded 90s poll timeouts. The Rust tests use the CLI's built-in -//! `--wait` flag for reliable synchronization. -//! -//! Note: the removed Python tests also covered `GetSandboxLogs` RPC and -//! verified actual proxy connectivity after policy update. Those are tracked -//! as follow-up coverage gaps -- the proxy enforcement path is covered by the -//! existing L4/L7/SSRF Python e2e tests, and log fetching needs a dedicated -//! test. - -#![cfg(feature = "e2e")] - -use std::fmt::Write as _; -use std::io::Write; -use std::process::Stdio; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::{extract_field, strip_ansi}; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const SPARSE_POLICY: &str = include_str!(concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../examples/policy-advisor/sandbox-policy.yaml" -)); - -// --------------------------------------------------------------------------- -// Policy YAML builders -// --------------------------------------------------------------------------- - -/// Build a policy YAML that allows any binary to reach the given hosts on -/// port 443. Keep its filesystem paths aligned with the empty-network policy -/// so live network updates do not remove startup filesystem access. -/// -/// NOTE: The indentation in the format string is load-bearing YAML structure. -fn write_policy(hosts: &[&str]) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - - let mut network_rules = String::new(); - for (i, host) in hosts.iter().enumerate() { - let _ = write!( - network_rules, - r#" rule_{i}: - name: rule_{i} - endpoints: - - host: {host} - port: 443 - binaries: - - path: "/**" -"# - ); - } - - let policy = format!( - r"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /bin - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -network_policies: -{network_rules}" - ); - - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -/// Build a minimal policy YAML with no network rules. Both /bin and /usr -/// are readable so shell entrypoints work on merged and unmerged images. -fn write_empty_network_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - - let policy = r"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /bin - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort -"; - - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -fn write_sparse_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - file.write_all(SPARSE_POLICY.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -// --------------------------------------------------------------------------- -// CLI helpers -// --------------------------------------------------------------------------- - -struct CliResult { - success: bool, - output: String, - exit_code: Option, -} - -/// Run an `openshell` CLI command and return the result. -async fn run_cli(args: &[&str]) -> CliResult { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd.output().await.expect("spawn openshell command"); - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = strip_ansi(&format!("{stdout}{stderr}")); - - CliResult { - success: output.status.success(), - output: combined, - exit_code: output.status.code(), - } -} - -/// Extract the policy version number from `policy get` output. -/// -/// Uses the shared `extract_field` helper to find `Version: ` or -/// `Revision: ` in CLI tabular output. -fn extract_version(output: &str) -> Option { - extract_field(output, "Version") - .or_else(|| extract_field(output, "Revision")) - .and_then(|v| v.parse::().ok()) -} - -/// Extract the policy hash from `policy get` output. -fn extract_hash(output: &str) -> Option { - extract_field(output, "Hash").or_else(|| extract_field(output, "Policy hash")) -} - -/// Check that a version number appears in `policy list` output as a -/// distinct field value (not just a substring of some other number). -/// -/// Looks for the version number preceded by whitespace or at the start -/// of a line, to avoid matching "2" inside "12" or timestamps. -fn list_output_contains_version(output: &str, version: u32) -> bool { - let v = version.to_string(); - output.lines().any(|line| { - line.split_whitespace() - .any(|word| word == v || word.starts_with(&format!("{v} "))) - }) -} - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -/// Read the effective policy through the same CLI boundary used by operators. -async fn l7_scope_snapshot(name: &str) -> serde_json::Value { - let result = run_cli(&["policy", "get", name, "--full", "--output", "json"]).await; - assert!(result.success, "policy snapshot failed: {}", result.output); - let snapshot: serde_json::Value = - serde_json::from_str(&result.output).expect("policy get returns JSON"); - assert!(snapshot["version"].as_u64().is_some()); - assert!( - snapshot["hash"] - .as_str() - .is_some_and(|hash| !hash.is_empty()) - ); - assert!(snapshot["policy"]["network_policies"].is_object()); - snapshot -} - -/// Incomplete declarations reject without a revision; an explicit target -/// changes only its endpoint even when another rule shares the host and ports. -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn l7_append_target_scope_round_trip() { - let mut policy = write_empty_network_policy().expect("write base policy"); - policy - .write_all( - br" -network_policies: - internal_api: - name: internal_api - binaries: - - path: /usr/bin/curl - - path: /usr/bin/python3 - endpoints: - - host: api.example.com - ports: [443, 8443] - protocol: rest - access: read-only - - host: other.example.com - port: 443 - protocol: rest - access: read-only - sibling: - name: sibling - binaries: - - path: /usr/bin/wget - endpoints: - - host: api.example.com - ports: [443, 8443] - protocol: rest - access: read-only -", - ) - .expect("write scoped policy"); - policy.flush().expect("flush scoped policy"); - let path = policy.path().to_str().expect("UTF-8 policy path"); - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", path, "--no-tty"], - &["sh", "-c", "echo Ready && sleep infinity"], - "Ready", - ) - .await - .expect("create scoped-policy sandbox"); - let before = l7_scope_snapshot(&guard.name).await; - - // Check each independent axis with the other fully declared, so a guard - // requiring both mismatches at once cannot satisfy this regression. - for (ports, binaries) in [ - ( - "api.example.com:443:POST:/admin", - vec!["/usr/bin/curl", "/usr/bin/python3"], - ), - ( - "api.example.com:443,8443:POST:/admin", - vec!["/usr/bin/curl"], - ), - ] { - let mut args = vec![ - "policy", - "update", - &guard.name, - "--rule-name", - "internal_api", - "--add-allow", - ports, - ]; - for binary in binaries { - args.extend(["--binary", binary]); - } - let rejected = run_cli(&args).await; - assert!( - !rejected.success, - "partial scope was accepted: {}", - rejected.output - ); - let unchanged = l7_scope_snapshot(&guard.name).await; - for field in ["version", "hash", "policy"] { - assert_eq!(unchanged[field], before[field], "rejection changed {field}"); - } - } - - let common = [ - "policy", - "update", - &guard.name, - "--rule-name", - "internal_api", - "--binary", - "/usr/bin/curl", - "--binary", - "/usr/bin/python3", - ]; - let mut allow_args = common.to_vec(); - allow_args.extend([ - "--add-allow", - "api.example.com:443,8443:POST:/admin", - "--wait", - ]); - let accepted = run_cli(&allow_args).await; - assert!( - accepted.success, - "explicit allow failed: {}", - accepted.output - ); - let after_allow = l7_scope_snapshot(&guard.name).await; - assert!(after_allow["version"].as_u64() > before["version"].as_u64()); - assert_ne!(after_allow["hash"], before["hash"]); - let rules = &after_allow["policy"]["network_policies"]; - assert_eq!( - rules["sibling"], - before["policy"]["network_policies"]["sibling"] - ); - assert_eq!( - rules["internal_api"]["endpoints"][1], - before["policy"]["network_policies"]["internal_api"]["endpoints"][1] - ); - assert!( - rules["internal_api"]["endpoints"][0]["rules"] - .as_array() - .expect("allow rules") - .iter() - .any(|rule| rule["allow"]["method"] == "POST" && rule["allow"]["path"] == "/admin") - ); - - let mut deny_args = common.to_vec(); - deny_args.extend([ - "--add-deny", - "api.example.com:443,8443:POST:/admin/private", - "--wait", - ]); - let denied = run_cli(&deny_args).await; - assert!(denied.success, "explicit deny failed: {}", denied.output); - let after_deny = l7_scope_snapshot(&guard.name).await; - assert!(after_deny["version"].as_u64() > after_allow["version"].as_u64()); - assert_eq!( - after_deny["policy"]["network_policies"]["sibling"], - before["policy"]["network_policies"]["sibling"] - ); - assert!( - after_deny["policy"]["network_policies"]["internal_api"]["endpoints"][0]["deny_rules"] - .as_array() - .expect("deny rules") - .iter() - .any(|rule| rule["method"] == "POST" && rule["path"] == "/admin/private") - ); - guard.cleanup().await; -} - -/// Test the full live policy update lifecycle: -/// -/// 1. Create sandbox with policy A and `--keep` -/// 2. Verify initial version >= 1 -/// 3. Push same policy A -> version unchanged (idempotent) -/// 4. Push policy B (adds example.com) with `--wait` -> new version -/// 5. Push policy B again -> idempotent -/// 6. Verify policy list shows both versions -#[tokio::test] -#[allow(clippy::too_many_lines)] -async fn live_policy_update_round_trip() { - // --- Write two distinct policy files --- - let policy_a = write_policy(&["api.anthropic.com"]).expect("write policy A"); - let policy_b = write_policy(&["api.anthropic.com", "example.com"]).expect("write policy B"); - - let policy_a_path = policy_a - .path() - .to_str() - .expect("policy A path should be utf-8") - .to_string(); - let policy_b_path = policy_b - .path() - .to_str() - .expect("policy B path should be utf-8") - .to_string(); - - // --- Create a long-running sandbox with its startup-only policy fields --- - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &policy_a_path, "--no-tty"], - &["sh", "-c", "echo Ready && sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox with policy A"); - - // --- Verify initial policy version --- - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!( - r.success, - "policy get should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - let initial_version = extract_version(&r.output).unwrap_or_else(|| { - panic!( - "could not parse version from policy get output:\n{}", - r.output - ) - }); - assert!( - initial_version >= 1, - "initial policy version should be >= 1, got {initial_version}" - ); - - let initial_hash = extract_hash(&r.output); - - // --- Push same policy A again -> should be idempotent --- - let r = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &policy_a_path, - "--wait", - "--timeout", - "120", - ]) - .await; - assert!( - r.success, - "policy set A (repeat) should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!( - r.success, - "policy get after repeat should succeed:\n{}", - r.output - ); - - let repeat_version = extract_version(&r.output) - .unwrap_or_else(|| panic!("could not parse version after repeat:\n{}", r.output)); - assert_eq!( - repeat_version, initial_version, - "same policy should not bump version: expected {initial_version}, got {repeat_version}" - ); - - if let (Some(ih), Some(rh)) = (&initial_hash, &extract_hash(&r.output)) { - assert_eq!(ih, rh, "same policy should produce same hash"); - } - - // --- Push policy B -> should create new version --- - let r = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &policy_b_path, - "--wait", - "--timeout", - "120", - ]) - .await; - assert!( - r.success, - "policy set B should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!( - r.success, - "policy get after B should succeed:\n{}", - r.output - ); - - let new_version = extract_version(&r.output) - .unwrap_or_else(|| panic!("could not parse version after B:\n{}", r.output)); - assert!( - new_version > initial_version, - "different policy should bump version: expected > {initial_version}, got {new_version}" - ); - - if let (Some(ih), Some(nh)) = (&initial_hash, &extract_hash(&r.output)) { - assert_ne!(ih, nh, "different policy should produce different hash"); - } - - // --- Push policy B again -> idempotent --- - let r = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &policy_b_path, - "--wait", - "--timeout", - "120", - ]) - .await; - assert!( - r.success, - "policy set B (repeat) should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!( - r.success, - "policy get after B repeat should succeed:\n{}", - r.output - ); - - let repeat_b_version = extract_version(&r.output) - .unwrap_or_else(|| panic!("could not parse version after B repeat:\n{}", r.output)); - assert_eq!( - repeat_b_version, new_version, - "same policy B should not bump version: expected {new_version}, got {repeat_b_version}" - ); - - // --- Verify policy list shows revision history --- - let r = run_cli(&["policy", "list", &guard.name]).await; - assert!( - r.success, - "policy list should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - // Both versions should appear in the list output. - assert!( - list_output_contains_version(&r.output, new_version), - "policy list should contain version {new_version}:\n{}", - r.output - ); - assert!( - list_output_contains_version(&r.output, initial_version), - "policy list should contain initial version {initial_version}:\n{}", - r.output - ); - - guard.cleanup().await; -} - -/// Test live policy update from an initially empty network policy: -/// -/// 1. Create sandbox with no network rules and `--keep` -/// 2. Push policy with a network rule using `--wait` -/// 3. Verify the version bumped -#[tokio::test] -async fn live_policy_update_from_empty_network_policies() { - let empty_policy = write_empty_network_policy().expect("write empty network policy"); - let full_policy = write_policy(&["example.com"]).expect("write full policy"); - - let empty_path = empty_policy - .path() - .to_str() - .expect("empty policy path should be utf-8") - .to_string(); - let full_path = full_policy - .path() - .to_str() - .expect("full policy path should be utf-8") - .to_string(); - - // Create the sandbox with the empty network policy so subsequent live - // updates retain the same startup-only filesystem, landlock, and process - // fields. - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &empty_path, "--no-tty"], - &["sh", "-c", "echo Ready && sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox with empty network policy"); - - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!( - r.success, - "policy get (empty) should succeed:\n{}", - r.output - ); - - let initial_version = extract_version(&r.output) - .unwrap_or_else(|| panic!("could not parse version from empty policy:\n{}", r.output)); - - // Push policy with network rules. - let r = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &full_path, - "--wait", - "--timeout", - "120", - ]) - .await; - assert!( - r.success, - "policy set (full) should succeed (exit {:?}):\n{}", - r.exit_code, r.output - ); - - let r = run_cli(&["policy", "get", &guard.name]).await; - assert!(r.success, "policy get (full) should succeed:\n{}", r.output); - - let new_version = extract_version(&r.output).unwrap_or_else(|| { - panic!( - "could not parse version after adding network rules:\n{}", - r.output - ) - }); - assert!( - new_version > initial_version, - "adding network rules should create new version > {initial_version}, got {new_version}" - ); - - guard.cleanup().await; -} - -/// Regression for #2159: a sparse initial policy that the supervisor enriches -/// with baseline filesystem paths during startup must have its resulting -/// revision acknowledged as loaded, not left `Pending`. -/// -/// Reproduction (matches the maintainer's triage): create a sandbox with the -/// network-only `examples/policy-advisor/sandbox-policy.yaml`. The supervisor -/// adds baseline filesystem paths, syncs the enriched policy back to the -/// gateway (creating revision 2, superseding revision 1), builds the OPA engine -/// and then acknowledges revision 2 as loaded. Before the fix, revision 2 -/// stayed `Pending` even though the sandbox was `Ready` and the policy was -/// effective. -/// -/// NOTE: This exercises the Docker-backed supervisor built from this branch. -/// The exact `policy list` status wording ("Loaded"/"Superseded") may differ by -/// CLI version; the assertions below key on the effective version reaching 2 and -/// no revision remaining `Pending` once the acknowledgement lands. -#[tokio::test] -async fn initial_sparse_policy_is_acknowledged_as_loaded() { - let sparse_policy = write_sparse_policy().expect("write sparse policy fixture"); - let sparse_policy_path = sparse_policy - .path() - .to_str() - .expect("sparse policy path is not UTF-8"); - - let mut guard = SandboxGuard::create_keep_with_args( - &[ - "--name", - "e2e-sparse-enrich", - "--policy", - sparse_policy_path, - "--no-tty", - ], - &["sh", "-c", "echo Ready && sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox with sparse policy"); - - // The enriched revision (2) is synced during startup; the acknowledgement - // (LOADED) is delivered by the supervisor's poll loop shortly after Ready. - // Poll until the effective policy is version 2 and no revision is Pending. - let mut acknowledged = false; - let mut last_list = String::new(); - for _ in 0..30 { - let get = run_cli(&["policy", "get", &guard.name]).await; - let version = get.success.then(|| extract_version(&get.output)).flatten(); - - let list = run_cli(&["policy", "list", &guard.name]).await; - last_list = list.output.clone(); - let pending = list.output.to_lowercase().contains("pending"); - - if version == Some(2) && list.success && !pending { - acknowledged = true; - break; - } - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - } - - assert!( - acknowledged, - "enriched initial policy should reach revision 2 with no Pending revision.\n\ - last `policy list` output:\n{last_list}" - ); - - // Both the superseded original (1) and the loaded enriched revision (2) - // must appear in the revision history. - assert!( - list_output_contains_version(&last_list, 2), - "policy list should contain revision 2:\n{last_list}" - ); - assert!( - list_output_contains_version(&last_list, 1), - "policy list should contain revision 1:\n{last_list}" - ); - - guard.cleanup().await; -} diff --git a/e2e/rust/tests/mcp_sessionless.rs b/e2e/rust/tests/mcp_sessionless.rs deleted file mode 100644 index bf2261efa1..0000000000 --- a/e2e/rust/tests/mcp_sessionless.rs +++ /dev/null @@ -1,381 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! MCP request profiles through the sandbox's transparent network interception. -//! -//! A shared fixture serves legacy initialization, sessionless discovery, named -//! tools, and a bounded subscription stream. Upstream receipts distinguish a -//! policy denial from a tool request accepted by the fixture. - -#![cfg(feature = "e2e-host-gateway")] - -use std::io::Write; - -use openshell_e2e::harness::container::ContainerHttpServer; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tempfile::NamedTempFile; - -const SERVER_ALIAS: &str = "mcp-sessionless.openshell.test"; - -const SERVER_SCRIPT: &str = r#" -import json -from http.server import BaseHTTPRequestHandler, HTTPServer - -SESSIONLESS_VERSION = "2026-07-28" -received = [] - -class Handler(BaseHTTPRequestHandler): - def reply(self, status, payload, content_type="application/json"): - self.send_response(status) - self.send_header("Content-Type", content_type) - self.send_header("Content-Length", str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def do_GET(self): - # The container fixture's readiness probe uses the root URL. - self.reply(200 if self.path == "/" else 405, b"") - - def read_body(self): - if self.headers.get("Transfer-Encoding", "").lower() != "chunked": - return self.rfile.read(int(self.headers.get("Content-Length", "0"))) - body = bytearray() - while True: - size = int(self.rfile.readline().split(b";", 1)[0].strip(), 16) - if size == 0: - while self.rfile.readline().strip(): - pass - return bytes(body) - body.extend(self.rfile.read(size)) - self.rfile.read(2) - - def do_POST(self): - message = json.loads(self.read_body()) - method = message["method"] - params = message.get("params", {}) - version = (params.get("protocolVersion") if method == "initialize" - else self.headers.get("MCP-Protocol-Version")) - # Record parsed requests before fixture admission so rejected revisions - # and tool attempts stay visible. HTTPServer handles requests serially. - received.append([version, method, params.get("name")]) - if self.path != "/mcp" or version not in SUPPORTED_VERSIONS: - self.reply(400, b"request revision did not reach the fixture intact") - return - if version == SESSIONLESS_VERSION: - meta = params.get("_meta", {}) - if (self.headers.get("MCP-Protocol-Version") != version - or self.headers.get("Mcp-Method") != method - or meta.get("io.modelcontextprotocol/protocolVersion") != version - or meta.get("io.modelcontextprotocol/clientCapabilities") != {} - or (method == "tools/call" - and self.headers.get("Mcp-Name") != params["name"])): - self.reply(400, b"request metadata did not reach the fixture intact") - return - - if method == "initialize" and version != SESSIONLESS_VERSION: - result = { - "protocolVersion": version, - "capabilities": {"tools": {}}, - "serverInfo": {"name": "openshell-profile-fixture", "version": "1"}, - } - elif method == "notifications/initialized" and version != SESSIONLESS_VERSION: - self.reply(202, b"") - return - elif method == "server/discover" and version == SESSIONLESS_VERSION: - result = { - "supportedVersions": [version], - "capabilities": {"tools": {"listChanged": True}}, - "ttlMs": 0, - "cacheScope": "private", - "_meta": { - "io.modelcontextprotocol/serverInfo": { - "name": "openshell-sessionless-fixture", "version": "1" - } - }, - } - elif method == "tools/call": - # Both tool names work upstream; OpenShell owns the policy denial. - result = { - "content": [{"type": "text", "text": params["name"]}], - "isError": False, - "_meta": {"fixtureRequests": list(received)}, - } - if version == SESSIONLESS_VERSION: - result["resultType"] = "complete" - elif method == "subscriptions/listen" and version == SESSIONLESS_VERSION: - subscription_meta = {"io.modelcontextprotocol/subscriptionId": message["id"]} - events = [ - { - "jsonrpc": "2.0", - "method": "notifications/subscriptions/acknowledged", - "params": {"notifications": params["notifications"], "_meta": subscription_meta}, - }, - { - "jsonrpc": "2.0", - "method": "notifications/tools/list_changed", - "params": {"_meta": subscription_meta}, - }, - { - "jsonrpc": "2.0", - "id": message["id"], - "result": {"resultType": "complete", "_meta": subscription_meta}, - }, - ] - body = "".join("event: message\ndata: " + json.dumps(event) + "\n\n" for event in events) - self.reply(200, body.encode(), "text/event-stream") - return - else: - self.reply(400, b"unexpected method for the selected fixture revision") - return - - self.reply(200, json.dumps({"jsonrpc": "2.0", "id": message["id"], "result": result}).encode()) - - def log_message(self, format, *args): - pass - -HTTPServer(("0.0.0.0", 8000), Handler).serve_forever() -"#; - -const CLIENT_HELPERS: &str = r#" -import json -import urllib.error -import urllib.request -# Direct client connections pass through the sandbox's transparent interception. -opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) - -def post(request_id, method, params, version): - params = dict(params) - headers = { - "Content-Type": "application/json", - "Accept": "application/json, text/event-stream", - } - if method != "initialize": - headers["MCP-Protocol-Version"] = version - if version == "2026-07-28": - params["_meta"] = { - "io.modelcontextprotocol/protocolVersion": version, - "io.modelcontextprotocol/clientCapabilities": {}, - "io.modelcontextprotocol/clientInfo": {"name": "openshell-e2e", "version": "1"}, - } - headers["Mcp-Method"] = method - if method == "tools/call": - headers["Mcp-Name"] = params["name"] - message = {"jsonrpc": "2.0", "method": method, "params": params} - if request_id is not None: - message["id"] = request_id - request = urllib.request.Request( - f"http://{HOST}:{PORT}/mcp", - data=json.dumps(message).encode(), - headers=headers, - method="POST", - ) - try: - with opener.open(request, timeout=15) as response: - return response.status, response.headers.get_content_type(), response.read() - except urllib.error.HTTPError as error: - return error.code, error.headers.get_content_type(), error.read() -"#; - -const CLIENT_SCRIPT: &str = r#" -VERSION = "2026-07-28" - -status, content_type, body = post(1, "server/discover", {}, VERSION) -assert status == 200, ("discovery", status, body) -assert content_type == "application/json", content_type -discovery = json.loads(body) -assert discovery["id"] == 1, discovery -assert discovery["result"]["supportedVersions"] == [VERSION], discovery - -status, _, body = post(2, "tools/call", {"name": "read_status", "arguments": {}}, VERSION) -assert status == 200, ("allowed tool", status, body) -tool = json.loads(body) -assert tool["id"] == 2, tool -assert tool["result"]["content"] == [{"type": "text", "text": "read_status"}], tool - -status, _, body = post(3, "tools/call", {"name": "read_details", "arguments": {}}, VERSION) -assert status == 403, ("denied tool", status, body) - -status, content_type, body = post(4, "subscriptions/listen", {"notifications": {"toolsListChanged": True}}, VERSION) -assert status == 200, ("subscription", status, body) -assert content_type == "text/event-stream", (content_type, body) -events = [json.loads(line[6:]) for line in body.decode().splitlines() if line.startswith("data: ")] -assert len(events) == 3, events -assert events[0]["method"] == "notifications/subscriptions/acknowledged", events -assert events[0]["params"]["notifications"] == {"toolsListChanged": True}, events -assert events[1]["method"] == "notifications/tools/list_changed", events -assert events[1]["params"]["_meta"]["io.modelcontextprotocol/subscriptionId"] == 4, events -assert events[2]["id"] == 4 and events[2]["result"]["resultType"] == "complete", events - -print("MCP_SESSIONLESS_OK discovery=200 allowed_tool=200 denied_tool=403 subscription=200") -"#; - -const PROFILE_CLIENT_SCRIPT: &str = r#" -expected_receipts = [] -for version in SELECTED_VERSIONS: - if version == "2026-07-28": - status, _, body = post(1, "server/discover", {}, version) - assert status == 200, (version, "discovery", status, body) - assert json.loads(body)["result"]["supportedVersions"] == [version], body - expected_receipts.append([version, "server/discover", None]) - else: - status, _, body = post(1, "initialize", { - "protocolVersion": version, - "capabilities": {}, - "clientInfo": {"name": "openshell-e2e", "version": "1"}, - }, version) - assert status == 200, (version, "initialize", status, body) - assert json.loads(body)["result"]["protocolVersion"] == version, body - expected_receipts.append([version, "initialize", None]) - status, _, body = post(None, "notifications/initialized", {}, version) - assert status == 202, (version, "initialized", status, body) - expected_receipts.append([version, "notifications/initialized", None]) - - status, _, body = post(2, "tools/call", {"name": "read_status", "arguments": {}}, version) - assert status == 200, (version, "allowed tool", status, body) - tool = json.loads(body) - assert tool["id"] == 2, tool - assert tool["result"]["content"] == [{"type": "text", "text": "read_status"}], tool - expected_receipts.append([version, "tools/call", "read_status"]) - assert tool["result"]["_meta"]["fixtureRequests"] == expected_receipts, tool - - status, _, body = post(3, "tools/call", {"name": "read_details", "arguments": {}}, version) - assert status == 403, (version, "denied tool", status, body) - - # The fixture accepts both tools. A later allowed call proves the denial - # came from the proxy and no denied operation reached the upstream. - status, _, body = post(4, "tools/call", {"name": "read_status", "arguments": {}}, version) - assert status == 200, (version, "receipt tool", status, body) - receipt = json.loads(body) - assert receipt["id"] == 4, receipt - expected_receipts.append([version, "tools/call", "read_status"]) - assert receipt["result"]["_meta"]["fixtureRequests"] == expected_receipts, receipt - print(f"MCP_PROFILE_OK version={version} allowed_tool=200 denied_tool=403 receipts=verified") -"#; - -async fn start_server(alias: &str, versions: &[&str]) -> Result { - let versions = serde_json::to_string(versions).map_err(|err| err.to_string())?; - let script = format!("SUPPORTED_VERSIONS = {versions}\n{SERVER_SCRIPT}"); - ContainerHttpServer::start_python(alias, &script).await -} - -fn write_policy(host: &str, port: u16, versions: &[&str]) -> Result { - let mut file = NamedTempFile::new().map_err(|err| format!("create temp policy: {err}"))?; - let legacy_rules = if versions.iter().any(|version| *version != "2026-07-28") { - " - allow:\n method: initialize\n - allow:\n method: notifications/initialized\n" - } else { - "" - }; - let versions = serde_json::to_string(versions).map_err(|err| err.to_string())?; - let policy = format!( - r#"version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox -network_policies: - mcp_sessionless: - name: mcp_sessionless - endpoints: - - host: {host} - port: {port} - path: /mcp - protocol: mcp - enforcement: enforce - allowed_ips: ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "fc00::/7"] - mcp: - versions: {versions} - max_body_bytes: 65536 - rules: -{legacy_rules} - allow: - method: server/discover - - allow: - method: tools/call - tool: read_status - - allow: - method: subscriptions/listen - deny_rules: - - method: tools/call - tool: read_details - binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|err| format!("write temp policy: {err}"))?; - file.flush() - .map_err(|err| format!("flush temp policy: {err}"))?; - Ok(file) -} - -async fn run_client( - server: &ContainerHttpServer, - versions: &[&str], - client: &str, -) -> Result { - let policy = write_policy(&server.host, server.port, versions)?; - let policy_path = policy - .path() - .to_str() - .ok_or("temp policy path is not UTF-8")?; - let selected_versions = serde_json::to_string(versions).map_err(|err| err.to_string())?; - let script = format!( - "HOST = {:?}\nPORT = {}\nSELECTED_VERSIONS = {selected_versions}\n{CLIENT_HELPERS}\n{client}", - server.host, server.port - ); - SandboxGuard::create(&["--policy", policy_path, "--", "python3", "-c", &script]).await -} - -#[tokio::test] -async fn sessionless_discovery_tools_and_subscription_use_request_metadata() { - let versions = ["2026-07-28"]; - let server = start_server(SERVER_ALIAS, &versions) - .await - .expect("start sessionless MCP fixture"); - let sandbox = run_client(&server, &versions, CLIENT_SCRIPT) - .await - .expect("run sessionless MCP client in sandbox"); - - assert!( - sandbox.create_output.contains( - "MCP_SESSIONLESS_OK discovery=200 allowed_tool=200 denied_tool=403 subscription=200" - ), - "expected completed sessionless MCP assertions, got:\n{}", - sandbox.create_output - ); -} - -#[tokio::test] -async fn legacy_and_multi_version_profiles_authorize_tools_through_sandbox() { - for versions in [ - &["2025-03-26"][..], - &["2025-06-18"][..], - &["2025-11-25", "2026-07-28"][..], - ] { - // Each scenario starts with fresh upstream receipts. Its distinct alias - // avoids the sessionless test's fixture, and cleanup precedes alias reuse. - let server = start_server("mcp-profiles.openshell.test", versions) - .await - .unwrap_or_else(|err| panic!("{versions:?}: start MCP fixture: {err}")); - let mut sandbox = run_client(&server, versions, PROFILE_CLIENT_SCRIPT) - .await - .unwrap_or_else(|err| panic!("{versions:?}: run MCP sandbox client: {err}")); - for version in versions { - let marker = format!( - "MCP_PROFILE_OK version={version} allowed_tool=200 denied_tool=403 receipts=verified" - ); - assert!( - sandbox.create_output.contains(&marker), - "{versions:?}: expected completed {version} assertions, got:\n{}", - sandbox.create_output - ); - } - sandbox.cleanup().await; - } -} diff --git a/e2e/rust/tests/no_proxy.rs b/e2e/rust/tests/no_proxy.rs deleted file mode 100644 index 447c408f8b..0000000000 --- a/e2e/rust/tests/no_proxy.rs +++ /dev/null @@ -1,59 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -use openshell_e2e::harness::sandbox::SandboxGuard; - -fn localhost_transparent_script() -> &'static str { - r#" -import json -import os -import threading -import urllib.request -from http.server import BaseHTTPRequestHandler, HTTPServer - -for name in ('HTTP_PROXY', 'HTTPS_PROXY', 'NO_PROXY', 'http_proxy', 'https_proxy', 'no_proxy'): - assert name not in os.environ, f'unexpected proxy environment variable: {name}' - -class Handler(BaseHTTPRequestHandler): - def log_message(self, format, *args): - pass - - def do_GET(self): - self.send_response(200) - self.send_header('Content-Type', 'application/json') - self.end_headers() - self.wfile.write(b'{"message":"hello"}') - -server = HTTPServer(('127.0.0.1', 0), Handler) -thread = threading.Thread(target=server.serve_forever, daemon=True) -thread.start() - -try: - with urllib.request.urlopen(f'http://127.0.0.1:{server.server_port}', timeout=10) as response: - print(json.dumps({ - 'proxy_env_absent': True, - 'payload': json.loads(response.read().decode()), - }), flush=True) -finally: - server.shutdown() - thread.join(timeout=5) - server.server_close() -"# -} - -#[tokio::test] -async fn sandbox_reaches_localhost_without_proxy_environment() { - let guard = SandboxGuard::create(&["--", "python3", "-c", localhost_transparent_script()]) - .await - .expect("sandbox create with transparent localhost check"); - - assert!( - guard - .create_output - .contains(r#"{"proxy_env_absent": true, "payload": {"message": "hello"}}"#), - "expected localhost HTTP request to stay local and succeed:\n{}", - guard.create_output - ); -} diff --git a/e2e/rust/tests/port_forward.rs b/e2e/rust/tests/port_forward.rs deleted file mode 100644 index e91919d177..0000000000 --- a/e2e/rust/tests/port_forward.rs +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E test: TCP port forwarding through a sandbox. -//! -//! Prerequisites: -//! - A running openshell gateway (`mise run gateway:docker`) -//! - The `openshell` binary (built automatically from the workspace) - -use std::time::Duration; - -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::TcpStream; - -use openshell_e2e::harness::port::{find_free_port, wait_for_port}; -use openshell_e2e::harness::sandbox::SandboxGuard; - -/// Python script that runs a single-threaded TCP echo server inside the -/// sandbox. It prints `echo-server-ready` to stdout once listening, which -/// the harness uses as the readiness marker. -fn echo_server_script(port: u16) -> String { - format!( - r" -import socket, sys, signal -signal.signal(signal.SIGHUP, signal.SIG_IGN) -signal.signal(signal.SIGTERM, lambda *_: sys.exit(0)) -port = {port} -sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) -sock.bind(('127.0.0.1', port)) -sock.listen(1) -sock.settimeout(300) -print('echo-server-ready', flush=True) -try: - while True: - conn, _ = sock.accept() - data = conn.recv(4096) - if data: - conn.sendall(b'echo:' + data) - conn.close() -except (socket.timeout, OSError): - pass -finally: - sock.close() -" - ) -} - -/// Create a sandbox with a TCP echo server, forward the port locally, send -/// data through it, and verify the echoed response. -#[tokio::test] -async fn port_forward_echo() { - let port = find_free_port(); - let script = echo_server_script(port); - - // --------------------------------------------------------------- - // Step 1 — Create a sandbox with the echo server running. - // --------------------------------------------------------------- - let mut guard = SandboxGuard::create_keep(&["python3", "-c", &script], "echo-server-ready") - .await - .expect("sandbox create with echo server"); - - // --------------------------------------------------------------- - // Step 2 — Start port forwarding in the background. - // --------------------------------------------------------------- - let mut forward_child = guard.spawn_forward(port).expect("spawn port forward"); - - // Wait for the local port to accept connections. - wait_for_port("127.0.0.1", port, Duration::from_secs(30)) - .await - .expect("local port should open for forwarding"); - - // Give the SSH tunnel a moment to fully establish the direct-tcpip channel. - tokio::time::sleep(Duration::from_secs(2)).await; - - // --------------------------------------------------------------- - // Step 3 — Send data through the forwarded port and verify response. - // --------------------------------------------------------------- - let expected = "echo:hello-nav"; - let mut last_response = String::new(); - - for attempt in 1..=5 { - match try_echo(port).await { - Ok(resp) if resp.starts_with(expected) => { - last_response = resp; - break; - } - Ok(resp) => { - last_response = resp; - eprintln!("attempt {attempt}: unexpected response '{last_response}', retrying..."); - } - Err(e) => { - eprintln!("attempt {attempt}: connection error: {e}, retrying..."); - } - } - tokio::time::sleep(Duration::from_secs(2)).await; - } - - assert!( - last_response.starts_with(expected), - "expected response starting with '{expected}', got '{last_response}'" - ); - - // --------------------------------------------------------------- - // Cleanup — kill forward process, then sandbox guard handles the rest. - // --------------------------------------------------------------- - let _ = forward_child.kill().await; - let _ = forward_child.wait().await; - guard.cleanup().await; -} - -/// Attempt to send `hello-nav\n` to the echo server and read the response. -#[allow( - clippy::disallowed_methods, - reason = "the e2e workspace does not depend on openshell-core" -)] -async fn try_echo(port: u16) -> Result { - let mut stream = TcpStream::connect(format!("127.0.0.1:{port}")) - .await - .map_err(|e| format!("connect: {e}"))?; - - stream - .write_all(b"hello-nav\n") - .await - .map_err(|e| format!("write: {e}"))?; - - let mut buf = vec![0u8; 4096]; - let n = tokio::time::timeout(Duration::from_secs(10), stream.read(&mut buf)) - .await - .map_err(|_| "read timeout".to_string())? - .map_err(|e| format!("read: {e}"))?; - - let response = String::from_utf8_lossy(&buf[..n]) - .trim_end_matches(['\r', '\n']) - .to_string(); - - Ok(response) -} diff --git a/e2e/rust/tests/proxy_egress_pipeline.rs b/e2e/rust/tests/proxy_egress_pipeline.rs deleted file mode 100644 index 09eda47615..0000000000 --- a/e2e/rust/tests/proxy_egress_pipeline.rs +++ /dev/null @@ -1,1683 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E coverage for the transparent sandbox egress pipeline. -//! -//! Workloads connect directly to their requested destinations. Seccomp -//! notification diverts those sockets to the supervisor without proxy -//! environment variables or explicit CONNECT requests. These tests cover: -//! - live policy reloads affect new requests and close a pre-existing HTTP -//! stream before its next request is forwarded; -//! - `tls: skip` selects a byte-transparent TCP relay; -//! - provider placeholders in HTTP headers and opted-in REST bodies are -//! resolved without appearing in test output. - -use std::io::{self, Error, ErrorKind, Write}; -use std::process::Stdio; -use std::sync::{ - Arc, Mutex, - atomic::{AtomicUsize, Ordering}, -}; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::container::{SupportContainer, e2e_network_name}; -use openshell_e2e::harness::sandbox::SandboxGuard; -use serde_json::Value; -use serial_test::serial; -use tempfile::{Builder as TempFileBuilder, NamedTempFile}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinHandle; - -const TEST_SERVER_HOST: &str = "host.openshell.internal"; -const PROVIDER_NAME: &str = "e2e-proxy-egress-credentials"; -const PROVIDER_PROFILE_ID: &str = "e2e-proxy-egress-credentials"; -const TOKEN_ENV: &str = "PROXY_E2E_TOKEN"; -const TEST_SECRET: &str = "sk-e2e-proxy-egress-secret"; -const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -const PRIVATE_ALLOWED_IPS: &str = r#" allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7""#; -static PROVIDER_LOCK: Mutex<()> = Mutex::new(()); - -async fn run_cli(args: &[&str]) -> Result { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .map_err(|error| format!("failed to spawn openshell {}: {error}", args.join(" ")))?; - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - let combined = format!("{stdout}{stderr}"); - - if !output.status.success() { - return Err(format!( - "openshell {} failed (exit {:?}):\n{combined}", - args.join(" "), - output.status.code() - )); - } - - Ok(combined) -} - -async fn wait_for_sandbox_logs( - sandbox_name: &str, - expected: impl Fn(&str) -> bool, -) -> Result { - let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(10); - - loop { - let logs = run_cli(&[ - "logs", - sandbox_name, - "-n", - "500", - "--since", - "2m", - "--source", - "sandbox", - ]) - .await?; - if expected(&logs) { - return Ok(logs); - } - if tokio::time::Instant::now() >= deadline { - return Err(format!( - "timed out waiting for expected sandbox logs:\n{logs}" - )); - } - tokio::time::sleep(std::time::Duration::from_millis(250)).await; - } -} - -async fn delete_provider(name: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["provider", "delete", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -async fn delete_provider_profile(id: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["profile", "delete", id]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -async fn create_bound_provider(name: &str) -> Result { - let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); - run_cli(&[ - "provider", - "create", - "--name", - name, - "--type", - PROVIDER_PROFILE_ID, - "--credential", - &credential, - ]) - .await -} - -fn write_credential_profile(port: u16) -> Result { - let mut file = TempFileBuilder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| format!("create provider profile: {error}"))?; - let profile = format!( - r#"id: {PROVIDER_PROFILE_ID} -display_name: E2E proxy egress credentials -category: other -credentials: - - name: proxy_e2e_token - env_vars: [{TOKEN_ENV}] - required: true - auth_style: bearer - header_name: authorization -endpoints: - - host: {TEST_SERVER_HOST} - port: {port} - path: /probe - protocol: rest - access: full - enforcement: enforce - request_body_credential_rewrite: true - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" -binaries: ["/**"] -"# - ); - file.write_all(profile.as_bytes()) - .map_err(|error| format!("write provider profile: {error}"))?; - file.flush() - .map_err(|error| format!("flush provider profile: {error}"))?; - Ok(file) -} - -fn write_policy_document( - host: &str, - port: u16, - endpoint_options: &str, - network_middlewares: &str, -) -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -{network_middlewares} -network_policies: - proxy_egress_test: - name: proxy_egress_test - endpoints: - - host: {host} - port: {port} -{endpoint_options} -{PRIVATE_ALLOWED_IPS} - binaries: - - path: "/**" -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -fn write_policy(host: &str, port: u16, endpoint_options: &str) -> Result { - write_policy_document(host, port, endpoint_options, "") -} - -fn write_middleware_policy( - host: &str, - port: u16, - endpoint_options: &str, - on_error: &str, -) -> Result { - let network_middlewares = format!( - r#"network_middlewares: - regex-redactor: - name: Redact API tokens - middleware: openshell/regex - order: 10 - config: - mode: redact - on_error: {on_error} - endpoints: - include: ["{host}"] - exclude: [] -"# - ); - write_policy_document(host, port, endpoint_options, &network_middlewares) -} - -fn write_denied_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let policy = r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: {} -"#; - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -fn write_ambiguous_policy(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - terminating: - name: terminating - endpoints: - - host: {host} - port: {port} -{PRIVATE_ALLOWED_IPS} - binaries: - - path: "/**" - passthrough: - name: passthrough - endpoints: - - host: {host} - port: {port} - tls: skip -{PRIVATE_ALLOWED_IPS} - binaries: - - path: "/**" -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -fn write_destination_denial_policy() -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let policy = r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - destination_denials: - name: destination_denials - endpoints: - - { host: 169.254.169.254, port: 80 } - - { host: 127.0.0.1, port: 80 } - - { host: 203.0.113.10, port: 6443 } - - host: 203.0.113.10 - port: 8080 - allowed_ips: ["198.51.100.0/24"] - binaries: - - path: "/**" -"#; - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -fn write_ip_literal_success_policy( - explicit_ip: &str, - implicit_ip: &str, - port: u16, -) -> Result { - let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - destination_successes: - name: destination_successes - endpoints: - - host: {ip} - port: {port} - allowed_ips: ["{explicit_ip}/32"] - - host: {implicit_ip} - port: {port} - binaries: - - path: "/**" -"#, - ip = explicit_ip, - ); - file.write_all(policy.as_bytes()) - .map_err(|error| format!("write policy: {error}"))?; - file.flush() - .map_err(|error| format!("flush policy: {error}"))?; - Ok(file) -} - -fn policy_path(file: &NamedTempFile) -> String { - file.path() - .to_str() - .expect("temporary policy path should be utf-8") - .to_string() -} - -async fn read_until(stream: &mut TcpStream, marker: &[u8]) -> io::Result> { - let mut data = Vec::new(); - let mut buffer = [0_u8; 4096]; - loop { - let read = stream.read(&mut buffer).await?; - if read == 0 { - return Ok(data); - } - data.extend_from_slice(&buffer[..read]); - if data.windows(marker.len()).any(|window| window == marker) { - return Ok(data); - } - } -} - -fn header_end(bytes: &[u8]) -> Option { - bytes - .windows(4) - .position(|window| window == b"\r\n\r\n") - .map(|position| position + 4) -} - -fn content_length(headers: &[u8]) -> io::Result { - let text = - std::str::from_utf8(headers).map_err(|error| Error::new(ErrorKind::InvalidData, error))?; - Ok(text - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.trim() - .eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0)) -} - -async fn read_http_request(stream: &mut TcpStream) -> io::Result>> { - let mut request = read_until(stream, b"\r\n\r\n").await?; - if request.is_empty() { - return Ok(None); - } - let headers_end = header_end(&request) - .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; - let body_length = content_length(&request[..headers_end])?; - let total_length = headers_end + body_length; - while request.len() < total_length { - let mut buffer = vec![0_u8; total_length - request.len()]; - let read = stream.read(&mut buffer).await?; - if read == 0 { - return Err(Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP body")); - } - request.extend_from_slice(&buffer[..read]); - } - request.truncate(total_length); - Ok(Some(request)) -} - -struct KeepAliveHttpServer { - port: u16, - connections: Arc, - task: JoinHandle<()>, -} - -impl KeepAliveHttpServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind HTTP server: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read HTTP server address: {error}"))? - .port(); - let connections = Arc::new(AtomicUsize::new(0)); - let task_connections = Arc::clone(&connections); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - task_connections.fetch_add(1, Ordering::AcqRel); - tokio::spawn(async move { - let _ = handle_keep_alive_connection(stream).await; - }); - } - }); - Ok(Self { - port, - connections, - task, - }) - } - - fn connection_count(&self) -> usize { - self.connections.load(Ordering::Acquire) - } -} - -impl Drop for KeepAliveHttpServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn handle_keep_alive_connection(mut stream: TcpStream) -> io::Result<()> { - while let Some(request) = read_http_request(&mut stream).await? { - let close = String::from_utf8_lossy(&request) - .lines() - .any(|line| line.eq_ignore_ascii_case("connection: close")); - let connection = if close { "close" } else { "keep-alive" }; - let response = - format!("HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: {connection}\r\n\r\nok"); - stream.write_all(response.as_bytes()).await?; - if close { - return Ok(()); - } - } - Ok(()) -} - -struct EchoServer { - port: u16, - observed: Arc>>, - task: JoinHandle<()>, -} - -struct RequestBodyEchoServer { - port: u16, - task: JoinHandle<()>, -} - -impl RequestBodyEchoServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind request body echo server: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read request body echo server address: {error}"))? - .port(); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(async move { - let _ = handle_request_body_echo(stream).await; - }); - } - }); - Ok(Self { port, task }) - } -} - -impl Drop for RequestBodyEchoServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn handle_request_body_echo(mut stream: TcpStream) -> io::Result<()> { - let request = read_http_request(&mut stream) - .await? - .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "missing HTTP request"))?; - let headers_end = header_end(&request) - .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; - let body = &request[headers_end..]; - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ); - stream.write_all(response.as_bytes()).await?; - stream.write_all(body).await -} - -struct PipelineProbeServer { - port: u16, - observed: Arc>>, - task: JoinHandle<()>, -} - -impl PipelineProbeServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind pipeline probe: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read pipeline probe address: {error}"))? - .port(); - let observed = Arc::new(Mutex::new(Vec::new())); - let observed_task = observed.clone(); - let task = tokio::spawn(async move { - while let Ok((mut stream, _)) = listener.accept().await { - let observed = observed_task.clone(); - tokio::spawn(async move { - let mut request = Vec::new(); - let mut buffer = [0_u8; 4096]; - loop { - match tokio::time::timeout( - std::time::Duration::from_millis(200), - stream.read(&mut buffer), - ) - .await - { - Ok(Ok(0)) | Err(_) => break, - Ok(Ok(read)) => request.extend_from_slice(&buffer[..read]), - Ok(Err(_)) => return, - } - } - observed.lock().unwrap().extend_from_slice(&request); - // Keep the first response reusable so the queued request reaches policy evaluation. - let _ = stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok") - .await; - }); - } - }); - Ok(Self { - port, - observed, - task, - }) - } - - fn observed_request(&self) -> Vec { - self.observed.lock().unwrap().clone() - } -} - -impl Drop for PipelineProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -impl EchoServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind echo server: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read echo server address: {error}"))? - .port(); - let observed = Arc::new(Mutex::new(Vec::new())); - let task_observed = Arc::clone(&observed); - let task = tokio::spawn(async move { - while let Ok((mut stream, _)) = listener.accept().await { - let observed = Arc::clone(&task_observed); - tokio::spawn(async move { - let mut buffer = [0_u8; 4096]; - loop { - let Ok(read) = stream.read(&mut buffer).await else { - break; - }; - if read == 0 { - break; - } - observed.lock().unwrap().extend_from_slice(&buffer[..read]); - if stream.write_all(&buffer[..read]).await.is_err() { - break; - } - } - }); - } - }); - Ok(Self { - port, - observed, - task, - }) - } - - fn observed_bytes(&self) -> Vec { - self.observed.lock().unwrap().clone() - } -} - -impl Drop for EchoServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -struct CredentialProbeServer { - port: u16, - task: JoinHandle<()>, -} - -impl CredentialProbeServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|error| format!("bind credential probe: {error}"))?; - let port = listener - .local_addr() - .map_err(|error| format!("read credential probe address: {error}"))? - .port(); - let task = tokio::spawn(async move { - while let Ok((stream, _)) = listener.accept().await { - tokio::spawn(async move { - let _ = handle_credential_probe(stream).await; - }); - } - }); - Ok(Self { port, task }) - } -} - -impl Drop for CredentialProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn handle_credential_probe(mut stream: TcpStream) -> io::Result<()> { - let request = read_http_request(&mut stream) - .await? - .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "missing HTTP request"))?; - let headers_end = header_end(&request) - .ok_or_else(|| Error::new(ErrorKind::UnexpectedEof, "incomplete HTTP headers"))?; - let headers = String::from_utf8_lossy(&request[..headers_end]); - let expected_authorization = format!("Bearer {TEST_SECRET}"); - let header_resolved = headers.lines().any(|line| { - line.split_once(':').is_some_and(|(name, value)| { - name.eq_ignore_ascii_case("authorization") && value.trim() == expected_authorization - }) - }); - let body_resolved = request[headers_end..] - .windows(TEST_SECRET.len()) - .any(|window| window == TEST_SECRET.as_bytes()); - let saw_placeholder = request - .windows(PLACEHOLDER_PREFIX.len()) - .any(|window| window == PLACEHOLDER_PREFIX.as_bytes()); - let body = serde_json::json!({ - "body_resolved": body_resolved, - "header_resolved": header_resolved, - "saw_placeholder": saw_placeholder, - }) - .to_string(); - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - stream.write_all(response.as_bytes()).await -} - -fn transparent_status_script(host: &str, port: u16) -> String { - format!( - r#" -import json -import socket - -HOST = {host:?} -PORT = {port} - -def read_headers(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def status(response): - parts = response.split(None, 2) - return int(parts[1]) if len(parts) > 1 else 0 - -def request_status(path): - target = f"{{HOST}}:{{PORT}}" - try: - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall( - f"GET {{path}} HTTP/1.1\r\n" - f"Host: {{target}}\r\nConnection: close\r\n\r\n".encode() - ) - return status(read_headers(sock)) - except OSError as error: - return {{"errno": error.errno, "error": repr(error)}} - -print(json.dumps({{ - "first": request_status("/first"), - "second": request_status("/second"), -}}, sort_keys=True)) -"#, - host = host, - port = port, - ) -} - -fn persistent_transparent_script(host: &str, port: u16) -> String { - format!( - r#" -import json -import os -import socket -import time - -HOST = {host:?} -PORT = {port} -READY = "/tmp/proxy-reload-ready" -GO = "/tmp/proxy-reload-go" -RESULT = "/tmp/proxy-reload-result" - -def read_response(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - return 0 - data += chunk - headers, body = data.split(b"\r\n\r\n", 1) - length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) - while len(body) < length: - chunk = sock.recv(4096) - if not chunk: - return 0 - body += chunk - return int(headers.split(None, 2)[1]) - -target = f"{{HOST}}:{{PORT}}" -failed_closed = False -second_status = 0 -try: - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall( - f"GET /before-reload HTTP/1.1\r\nHost: {{target}}\r\nConnection: keep-alive\r\n\r\n".encode() - ) - if read_response(sock) != 200: - raise RuntimeError("initial tunneled request was denied") - open(READY, "w").close() - deadline = time.monotonic() + 120 - while not os.path.exists(GO) and time.monotonic() < deadline: - time.sleep(0.1) - if not os.path.exists(GO): - raise RuntimeError("timed out waiting for policy reload signal") - try: - sock.sendall( - f"GET /after-reload HTTP/1.1\r\nHost: {{target}}\r\nConnection: close\r\n\r\n".encode() - ) - second_status = read_response(sock) - except OSError: - second_status = 0 - failed_closed = second_status != 200 -finally: - with open(RESULT, "w") as result: - json.dump({{"failed_closed": failed_closed, "second_status": second_status}}, result, sort_keys=True) -"#, - host = host, - port = port, - ) -} - -async fn wait_for_sandbox_file(guard: &SandboxGuard, path: &str, log_path: &str) -> String { - let script = format!( - r#"import os, time -deadline = time.monotonic() + 60 -while not os.path.exists({path:?}) and time.monotonic() < deadline: - time.sleep(0.1) -if not os.path.exists({path:?}): - if os.path.exists({log_path:?}): - print(open({log_path:?}).read()) - raise SystemExit("timed out waiting for {path}") -print(open({path:?}).read()) -"# - ); - guard - .exec(&["python3", "-c", &script]) - .await - .unwrap_or_else(|error| panic!("wait for sandbox file {path}: {error}")) -} - -fn parse_json_line(output: &str) -> Value { - output - .lines() - .filter_map(|line| serde_json::from_str::(line.trim()).ok()) - .next_back() - .unwrap_or_else(|| panic!("missing JSON result in sandbox output:\n{output}")) -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn policy_reload_updates_transparent_requests_and_closes_existing_http_stream() { - let server = KeepAliveHttpServer::start() - .await - .expect("start keep-alive HTTP server"); - let policy_a = write_policy(TEST_SERVER_HOST, server.port, "").expect("write policy A"); - let policy_b = write_denied_policy().expect("write policy B"); - let policy_a_path = policy_path(&policy_a); - let policy_b_path = policy_path(&policy_b); - - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &policy_a_path], - &["sh", "-c", "echo Ready; sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox"); - - run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &policy_a_path, - "--wait", - "--timeout", - "120", - ]) - .await - .expect("wait for policy A"); - - let persistent_script = persistent_transparent_script(TEST_SERVER_HOST, server.port); - guard - .exec(&[ - "sh", - "-c", - "nohup python3 -c \"$1\" >/tmp/proxy-reload-client.log 2>&1 &", - "proxy-reload-client", - &persistent_script, - ]) - .await - .expect("start persistent transparent client"); - wait_for_sandbox_file( - &guard, - "/tmp/proxy-reload-ready", - "/tmp/proxy-reload-client.log", - ) - .await; - - let status_script = transparent_status_script(TEST_SERVER_HOST, server.port); - let before = guard - .exec(&["python3", "-c", &status_script]) - .await - .expect("exercise transparent requests before reload"); - let before = parse_json_line(&before); - assert_eq!( - before["first"], 200, - "first request before reload: {before}" - ); - assert_eq!( - before["second"], 200, - "second request before reload: {before}" - ); - - run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &policy_b_path, - "--wait", - "--timeout", - "120", - ]) - .await - .expect("publish and wait for policy B"); - - guard - .exec(&["sh", "-c", "touch /tmp/proxy-reload-go"]) - .await - .expect("release persistent transparent client"); - let stale_tunnel = wait_for_sandbox_file( - &guard, - "/tmp/proxy-reload-result", - "/tmp/proxy-reload-client.log", - ) - .await; - let stale_tunnel = parse_json_line(&stale_tunnel); - assert_eq!( - stale_tunnel["failed_closed"], true, - "existing transparent HTTP stream forwarded after policy reload: {stale_tunnel}" - ); - - let after = guard - .exec(&["python3", "-c", &status_script]) - .await - .expect("exercise transparent requests after reload"); - let after = parse_json_line(&after); - assert_ne!(after["first"], 200, "first request after reload: {after}"); - assert_ne!(after["second"], 200, "second request after reload: {after}"); - - guard.cleanup().await; -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn ambiguous_policy_update_is_rejected_without_replacing_active_policy() { - let server = KeepAliveHttpServer::start() - .await - .expect("start keep-alive HTTP server"); - let valid_policy = write_policy(TEST_SERVER_HOST, server.port, "").expect("write valid policy"); - let ambiguous_policy = - write_ambiguous_policy(TEST_SERVER_HOST, server.port).expect("write ambiguous policy"); - let valid_policy_path = policy_path(&valid_policy); - let ambiguous_policy_path = policy_path(&ambiguous_policy); - - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &valid_policy_path], - &["sh", "-c", "echo Ready; sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox"); - - run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &valid_policy_path, - "--wait", - "--timeout", - "120", - ]) - .await - .expect("wait for valid policy"); - - let status_script = transparent_status_script(TEST_SERVER_HOST, server.port); - let before = guard - .exec(&["python3", "-c", &status_script]) - .await - .expect("exercise transparent requests before invalid update"); - let before = parse_json_line(&before); - assert_eq!( - before["first"], 200, - "first request before update: {before}" - ); - assert_eq!( - before["second"], 200, - "second request before update: {before}" - ); - let history_before = run_cli(&["policy", "list", &guard.name]) - .await - .expect("list policy history before rejected update"); - let connections_before_rejection = server.connection_count(); - - let update_error = run_cli(&[ - "policy", - "set", - &guard.name, - "--policy", - &ambiguous_policy_path, - "--wait", - "--timeout", - "120", - ]) - .await - .expect_err("ambiguous policy must be rejected before persistence"); - assert!( - update_error.contains("ambiguity validation failed"), - "policy update should explain the ambiguity:\n{update_error}" - ); - - let history_after = run_cli(&["policy", "list", &guard.name]) - .await - .expect("list policy history after rejected update"); - assert_eq!( - history_after, history_before, - "rejected policy must not create a revision" - ); - - let after_rejection = guard - .exec(&["python3", "-c", &status_script]) - .await - .expect("exercise transparent requests after rejected update"); - let after_rejection = parse_json_line(&after_rejection); - assert_eq!( - after_rejection["first"], 200, - "first request should keep using the active valid policy: {after_rejection}" - ); - assert_eq!( - after_rejection["second"], 200, - "second request should keep using the active valid policy: {after_rejection}" - ); - assert!( - server.connection_count() > connections_before_rejection, - "active-policy requests should still contact the upstream server" - ); - - guard.cleanup().await; -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn transparent_destination_denials_fail_connect_with_eacces() { - let policy = write_destination_denial_policy().expect("write destination denial policy"); - let policy_path = policy_path(&policy); - let script = r#" -import json -import socket - -targets = { - "metadata": ("169.254.169.254", 80), - "control_plane": ("203.0.113.10", 6443), - "outside_allowed_ips": ("203.0.113.10", 8080), -} -result = {} -for name, target in targets.items(): - try: - with socket.create_connection(target, timeout=10): - result[name] = 0 - except OSError as error: - result[name] = error.errno -print(json.dumps(result, sort_keys=True)) -"#; - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", script]) - .await - .expect("sandbox create"); - let result = parse_json_line(&guard.create_output); - for name in ["metadata", "control_plane", "outside_allowed_ips"] { - assert_eq!(result[name], 13, "{name} should fail with EACCES: {result}"); - } -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn explicit_allowed_ips_and_implicit_ip_literals_succeed_transparently() { - if e2e_network_name().is_none() { - eprintln!("skipping IP-literal success assertions without a shared container network"); - return; - } - - const HTTP_SERVER: &str = r#" -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - body = b"ok" - self.send_response(200) - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format, *args): - pass - -ThreadingHTTPServer(("0.0.0.0", 8000), Handler).serve_forever() -"#; - let explicit_server = - SupportContainer::start_python("explicit-ip.openshell.test", HTTP_SERVER, 8000) - .await - .expect("start explicit allowed_ips support container"); - let implicit_server = - SupportContainer::start_python("implicit-ip.openshell.test", HTTP_SERVER, 8000) - .await - .expect("start implicit IP-literal support container"); - let explicit_ip = explicit_server.ip().expect("explicit support container IP"); - let implicit_ip = implicit_server.ip().expect("implicit support container IP"); - let policy = write_ip_literal_success_policy(&explicit_ip, &implicit_ip, 8000) - .expect("write IP literal policy"); - let policy_path = policy_path(&policy); - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &policy_path], - &["sh", "-c", "echo Ready; sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox"); - - for (mode, destination) in [ - ("explicit_allowed_ips", explicit_ip.as_str()), - ("implicit_ip_literal", implicit_ip.as_str()), - ] { - let output = guard - .exec(&[ - "python3", - "-c", - &transparent_status_script(destination, 8000), - ]) - .await - .unwrap_or_else(|error| panic!("exercise {mode}: {error}")); - let statuses = parse_json_line(&output); - assert_eq!(statuses["first"], 200, "{mode} first request: {statuses}"); - assert_eq!(statuses["second"], 200, "{mode} second request: {statuses}"); - } - - guard.cleanup().await; -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn tls_skip_connect_relays_opaque_bytes_bidirectionally() { - let server = EchoServer::start().await.expect("start TCP echo server"); - let policy = write_policy(TEST_SERVER_HOST, server.port, " tls: skip") - .expect("write tls: skip policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import socket - -HOST = {host:?} -PORT = {port} -PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80, 0x0a]) + b"not-http-or-tls" + bytes(range(64)) - -with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(PAYLOAD) - echoed = b"" - while len(echoed) < len(PAYLOAD): - chunk = sock.recv(len(PAYLOAD) - len(echoed)) - if not chunk: - break - echoed += chunk - if echoed != PAYLOAD: - raise RuntimeError("opaque payload changed in transit") -print("RAW_RELAY_OK") -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - assert!( - guard.create_output.contains("RAW_RELAY_OK"), - "raw relay did not preserve the opaque payload:\n{}", - guard.create_output - ); -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn middleware_redacts_transparent_request_bodies() { - let server = RequestBodyEchoServer::start() - .await - .expect("start request body echo server"); - let policy = write_middleware_policy(TEST_SERVER_HOST, server.port, "", "fail_closed") - .expect("write middleware policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import json -import socket - -HOST = {host:?} -PORT = {port} -SECRET = "sk-1234567890abcdef" - -def read_response(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - raise RuntimeError("incomplete response headers") - data += chunk - headers, body = data.split(b"\r\n\r\n", 1) - length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) - while len(body) < length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - status = int(headers.split(None, 2)[1]) - if status != 200: - raise RuntimeError(f"request failed with HTTP {{status}}: {{body!r}}") - return json.loads(body[:length]) - -def request_bytes(target): - body = json.dumps({{"api_key": SECRET}}, separators=(",", ":")).encode() - return ( - f"POST {{target}} HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - "Content-Type: application/json\r\n" - f"Content-Length: {{len(body)}}\r\n" - "Connection: close\r\n\r\n" - ).encode() + body - -def request_once(): - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(request_bytes("/middleware")) - return read_response(sock) - -print(json.dumps({{"first": request_once(), "second": request_once()}}, sort_keys=True)) -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - let result = parse_json_line(&guard.create_output); - for request in ["first", "second"] { - assert_eq!( - result[request]["api_key"], "[REDACTED]", - "{request} did not deliver the middleware-transformed body: {result}" - ); - } -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn fail_closed_middleware_blocks_uninspectable_transparent_payload_before_upstream() { - let server = EchoServer::start().await.expect("start TCP echo server"); - let policy = write_middleware_policy(TEST_SERVER_HOST, server.port, "", "fail_closed") - .expect("write fail-closed middleware policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import socket - -HOST = {host:?} -PORT = {port} -PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37]) + b"not-http-or-tls" - -with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(PAYLOAD) - denial = b"" - while True: - try: - chunk = sock.recv(4096) - except ConnectionResetError: - break - if not chunk: - break - denial += chunk - if denial and ( - b"HTTP/1.1 403 Forbidden" not in denial - or b"unsupported_l7_protocol" not in denial - ): - raise RuntimeError(f"missing fail-closed middleware denial: {{denial!r}}") -print("UNINSPECTABLE_MIDDLEWARE_BLOCKED") -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let mut guard = SandboxGuard::create_keep_with_args( - &["--policy", &policy_path], - &["sh", "-c", "echo Ready; sleep infinity"], - "Ready", - ) - .await - .expect("create keep sandbox"); - let output = guard - .exec(&["python3", "-c", &script]) - .await - .expect("exercise uninspectable fail-closed middleware"); - assert!( - output.contains("UNINSPECTABLE_MIDDLEWARE_BLOCKED"), - "uninspectable payload was not blocked:\n{output}" - ); - tokio::time::sleep(std::time::Duration::from_millis(200)).await; - assert!( - server.observed_bytes().is_empty(), - "uninspectable payload reached upstream before middleware denial" - ); - - let logs = wait_for_sandbox_logs(&guard.name, |logs| { - logs.contains("openshell.middleware.traffic_uninspectable") - && logs - .contains("Unsupported tunnel protocol cannot be inspected by required middleware") - }) - .await - .expect("fetch sandbox logs after middleware denial"); - assert!( - logs.contains("openshell.middleware.traffic_uninspectable") - && logs - .contains("Unsupported tunnel protocol cannot be inspected by required middleware"), - "OCSF logs should explain the fail-closed denial:\n{logs}" - ); - - guard.cleanup().await; -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn fail_open_middleware_bypasses_uninspectable_transparent_tls_skip() { - let server = EchoServer::start().await.expect("start TCP echo server"); - let policy = write_middleware_policy( - TEST_SERVER_HOST, - server.port, - " tls: skip", - "fail_open", - ) - .expect("write fail-open middleware policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import socket - -HOST = {host:?} -PORT = {port} -PAYLOAD = bytes([0x00, 0xff, 0x13, 0x37, 0x80]) + b"middleware-bypass" - -with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(PAYLOAD) - echoed = b"" - while len(echoed) < len(PAYLOAD): - chunk = sock.recv(len(PAYLOAD) - len(echoed)) - if not chunk: - break - echoed += chunk - if echoed != PAYLOAD: - raise RuntimeError(f"fail-open middleware did not preserve raw relay: {{echoed!r}}") -print("UNINSPECTABLE_MIDDLEWARE_BYPASSED") -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - assert!( - guard - .create_output - .contains("UNINSPECTABLE_MIDDLEWARE_BYPASSED"), - "fail-open middleware did not bypass uninspectable traffic:\n{}", - guard.create_output - ); - assert_eq!( - server.observed_bytes(), - [0x00, 0xff, 0x13, 0x37, 0x80] - .into_iter() - .chain(*b"middleware-bypass") - .collect::>(), - "upstream did not receive the unchanged fail-open payload" - ); -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn transparent_pipeline_never_reaches_upstream_as_first_request_overflow() { - let server = PipelineProbeServer::start() - .await - .expect("start pipeline probe server"); - let endpoint_options = r#" protocol: rest - enforcement: enforce - rules: - - allow: - method: GET - path: "/allowed""#; - let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options) - .expect("write pipeline policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import socket - -target = "{host}:{port}" -first = ( - f"GET /allowed HTTP/1.1\r\n" - f"Host: {{target}}\r\nConnection: keep-alive\r\n\r\n" -) -second = ( - f"POST /blocked HTTP/1.1\r\n" - f"Host: {{target}}\r\nContent-Length: 0\r\n\r\n" -) -with socket.create_connection(({host:?}, {port}), timeout=10) as sock: - sock.sendall((first + second).encode()) - response = b"" - while True: - chunk = sock.recv(4096) - if not chunk: - break - response += chunk -responses = response.count(b"HTTP/1.1 ") -first_status = response.split(b"\r\n", 1)[0] -if responses != 2 or b" 200 " not in first_status or b"HTTP/1.1 403 Forbidden" not in response: - raise RuntimeError(f"unexpected pipelined response: {{response!r}}") -print("TRANSPARENT_PIPELINE_DENIED") -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - assert!( - guard.create_output.contains("TRANSPARENT_PIPELINE_DENIED"), - "transparent HTTP stream did not deny the disallowed pipelined request:\n{}", - guard.create_output - ); - - let observed = String::from_utf8(server.observed_request()).expect("upstream HTTP request"); - assert!(observed.starts_with("GET /allowed HTTP/1.1\r\n")); - assert!( - !observed.to_ascii_lowercase().contains("\r\nconnection:"), - "shared relay must remove hop-by-hop connection headers:\n{observed}" - ); - assert!(!observed.contains("/blocked")); -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn chunked_pipeline_is_authorized_separately_before_reaching_upstream() { - let server = PipelineProbeServer::start() - .await - .expect("start pipeline probe server"); - let endpoint_options = r#" protocol: rest - enforcement: enforce - rules: - - allow: - method: POST - path: "/allowed""#; - let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options) - .expect("write pipeline policy"); - let policy_path = policy_path(&policy); - let script = format!( - r#" -import socket - -target = "{host}:{port}" -first = ( - f"POST /allowed HTTP/1.1\r\n" - f"Host: {{target}}\r\nTransfer-Encoding: chunked\r\nConnection: keep-alive\r\n\r\n" - f"0\r\n\r\n" -) -second = ( - f"DELETE /blocked HTTP/1.1\r\n" - f"Host: {{target}}\r\nContent-Length: 0\r\n\r\n" -) -with socket.create_connection(({host:?}, {port}), timeout=10) as sock: - sock.sendall((first + second).encode()) - response = b"" - while True: - chunk = sock.recv(4096) - if not chunk: - break - response += chunk -responses = response.count(b"HTTP/1.1 ") -first_status = response.split(b"\r\n", 1)[0] -if responses != 2 or b" 200 " not in first_status or b"HTTP/1.1 403 Forbidden" not in response: - raise RuntimeError(f"unexpected chunked pipeline response: {{response!r}}") -print("CHUNKED_PIPELINE_DENIED") -"#, - host = TEST_SERVER_HOST, - port = server.port, - ); - - let guard = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script]) - .await - .expect("sandbox create"); - assert!( - guard.create_output.contains("CHUNKED_PIPELINE_DENIED"), - "transparent HTTP stream did not evaluate the chunked pipeline separately:\n{}", - guard.create_output - ); - - let observed = String::from_utf8(server.observed_request()).expect("upstream HTTP request"); - assert!(observed.starts_with("POST /allowed HTTP/1.1\r\n")); - assert!(observed.ends_with("0\r\n\r\n")); - assert!(!observed.contains("/blocked")); -} - -#[tokio::test] -#[serial(proxy_egress_pipeline)] -async fn http_credentials_are_rewritten_in_transparent_headers_and_bodies() { - let _provider_lock = PROVIDER_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - delete_provider(PROVIDER_NAME).await; - delete_provider_profile(PROVIDER_PROFILE_ID).await; - - let result = async { - let server = CredentialProbeServer::start().await?; - let profile = write_credential_profile(server.port)?; - let profile_path = profile.path().to_string_lossy().into_owned(); - run_cli(&["profile", "import", "--file", &profile_path]).await?; - create_bound_provider(PROVIDER_NAME).await?; - let endpoint_options = r#" path: /probe - protocol: rest - enforcement: enforce - request_body_credential_rewrite: true - access: full"#; - let policy = write_policy(TEST_SERVER_HOST, server.port, endpoint_options)?; - let policy_path = policy_path(&policy); - let script = format!( - r#" -import json -import os -import socket - -HOST = {host:?} -PORT = {port} -TOKEN = os.environ[{token_env:?}] - -def read_response(sock): - data = b"" - while b"\r\n\r\n" not in data: - chunk = sock.recv(4096) - if not chunk: - raise RuntimeError("incomplete response headers") - data += chunk - headers, body = data.split(b"\r\n\r\n", 1) - length = 0 - for line in headers.split(b"\r\n")[1:]: - if line.lower().startswith(b"content-length:"): - length = int(line.split(b":", 1)[1].strip()) - while len(body) < length: - chunk = sock.recv(4096) - if not chunk: - break - body += chunk - code = int(headers.split(None, 2)[1]) - if code != 200: - raise RuntimeError(f"request failed with HTTP {{code}}") - return json.loads(body[:length]) - -def request_bytes(target): - body = json.dumps({{"credential": TOKEN}}, separators=(",", ":")).encode() - return ( - f"POST {{target}} HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - f"Authorization: Bearer {{TOKEN}}\r\n" - "Content-Type: application/json\r\n" - f"Content-Length: {{len(body)}}\r\n" - "Connection: close\r\n\r\n" - ).encode() + body - -def request_once(): - with socket.create_connection((HOST, PORT), timeout=10) as sock: - sock.sendall(request_bytes("/probe")) - return read_response(sock) - -print(json.dumps({{"first": request_once(), "second": request_once()}}, sort_keys=True)) -"#, - host = TEST_SERVER_HOST, - port = server.port, - token_env = TOKEN_ENV, - ); - - SandboxGuard::create(&[ - "--policy", - &policy_path, - "--provider", - PROVIDER_NAME, - "--", - "python3", - "-c", - &script, - ]) - .await - } - .await; - - delete_provider(PROVIDER_NAME).await; - delete_provider_profile(PROVIDER_PROFILE_ID).await; - - let guard = result.expect("sandbox create"); - let result = parse_json_line(&guard.create_output); - for request in ["first", "second"] { - assert_eq!( - result[request]["header_resolved"], true, - "{request} header placeholder was not resolved: {result}" - ); - assert_eq!( - result[request]["body_resolved"], true, - "{request} body placeholder was not resolved: {result}" - ); - assert_eq!( - result[request]["saw_placeholder"], false, - "{request} leaked an unresolved placeholder upstream: {result}" - ); - } - assert!( - !guard.create_output.contains(TEST_SECRET), - "sandbox output exposed the raw provider credential:\n{}", - guard.create_output - ); - assert!( - !guard.create_output.contains(PLACEHOLDER_PREFIX), - "sandbox output exposed an unresolved provider placeholder:\n{}", - guard.create_output - ); -} diff --git a/e2e/rust/tests/websocket_conformance.rs b/e2e/rust/tests/websocket_conformance.rs deleted file mode 100644 index 1150e020bc..0000000000 --- a/e2e/rust/tests/websocket_conformance.rs +++ /dev/null @@ -1,517 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E regression: WebSocket credential placeholders are resolved on the -//! sandbox path after an RFC 6455 upgrade. -//! -//! The sandbox process sends its provider-managed placeholder in a masked text -//! frame. The local upstream only reports whether it saw the real secret and -//! whether any placeholder survived; it never echoes payload bytes, placeholder -//! text, or secret material back into test output. - -use std::io::{self, Error, ErrorKind, Write}; -use std::process::Stdio; -use std::sync::Mutex; - -use base64::Engine as _; -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::sandbox::SandboxGuard; -use sha1::{Digest, Sha1}; -use tempfile::{Builder as TempFileBuilder, NamedTempFile}; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tokio::net::{TcpListener, TcpStream}; -use tokio::task::JoinHandle; - -const WEBSOCKET_GUID: &str = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; -const PROVIDER_NAME: &str = "e2e-websocket-conformance"; -const PROVIDER_PROFILE_ID: &str = "e2e-websocket-conformance"; -const TEST_SERVER_HOST: &str = "host.openshell.internal"; -const TEST_SECRET: &str = "sk-e2e-websocket-conformance-secret"; -const TOKEN_ENV: &str = "WS_E2E_TOKEN"; -const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; -static PROVIDER_LOCK: Mutex<()> = Mutex::new(()); - -async fn run_cli(args: &[&str]) -> Result { - let mut cmd = openshell_cmd(); - cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .map_err(|e| format!("failed to spawn openshell {}: {e}", args.join(" ")))?; - - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - - if !output.status.success() { - return Err(format!( - "openshell {} failed (exit {:?}):\n{combined}", - args.join(" "), - output.status.code() - )); - } - - Ok(combined) -} - -async fn delete_provider(name: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("provider") - .arg("delete") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -async fn delete_provider_profile(id: &str) { - let mut cmd = openshell_cmd(); - cmd.args(["profile", "delete", id]) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -async fn create_bound_provider(name: &str) -> Result { - let credential = format!("{TOKEN_ENV}={TEST_SECRET}"); - run_cli(&[ - "provider", - "create", - "--name", - name, - "--type", - PROVIDER_PROFILE_ID, - "--credential", - &credential, - ]) - .await -} - -fn write_credential_profile(port: u16) -> Result { - let mut file = TempFileBuilder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| format!("create provider profile: {error}"))?; - let profile = format!( - r#"id: {PROVIDER_PROFILE_ID} -display_name: E2E WebSocket conformance credentials -category: other -credentials: - - name: websocket_token - env_vars: [{TOKEN_ENV}] - required: true - auth_style: bearer - header_name: authorization -endpoints: - - host: {TEST_SERVER_HOST} - port: {port} - path: /ws - protocol: websocket - enforcement: enforce - access: read-write - websocket_credential_rewrite: true - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" -binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(profile.as_bytes()) - .map_err(|error| format!("write provider profile: {error}"))?; - file.flush() - .map_err(|error| format!("flush provider profile: {error}"))?; - Ok(file) -} - -struct WebSocketProbeServer { - port: u16, - task: JoinHandle<()>, -} - -impl WebSocketProbeServer { - async fn start() -> Result { - let listener = TcpListener::bind(("0.0.0.0", 0)) - .await - .map_err(|e| format!("bind websocket probe server: {e}"))?; - let port = listener - .local_addr() - .map_err(|e| format!("read websocket probe server address: {e}"))? - .port(); - let task = tokio::spawn(async move { - loop { - let Ok((stream, _)) = listener.accept().await else { - break; - }; - tokio::spawn(async move { - let _ = handle_websocket_probe_connection(stream).await; - }); - } - }); - - Ok(Self { port, task }) - } -} - -impl Drop for WebSocketProbeServer { - fn drop(&mut self) { - self.task.abort(); - } -} - -async fn recv_until(stream: &mut TcpStream, marker: &[u8]) -> io::Result> { - let mut data = Vec::new(); - let mut buf = [0_u8; 1024]; - loop { - let read = stream.read(&mut buf).await?; - if read == 0 { - return Ok(data); - } - data.extend_from_slice(&buf[..read]); - if data.windows(marker.len()).any(|window| window == marker) { - return Ok(data); - } - } -} - -async fn read_websocket_text(stream: &mut TcpStream) -> io::Result { - let mut header = [0_u8; 2]; - stream.read_exact(&mut header).await?; - let length = match header[1] & 0x7F { - len @ 0..=125 => usize::from(len), - 126 => { - let mut bytes = [0_u8; 2]; - stream.read_exact(&mut bytes).await?; - usize::from(u16::from_be_bytes(bytes)) - } - 127 => { - let mut bytes = [0_u8; 8]; - stream.read_exact(&mut bytes).await?; - usize::try_from(u64::from_be_bytes(bytes)) - .map_err(|_| Error::new(ErrorKind::InvalidData, "websocket frame too large"))? - } - _ => unreachable!(), - }; - - let mut mask = [0_u8; 4]; - if header[1] & 0x80 != 0 { - stream.read_exact(&mut mask).await?; - } else { - mask = [0, 0, 0, 0]; - } - - let mut payload = vec![0_u8; length]; - stream.read_exact(&mut payload).await?; - if header[1] & 0x80 != 0 { - for (index, byte) in payload.iter_mut().enumerate() { - *byte ^= mask[index % mask.len()]; - } - } - - String::from_utf8(payload).map_err(|e| { - Error::new( - ErrorKind::InvalidData, - format!("invalid websocket text: {e}"), - ) - }) -} - -async fn send_websocket_text(stream: &mut TcpStream, payload: &str) -> io::Result<()> { - let data = payload.as_bytes(); - let mut frame = Vec::with_capacity(data.len() + 10); - frame.push(0x81); - if data.len() < 126 { - frame.push( - u8::try_from(data.len()) - .map_err(|_| Error::new(ErrorKind::InvalidData, "websocket frame too large"))?, - ); - } else if data.len() <= usize::from(u16::MAX) { - frame.push(126); - frame.extend_from_slice( - &u16::try_from(data.len()) - .map_err(|_| Error::new(ErrorKind::InvalidData, "websocket frame too large"))? - .to_be_bytes(), - ); - } else { - frame.push(127); - frame.extend_from_slice( - &u64::try_from(data.len()) - .map_err(|_| Error::new(ErrorKind::InvalidData, "websocket frame too large"))? - .to_be_bytes(), - ); - } - frame.extend_from_slice(data); - stream.write_all(&frame).await -} - -fn header_value(request: &str, name: &str) -> Option { - request.lines().find_map(|line| { - let (header, value) = line.split_once(':')?; - if header.trim().eq_ignore_ascii_case(name) { - Some(value.trim().to_string()) - } else { - None - } - }) -} - -fn websocket_accept_for_key(key: &str) -> String { - let mut hasher = Sha1::new(); - hasher.update(key.as_bytes()); - hasher.update(WEBSOCKET_GUID.as_bytes()); - base64::engine::general_purpose::STANDARD.encode(hasher.finalize()) -} - -async fn handle_websocket_probe_connection(mut stream: TcpStream) -> io::Result<()> { - let request_bytes = recv_until(&mut stream, b"\r\n\r\n").await?; - let request = String::from_utf8_lossy(&request_bytes); - if !request.to_ascii_lowercase().contains("upgrade: websocket") { - stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok") - .await?; - return Ok(()); - } - - let accept = header_value(&request, "Sec-WebSocket-Key") - .map(|key| websocket_accept_for_key(&key)) - .ok_or_else(|| Error::new(ErrorKind::InvalidData, "missing Sec-WebSocket-Key"))?; - let response = format!( - "HTTP/1.1 101 Switching Protocols\r\n\ - Upgrade: websocket\r\n\ - Connection: Upgrade\r\n\ - Sec-WebSocket-Accept: {accept}\r\n\ - \r\n" - ); - stream.write_all(response.as_bytes()).await?; - - let text = read_websocket_text(&mut stream).await?; - let response = format!( - r#"{{"saw_placeholder": {}, "saw_secret": {}}}"#, - text.contains(PLACEHOLDER_PREFIX), - text.contains(TEST_SECRET) - ); - send_websocket_text(&mut stream, &response).await -} - -fn write_websocket_policy(host: &str, port: u16) -> Result { - let mut file = NamedTempFile::new().map_err(|e| format!("create temp policy file: {e}"))?; - let policy = format!( - r#"version: 1 - -filesystem_policy: - include_workdir: true - read_only: - - /usr - - /lib - - /proc - - /dev/urandom - - /app - - /etc - - /var/log - read_write: - - /sandbox - - /tmp - - /dev/null - -landlock: - compatibility: best_effort - -process: - run_as_user: sandbox - run_as_group: sandbox - -network_policies: - websocket_conformance: - name: websocket_conformance - endpoints: - - host: {host} - port: {port} - protocol: websocket - enforcement: enforce - access: read-write - websocket_credential_rewrite: true - allowed_ips: - - "10.0.0.0/8" - - "172.0.0.0/8" - - "192.168.0.0/16" - - "fc00::/7" - binaries: - - path: /usr/bin/python* - - path: /usr/local/bin/python* - - path: /sandbox/.uv/python/*/bin/python* -"# - ); - file.write_all(policy.as_bytes()) - .map_err(|e| format!("write temp policy file: {e}"))?; - file.flush() - .map_err(|e| format!("flush temp policy file: {e}"))?; - Ok(file) -} - -fn websocket_client_script(host: &str, port: u16) -> String { - format!( - r#" -import base64 -import json -import os -import socket -import struct -import time - -HOST = {host:?} -PORT = {port} -TOKEN_ENV = {token_env:?} - -def recv_until(sock, marker): - data = b"" - while marker not in data: - chunk = sock.recv(4096) - if not chunk: - break - data += chunk - return data - -def read_exact(sock, size): - data = b"" - while len(data) < size: - chunk = sock.recv(size - len(data)) - if not chunk: - raise EOFError("unexpected end of websocket frame") - data += chunk - return data - -def masked_text_frame(payload): - data = payload.encode("utf-8") - mask = os.urandom(4) - if len(data) < 126: - header = bytes([0x81, 0x80 | len(data)]) - elif len(data) <= 0xFFFF: - header = bytes([0x81, 0x80 | 126]) + struct.pack("!H", len(data)) - else: - header = bytes([0x81, 0x80 | 127]) + struct.pack("!Q", len(data)) - masked = bytes(byte ^ mask[index % 4] for index, byte in enumerate(data)) - return header + mask + masked - -def read_frame(sock): - first, second = read_exact(sock, 2) - length = second & 0x7F - if length == 126: - length = struct.unpack("!H", read_exact(sock, 2))[0] - elif length == 127: - length = struct.unpack("!Q", read_exact(sock, 8))[0] - mask = read_exact(sock, 4) if second & 0x80 else b"" - payload = read_exact(sock, length) - if mask: - payload = bytes(byte ^ mask[index % 4] for index, byte in enumerate(payload)) - return first, payload - -def transparent_socket_with_retry(host, port, timeout_seconds=20): - deadline = time.monotonic() + timeout_seconds - last_error = None - while time.monotonic() < deadline: - sock = None - try: - sock = socket.create_connection((host, port), timeout=5) - return sock - except OSError as error: - if sock is not None: - sock.close() - last_error = error - time.sleep(0.25) - raise last_error - -token = os.environ[TOKEN_ENV] -payload = json.dumps({{"authorization": "Bearer " + token}}, sort_keys=True) -key = base64.b64encode(os.urandom(16)).decode("ascii") -with transparent_socket_with_retry(HOST, PORT) as sock: - request = ( - "GET /ws HTTP/1.1\r\n" - f"Host: {{HOST}}:{{PORT}}\r\n" - "Upgrade: websocket\r\n" - "Connection: Upgrade\r\n" - f"Sec-WebSocket-Key: {{key}}\r\n" - "Sec-WebSocket-Version: 13\r\n" - "\r\n" - ) - sock.sendall(request.encode("ascii")) - response = recv_until(sock, b"\r\n\r\n").decode("iso-8859-1", "replace") - if not response.startswith("HTTP/1.1 101"): - raise RuntimeError(f"websocket upgrade failed: {{response!r}}") - sock.sendall(masked_text_frame(payload)) - _, response_payload = read_frame(sock) - result = json.loads(response_payload.decode("utf-8")) -results = {{"transparent": result}} -print(json.dumps(results, sort_keys=True)) -"#, - host = host, - port = port, - token_env = TOKEN_ENV, - ) -} - -#[tokio::test] -async fn websocket_text_placeholder_is_rewritten_transparently() { - let _provider_lock = PROVIDER_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - - delete_provider(PROVIDER_NAME).await; - delete_provider_profile(PROVIDER_PROFILE_ID).await; - - let result = async { - let server = WebSocketProbeServer::start().await?; - let profile = write_credential_profile(server.port)?; - let profile_path = profile.path().to_string_lossy().into_owned(); - run_cli(&["profile", "import", "--file", &profile_path]).await?; - create_bound_provider(PROVIDER_NAME).await?; - let policy = write_websocket_policy(TEST_SERVER_HOST, server.port)?; - let policy_path = policy - .path() - .to_str() - .ok_or_else(|| "temp policy path should be utf-8".to_string())? - .to_string(); - let script = websocket_client_script(TEST_SERVER_HOST, server.port); - - SandboxGuard::create(&[ - "--policy", - &policy_path, - "--provider", - PROVIDER_NAME, - "--", - "python3", - "-c", - &script, - ]) - .await - } - .await; - - delete_provider(PROVIDER_NAME).await; - delete_provider_profile(PROVIDER_PROFILE_ID).await; - - let guard = result.expect("sandbox create"); - assert!( - guard - .create_output - .contains(r#""transparent": {"saw_placeholder": false, "saw_secret": true}"#), - "expected upstream to see only the resolved secret marker:\n{}", - guard.create_output - ); - assert!( - !guard.create_output.contains(TEST_SECRET), - "test output should not expose the raw WebSocket credential:\n{}", - guard.create_output - ); - assert!( - !guard.create_output.contains(PLACEHOLDER_PREFIX), - "test output should not expose unresolved credential placeholders:\n{}", - guard.create_output - ); -} diff --git a/tasks/test.toml b/tasks/test.toml index da750c904f..233d78273b 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -199,7 +199,7 @@ run = "e2e/with-docker-gateway.sh cargo nextest run --locked --manifest-path tes ["e2e:rust:push"] description = "Run conformance and policy e2e tests against a Docker gateway that pushes configuration" -env = { OPENSHELL_CONFIG_DELIVERY_MODE = "push", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-config-push", OPENSHELL_E2E_DOCKER_TEST = "config_push live_policy_update policy_activation" } +env = { OPENSHELL_CONFIG_DELIVERY_MODE = "push", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-config-push", OPENSHELL_E2E_DOCKER_TEST = "config_push policy_activation" } run = "e2e/rust/e2e-docker.sh" ["e2e:workload:build"] @@ -222,13 +222,6 @@ exec e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml \ --features e2e-docker --test supervisor_middleware_http_hooks -- --nocapture """ -["e2e:websocket-conformance"] -description = "Run focused WebSocket conformance e2e tests against a Docker-backed gateway" -run = [ - "CONTAINER_ENGINE=docker bash tasks/scripts/e2e-build-workload.sh", - "e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test websocket_conformance", -] - ["e2e:mcp"] description = "Run MCP conformance e2e scenarios against one Docker-backed gateway (static defaults for spec 2025-11-25; set OPENSHELL_MCP_CONFORMANCE_SCENARIOS for a focused subset)" run = "bash e2e/mcp-conformance.sh" diff --git a/tests/tmachine/artifacts.nix b/tests/tmachine/artifacts.nix index 5b605169b3..4c4d009596 100644 --- a/tests/tmachine/artifacts.nix +++ b/tests/tmachine/artifacts.nix @@ -102,10 +102,6 @@ let "provider_auto_create" # Needs the SPIFFE fixtures that the wrapper sets up. Wrapper lane. "provider_token_exchange" - # One test times out reaching support-container IPs from the sandbox in the - # rootless guest; the other tests in the binary pass. Nextest archive - # filters cannot select individual tests, so keep the complete binary here. - "proxy_egress_pipeline" # Holds only the on-failure restart test until #5507; the rest moved to # testkit. Nextest archive filters cannot select individual tests, so keep # the complete binary in the follow-up bucket. From 858dbc6a950398c40917b3c5dac818cffa384e18 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 12:45:44 -0800 Subject: [PATCH 15/15] docs(testing): describe the network, policy update, and port forward suites Signed-off-by: Matthew Grossman --- TESTING.md | 13 +++++++++++++ tests/suites/conformance/README.md | 12 +++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/TESTING.md b/TESTING.md index 4e642f338e..9dca2e3075 100644 --- a/TESTING.md +++ b/TESTING.md @@ -345,6 +345,11 @@ can run the suite. The expectations characterize the sandbox's behavior today, not a specification, and the `echo-socket` check needs `perl` in the workload image, which the default image has. +`ubuntu-docker-rootful-config-push` sets the gateway's +`config_delivery_mode = "push"` through a gateway override. CI runs the +conformance suite there with the deb installer, so policy updates are also +tested when the gateway pushes them to the supervisor. + ### Python E2E (`e2e/python/`) `mise run e2e:python` builds `openshell/e2e-python:dev` from @@ -650,6 +655,14 @@ rootful and rootless, and the `k3s` and `k3s-ha` installers. The tests run on the K3s node, so the K3s installers set the chart's `server.hostGatewayIP` to the node's address. +The `routable-host-ip` feature suite (`tests/suites/features/routable-host-ip`) +checks that a policy endpoint whose host is an IP literal reaches that address, +with or without `allowed_ips`. The sandbox uses the address +`host.openshell.internal` resolves to, which must be one the supervisor lets an +IP literal reach. Rootless Podman's is link-local and always blocked, so CI runs +the suite on Docker rootful, Podman rootful, and the `k3s` and `k3s-ha` +installers. + The `connect-ssh-supervision` feature suite (`tests/suites/features/connect-ssh-supervision`) checks how `sandbox connect` supervises its `ssh` process: it reconnects after losing the SSH transport and diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index 0639d70d52..45d61eb60b 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -17,7 +17,12 @@ nextest run the tests as the library's unit tests. The crate also builds the | `persistent-workspace` | Files in `/sandbox` survive stop and start | | `policy-advisor` | `policy.local` accepts a proposed rule; a denied address or hostname yields one draft rule | | `file-transfer` | `sandbox upload` and `download`: round trips, file modes and symlinks outside the workspace, `.gitignore` filtering, workspace-escape and option-looking names, a large upload | -| `credential-gating` | Provider credentials: admission of credentialed endpoints, REST body placeholders and rewriting, WebSocket binary frames, live policy binding | +| `credential-gating` | Provider credentials: admission of credentialed endpoints, REST header and body placeholders and rewriting, WebSocket text and binary frames, live policy binding, static credentials bound to their profile endpoints | +| `network` | Direct connections that bypass the proxy are denied; reserved destinations fail with `EACCES`; localhost works without proxy settings; closed listeners release their ports | +| `network-l7` | REST, GraphQL, JSON-RPC, and MCP rules allow and deny requests; REST upgrades are refused | +| `network-egress` | Transparent egress: policy reloads close open connections, ambiguous updates are rejected, `tls: skip` relays raw bytes, middleware redacts or blocks traffic, pipelined requests are authorized one at a time | +| `policy-update` | `policy set` and `policy update`: revisions and versions, repeated updates, scoped endpoint changes, the revision for a sparse initial policy | +| `port-forward` | `forward start` reaches a server on the sandbox's loopback; `forward stop` closes the port | | `image` | A sandbox runs the default image or the one it is created from | | `labels` | Labels are stored and select sandboxes in `sandbox list` | | `templates` | A sandbox created from a template uses its workload; duplicate, deleted and missing templates return the right errors | @@ -31,8 +36,9 @@ tmachine environment on CI. The other suites are ports of the old (credited in each file). `file-transfer` runs `git` on the machine that runs the tests, so a test environment needs it installed (`tests/tmachine/ansible/playbooks/test-tools.yaml` does that for the tmachine -environments). `credential-gating`, ported from NVIDIA/OpenShell#5264, starts -servers (`tests/support/host-server`) on the machine that runs the tests and +environments). `credential-gating`, `network`, `network-l7`, and +`network-egress` start servers (`tests/support/host-server`) on the machine +that runs the tests and reaches them from sandboxes created from `testkit::python_image()` through `host.openshell.internal`. `ssh` runs `ssh` and `ssh-keygen` there and needs a named gateway, which `sandbox ssh-config` writes into its `ProxyCommand`.