From 6d9ee5e07d3f64f9147cf07c82c3285b5ad8d909 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:29:33 -0800 Subject: [PATCH 1/6] test(testkit): let create_provider set provider config create_provider and install_provider return a builder that creates the provider when awaited, so tests can chain .config(key, value) for provider create --config. Existing callers are unchanged. Signed-off-by: Matthew Grossman --- tests/testkit/README.md | 3 +- tests/testkit/testkit/src/lib.rs | 1 + tests/testkit/testkit/src/provider.rs | 140 +++++++++++++++++++------- 3 files changed, 109 insertions(+), 35 deletions(-) diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 3ce6f58013..59bbd491d7 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -105,7 +105,8 @@ fails. A killed test process skips cleanup, so names are unique to the run. - `cx.import_profile(profile)` to import a provider profile and return its `id`, and `cx.create_provider(name, type, credentials)` to create a provider, each with cleanup. `cx.install_provider(name, profile, - credentials)` does both. + credentials)` does both. Chain `.config(key, value)` before the `.await` + to set provider config. - `testkit::YamlFile` for YAML a CLI command reads from a file. - `cx.client()` for SDK calls testkit does not wrap. - `?` and `.context(..)` on I/O, JSON, YAML, SDK and gRPC results, which diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index 86f28e2599..a8e782993f 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -56,6 +56,7 @@ pub use image::{PYTHON_BINARIES, base_image, python_image}; pub use openshell_sdk as sdk; pub use poll::{Poll, poll_until, with_deadline}; pub use program::main; +pub use provider::ProviderBuilder; pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; diff --git a/tests/testkit/testkit/src/provider.rs b/tests/testkit/testkit/src/provider.rs index b68b20989c..3cd6a89114 100644 --- a/tests/testkit/testkit/src/provider.rs +++ b/tests/testkit/testkit/src/provider.rs @@ -1,6 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use std::future::{Future, IntoFuture}; +use std::pin::Pin; + use serde::Deserialize; use crate::context::Context; @@ -31,57 +34,126 @@ impl Context { } /// Create a provider called `name` of `provider_type` with `credentials`, - /// as pairs of environment variable and value. Cleanup deletes the - /// provider after the test's sandboxes. + /// as pairs of environment variable and value, when awaited. Cleanup + /// deletes the provider after the test's sandboxes. /// /// The CLI reads the credential values from its environment, so they - /// aren't in the reported command. - pub async fn create_provider( - &self, - name: &str, - provider_type: &str, - credentials: &[(&str, &str)], - ) -> Result { + /// aren't in the reported command. [`ProviderBuilder`] sets other + /// `provider create` options before the await. + pub fn create_provider<'a>( + &'a self, + name: &'a str, + provider_type: &'a str, + credentials: &'a [(&'a str, &'a str)], + ) -> ProviderBuilder<'a> { + ProviderBuilder::new(self, name, ProviderType::Named(provider_type), credentials) + } + + /// Import `profile` with [`Context::import_profile`] and create a provider + /// called `name` of its type with [`Context::create_provider`], when + /// awaited. + pub fn install_provider<'a>( + &'a self, + name: &'a str, + profile: &'a str, + credentials: &'a [(&'a str, &'a str)], + ) -> ProviderBuilder<'a> { + ProviderBuilder::new(self, name, ProviderType::Profile(profile), credentials) + } + + fn on_cleanup_delete(&self, kind: &'static str, name: &str) { + let cli = self.cli_handle(); + let owned = name.to_string(); + self.on_cleanup(format!("delete {kind} {name}"), async move || { + cli.run([kind, "delete", &owned]).await?.success()?; + Ok(()) + }); + } +} + +/// Where a [`ProviderBuilder`] gets its provider type. +enum ProviderType<'a> { + /// A profile already on the gateway. + Named(&'a str), + /// A profile, in YAML, to import first. + Profile(&'a str), +} + +/// A provider to create, from [`Context::create_provider`] or +/// [`Context::install_provider`]. Awaiting it creates the provider. +/// +/// ```ignore +/// cx.create_provider(&name, &provider_type, &[]) +/// .config("project", "production") +/// .await?; +/// ``` +#[must_use = "a ProviderBuilder does nothing until awaited"] +pub struct ProviderBuilder<'a> { + cx: &'a Context, + name: &'a str, + provider_type: ProviderType<'a>, + credentials: &'a [(&'a str, &'a str)], + config: Vec, +} + +impl<'a> ProviderBuilder<'a> { + fn new( + cx: &'a Context, + name: &'a str, + provider_type: ProviderType<'a>, + credentials: &'a [(&'a str, &'a str)], + ) -> Self { + Self { + cx, + name, + provider_type, + credentials, + config: Vec::new(), + } + } + + /// Set the provider config `key` to `value` (`--config key=value`). + pub fn config(mut self, key: &str, value: &str) -> Self { + self.config.push(format!("{key}={value}")); + self + } + + async fn create(self) -> Result { + let provider_type = match self.provider_type { + ProviderType::Named(provider_type) => provider_type.to_string(), + ProviderType::Profile(profile) => self.cx.import_profile(profile).await?, + }; // Deleting a missing provider succeeds, so register the delete before // the create. - self.on_cleanup_delete("provider", name); + self.cx.on_cleanup_delete("provider", self.name); let mut args = vec![ "provider", "create", "--name", - name, + self.name, "--type", - provider_type, + &provider_type, ]; - for (key, _) in credentials { + for (key, _) in self.credentials { args.extend(["--credential", key]); } - self.cli_with_env(credentials, args) + for setting in &self.config { + args.extend(["--config", setting]); + } + self.cx + .cli_with_env(self.credentials, args) .await? .success() - .context(format!("create provider {name}"))?; + .context(format!("create provider {}", self.name))?; Ok(()) } +} - /// Import `profile` with [`Context::import_profile`] and create a provider - /// called `name` of its type with [`Context::create_provider`]. - pub async fn install_provider( - &self, - name: &str, - profile: &str, - credentials: &[(&str, &str)], - ) -> Result { - let provider_type = self.import_profile(profile).await?; - self.create_provider(name, &provider_type, credentials) - .await - } +impl<'a> IntoFuture for ProviderBuilder<'a> { + type Output = Result; + type IntoFuture = Pin + 'a>>; - fn on_cleanup_delete(&self, kind: &'static str, name: &str) { - let cli = self.cli_handle(); - let owned = name.to_string(); - self.on_cleanup(format!("delete {kind} {name}"), async move || { - cli.run([kind, "delete", &owned]).await?.success()?; - Ok(()) - }); + fn into_future(self) -> Self::IntoFuture { + Box::pin(self.create()) } } From a267f37d6d138be071451362fad731b4a7f45e2f Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:31:38 -0800 Subject: [PATCH 2/6] test(conformance): port provider files and auto-create tests to testkit Move provider_files and provider_auto_create from e2e/rust into a providers conformance suite. The auto-create test imports a run-unique profile instead of relying on a globally imported claude-code profile, so it no longer changes shared gateway state. Signed-off-by: Matthew Grossman --- e2e/rust/Cargo.toml | 5 - e2e/rust/tests/provider_auto_create.rs | 197 --------------------- e2e/rust/tests/provider_files.rs | 176 ------------------ tests/suites/conformance/README.md | 1 + tests/suites/conformance/src/lib.rs | 1 + tests/suites/conformance/src/providers.rs | 206 ++++++++++++++++++++++ tests/tmachine/artifacts.nix | 3 - 7 files changed, 208 insertions(+), 381 deletions(-) delete mode 100644 e2e/rust/tests/provider_auto_create.rs delete mode 100644 e2e/rust/tests/provider_files.rs create mode 100644 tests/suites/conformance/src/providers.rs diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 5e557abcdf..65982ed939 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -61,11 +61,6 @@ name = "policy_activation" path = "tests/policy_activation.rs" required-features = ["e2e-docker"] -[[test]] -name = "provider_files" -path = "tests/provider_files.rs" -required-features = ["e2e-docker"] - [[test]] name = "oidc_pkce" path = "tests/oidc_pkce.rs" diff --git a/e2e/rust/tests/provider_auto_create.rs b/e2e/rust/tests/provider_auto_create.rs deleted file mode 100644 index 2c8828deaf..0000000000 --- a/e2e/rust/tests/provider_auto_create.rs +++ /dev/null @@ -1,197 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e")] - -//! E2E test: `--provider ` auto-creates a provider from local credentials. -//! -//! When `--provider claude-code` is passed and no provider named "claude-code" exists, -//! the CLI should discover `ANTHROPIC_API_KEY` from the local environment, -//! auto-create a provider, and inject a supervisor-managed placeholder into the -//! sandbox child process environment. -//! -//! The sandbox command (`printenv ANTHROPIC_API_KEY`) verifies that the -//! placeholder made it all the way through to the sandbox process environment. -//! -//! Prerequisites: -//! - A running openshell gateway (`mise run gateway:docker`) -//! - The `openshell` binary (built automatically from the workspace) - -use std::process::Stdio; -use std::sync::Mutex; - -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::output::{extract_field, strip_ansi}; -#[cfg(feature = "e2e-local-container-driver")] -use openshell_e2e::harness::sandbox::E2E_WORKLOAD_IMAGE; - -const TEST_API_KEY: &str = "sk-e2e-auto-provider-test-key"; -static CLAUDE_PROVIDER_LOCK: Mutex<()> = Mutex::new(()); - -fn contains_placeholder_for_env_key(output: &str, key: &str) -> bool { - let legacy = format!("openshell:resolve:env:{key}"); - let revision_prefix = "openshell:resolve:env:v"; - let revision_suffix = format!("_{key}"); - output.split_whitespace().any(|token| { - token == legacy || (token.starts_with(revision_prefix) && token.ends_with(&revision_suffix)) - }) -} - -/// Helper: delete a provider by name, ignoring errors. -async fn delete_provider(name: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("provider") - .arg("delete") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -/// Helper: check whether a provider already exists. -async fn provider_exists(name: &str) -> bool { - let mut cmd = openshell_cmd(); - cmd.arg("provider") - .arg("get") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - cmd.status().await.is_ok_and(|status| status.success()) -} - -/// Helper: delete a sandbox by name, ignoring errors. -async fn delete_sandbox(name: &str) { - let mut cmd = openshell_cmd(); - cmd.arg("sandbox") - .arg("delete") - .arg(name) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let _ = cmd.status().await; -} - -/// `--provider claude-code --auto-providers` with `ANTHROPIC_API_KEY` set should -/// auto-create a "claude-code" provider and inject a placeholder into the sandbox. -#[tokio::test] -async fn auto_created_provider_credential_available_in_sandbox() { - let _provider_lock = CLAUDE_PROVIDER_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - - if provider_exists("claude-code").await { - eprintln!("Skipping test: existing provider 'claude-code' would make shared state unsafe"); - return; - } - - // Clean up any leftover from a previous run. - delete_provider("claude-code").await; - - // This test only reads the injected environment placeholder. Do not inherit - // the published image's network rules, which may be incompatible with the - // attached credential provider's startup validation. - let policy = tempfile::NamedTempFile::new().expect("create provider test policy"); - std::fs::write( - policy.path(), - r"version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /lib, /etc, /proc] - read_write: [/sandbox, /tmp, /dev/null] -landlock: - compatibility: best_effort -process: - run_as_user: sandbox - run_as_group: sandbox -network_policies: {} -", - ) - .expect("write provider test policy"); - - // Create a sandbox that prints the ANTHROPIC_API_KEY env var. - // --auto-providers skips the interactive prompt. - let mut cmd = openshell_cmd(); - cmd.arg("sandbox").arg("create"); - #[cfg(feature = "e2e-local-container-driver")] - cmd.arg("--from").arg(E2E_WORKLOAD_IMAGE); - cmd.arg("--detach") - .arg("--policy") - .arg(policy.path()) - .arg("--provider") - .arg("claude-code") - .arg("--auto-providers") - .args(["--", "sh", "-c", "exec sleep infinity"]) - .env("ANTHROPIC_API_KEY", TEST_API_KEY) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - let output = cmd - .output() - .await - .expect("failed to spawn openshell sandbox create"); - - let stdout = String::from_utf8_lossy(&output.stdout).to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).to_string(); - let combined = format!("{stdout}{stderr}"); - let clean = strip_ansi(&combined); - - // Parse sandbox name for cleanup. - let sandbox_name = extract_field(&combined, "Created sandbox"); - let exec_output = if let Some(ref name) = sandbox_name { - let mut exec_cmd = openshell_cmd(); - exec_cmd - .args([ - "sandbox", - "exec", - "--name", - name, - "--no-tty", - "--", - "printenv", - "ANTHROPIC_API_KEY", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - Some(exec_cmd.output().await.expect("failed to run sandbox exec")) - } else { - None - }; - - // Always clean up, even if assertions fail. - if let Some(ref name) = sandbox_name { - delete_sandbox(name).await; - } - delete_provider("claude-code").await; - - // Now assert. - assert!( - output.status.success(), - "sandbox create should succeed (exit {:?}):\n{clean}", - output.status.code() - ); - - let exec_output = exec_output.expect("sandbox name should be present"); - let exec_clean = strip_ansi(&format!( - "{}{}", - String::from_utf8_lossy(&exec_output.stdout), - String::from_utf8_lossy(&exec_output.stderr) - )); - assert!( - exec_output.status.success(), - "sandbox exec should succeed:\n{exec_clean}" - ); - - assert!( - clean.contains("Created provider claude-code"), - "output should confirm provider auto-creation:\n{clean}" - ); - - assert!( - contains_placeholder_for_env_key(&exec_clean, "ANTHROPIC_API_KEY"), - "sandbox should have placeholder ANTHROPIC_API_KEY in its environment:\n{exec_clean}" - ); - - assert!( - !exec_clean.contains(TEST_API_KEY), - "sandbox should not expose the raw ANTHROPIC_API_KEY secret:\n{exec_clean}" - ); -} diff --git a/e2e/rust/tests/provider_files.rs b/e2e/rust/tests/provider_files.rs deleted file mode 100644 index 18db5db9b7..0000000000 --- a/e2e/rust/tests/provider_files.rs +++ /dev/null @@ -1,176 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-docker")] - -//! A provider profile serves read-only config on each workload open. - -use std::io::Write as _; -use std::process::Stdio; -use std::time::Duration; - -use openshell_e2e::harness::binary::openshell_bin; -use openshell_e2e::harness::cli::run_cli; -use openshell_e2e::harness::sandbox::SandboxGuard; -use tokio::time::sleep; - -struct ProviderGuard { - profile: String, - provider: String, -} - -impl Drop for ProviderGuard { - fn drop(&mut self) { - let binary = openshell_bin(); - for _ in 0..20 { - let deleted = std::process::Command::new(&binary) - .args(["provider", "delete", &self.provider]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - if deleted.is_ok_and(|status| status.success()) { - break; - } - std::thread::sleep(Duration::from_millis(250)); - } - let _ = std::process::Command::new(&binary) - .args(["profile", "delete", &self.profile]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -async fn cli_ok(args: &[&str]) -> Result<(), String> { - let (output, code) = run_cli(args).await; - if code == 0 { - Ok(()) - } else { - Err(format!( - "{} failed (exit {code}):\n{output}", - args.join(" ") - )) - } -} - -#[tokio::test] -async fn provider_file_open_update_and_detach() -> Result<(), String> { - let suffix = format!("{}-{:08x}", std::process::id(), rand::random::()); - let profile_id = format!("pf-{suffix}"); - let provider = format!("pf-{suffix}"); - let profile_yaml = include_str!("../../../examples/provider-managed-files/acme-config.yaml") - .replace("id: acme-config", &format!("id: {profile_id}")); - let mut profile_file = tempfile::Builder::new() - .suffix(".yaml") - .tempfile() - .map_err(|error| error.to_string())?; - profile_file - .write_all(profile_yaml.as_bytes()) - .map_err(|error| error.to_string())?; - cli_ok(&[ - "profile", - "import", - "--file", - profile_file - .path() - .to_str() - .ok_or("profile path is not UTF-8")?, - ]) - .await?; - let _provider_guard = ProviderGuard { - profile: profile_id.clone(), - provider: provider.clone(), - }; - cli_ok(&[ - "provider", - "create", - "--name", - &provider, - "--type", - &profile_id, - "--config", - "endpoint=https://api.acme.example", - "--config", - "project=production", - ]) - .await?; - - let mut sandbox = SandboxGuard::create(&["--provider", &provider, "--no-tty"]).await?; - let path = format!("/run/openshell/providers/{provider}/client.toml"); - let environment_path = sandbox.exec(&["printenv", "ACME_CONFIG_FILE"]).await?; - if !environment_path.contains(&path) { - return Err(format!( - "provider path environment variable missing: {environment_path}" - )); - } - let before = sandbox.exec(&["cat", &path]).await?; - if !before.contains("project = \"production\"") { - return Err(format!("initial provider file content missing:\n{before}")); - } - let permissions = sandbox - .exec(&[ - "sh", - "-c", - &format!( - "set -eu; exec 3<{path}; test \"$(stat -Lc %a /proc/self/fd/3)\" = 600; \ - test \"$(stat -Lc %F /proc/self/fd/3)\" = 'regular file'; \ - if (printf x >&3) 2>/dev/null; then exit 1; fi; \ - echo provider-file-permissions-ok" - ), - ]) - .await?; - if !permissions.contains("provider-file-permissions-ok") { - return Err(format!( - "provider file permission assertion did not complete:\n{permissions}" - )); - } - - cli_ok(&[ - "provider", - "update", - &provider, - "--config", - "project=staging", - "--wait", - "--timeout", - "90", - ]) - .await?; - let after = sandbox.exec(&["cat", &path]).await?; - if !after.contains("project = \"staging\"") { - return Err(format!("updated provider file content missing:\n{after}")); - } - - cli_ok(&[ - "sandbox", - "provider", - "detach", - &sandbox.name, - &provider, - "--wait", - "--timeout", - "90", - ]) - .await?; - let (detached, code) = run_cli(&[ - "sandbox", - "exec", - "--name", - &sandbox.name, - "--no-tty", - "--", - "cat", - &path, - ]) - .await; - if code == 0 || !detached.contains("No such file or directory") { - return Err(format!( - "detached provider path remained readable: {detached}" - )); - } - sandbox.cleanup().await; - // Let the asynchronous sandbox deletion release the provider before - // ProviderGuard removes the provider record and its profile. - sleep(Duration::from_millis(250)).await; - Ok(()) -} diff --git a/tests/suites/conformance/README.md b/tests/suites/conformance/README.md index 9937db7738..8b2f3e32c9 100644 --- a/tests/suites/conformance/README.md +++ b/tests/suites/conformance/README.md @@ -16,6 +16,7 @@ nextest run the tests as the library's unit tests. The crate also builds the | `file-transfer` | `sandbox upload` and `download`: round trips, file modes and symlinks outside the workspace, `.gitignore` filtering, workspace-escape and option-looking names, a large upload | | `create-upload` | `sandbox create --upload`: directory basename and nested files, multiple uploads, a single file | | `credential-gating` | Provider credentials: admission of credentialed endpoints, REST body placeholders and rewriting, WebSocket binary frames, live policy binding | +| `providers` | Provider profile files: path, mode, config update, detach; `sandbox create --auto-providers` creates a provider from local credentials and injects a placeholder | | `image` | A sandbox runs the default image or the one it is created from | | `labels` | Labels are stored and select sandboxes in `sandbox list` | | `templates` | A sandbox created from a template uses its workload; duplicate, deleted and missing templates return the right errors | diff --git a/tests/suites/conformance/src/lib.rs b/tests/suites/conformance/src/lib.rs index b97238f84e..781fc4e2dd 100644 --- a/tests/suites/conformance/src/lib.rs +++ b/tests/suites/conformance/src/lib.rs @@ -19,6 +19,7 @@ mod lifecycle; mod persistent_workspace; mod policy_advisor; mod process; +mod providers; mod settings; mod smoke; mod templates; diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs new file mode 100644 index 0000000000..e67a349e3a --- /dev/null +++ b/tests/suites/conformance/src/providers.rs @@ -0,0 +1,206 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Providers attached to a sandbox: files a provider profile serves, and +//! providers that `sandbox create --provider` creates from local credentials. + +use testkit::{Context, ResultExt as _, YamlFile, policy}; + +/// The example profile that serves `client.toml` from the provider's config. +const FILES_PROFILE: &str = + include_str!("../../../../examples/provider-managed-files/acme-config.yaml"); +const AUTO_CREATE_ENV: &str = "E2E_AUTO_PROVIDER_KEY"; +const AUTO_CREATE_SECRET: &str = "sk-e2e-auto-provider-test-key"; +const PLACEHOLDER_PREFIX: &str = "openshell:resolve:env:"; + +/// A provider profile's file is readable at the path its environment variable +/// names, with mode 600 and no write access. Updating the provider's config +/// updates the file, and detaching the provider removes it. +#[testkit::test] +async fn provider_file_open_update_and_detach(cx: &mut Context) -> testkit::Result { + let provider = cx.name("pf"); + let profile = FILES_PROFILE.replace("id: acme-config", &format!("id: {provider}")); + cx.install_provider(&provider, &profile, &[]) + .config("endpoint", "https://api.acme.example") + .config("project", "production") + .await?; + + let sandbox = cx + .sandbox("pf") + .spec(|spec| spec.providers = vec![provider.clone()]) + .command(["sleep", "infinity"]) + .create_ready() + .await?; + let path = format!("/run/openshell/providers/{provider}/client.toml"); + sandbox + .exec(["printenv", "ACME_CONFIG_FILE"]) + .await? + .expect_stdout(&format!("{path}\n")) + .context("the provider file's environment variable")?; + expect_project(&sandbox, &path, "production").await?; + sandbox + .exec([ + "sh", + "-c", + &format!( + "set -eu; exec 3<{path}; test \"$(stat -Lc %a /proc/self/fd/3)\" = 600; \ + test \"$(stat -Lc %F /proc/self/fd/3)\" = 'regular file'; \ + if (printf x >&3) 2>/dev/null; then exit 1; fi" + ), + ]) + .await? + .success() + .context("the provider file is a read-only regular file with mode 600")?; + + cx.cli([ + "provider", + "update", + &provider, + "--config", + "project=staging", + "--wait", + "--timeout", + "90", + ]) + .await? + .success()?; + expect_project(&sandbox, &path, "staging").await?; + + cx.cli([ + "sandbox", + "provider", + "detach", + sandbox.name(), + &provider, + "--wait", + "--timeout", + "90", + ]) + .await? + .success()?; + let detached = sandbox.exec(["cat", &path]).await?.failure()?; + testkit::ensure!( + detached.stderr.contains("No such file or directory"), + "the detached provider's file is still there: {}", + detached.stderr + ); + Ok(()) +} + +/// `sandbox create --provider --auto-providers` creates a +/// provider named after a profile from the credential in the CLI's +/// environment. The sandbox gets a placeholder for the credential, not the +/// secret. +#[testkit::test] +async fn auto_created_provider_credential_available_in_sandbox( + cx: &mut Context, +) -> testkit::Result { + let provider = cx + .import_profile(&auto_create_profile(&cx.name("auto"))) + .await?; + // The CLI names the provider it creates after the profile. + let cli = cx.cli_handle(); + let name = provider.clone(); + cx.on_cleanup(format!("delete provider {provider}"), async move || { + cli.run(["provider", "delete", &name]).await?.success()?; + Ok(()) + }); + + // The test only reads the environment. A policy without network rules + // keeps the default policy's rules from conflicting with the provider's. + let policy = YamlFile::new(&policy::to_yaml(&policy::base())?)?; + let sandbox_name = cx.name("ac"); + cx.track(&sandbox_name); + let create = cx + .cli_with_env( + &[(AUTO_CREATE_ENV, AUTO_CREATE_SECRET)], + [ + "sandbox", + "create", + "--name", + &sandbox_name, + "--no-tty", + "--detach", + "--policy", + policy.path()?, + "--provider", + &provider, + "--auto-providers", + "--", + "sh", + "-c", + "exec sleep infinity", + ], + ) + .await? + .success()?; + testkit::ensure!( + create + .report_text() + .contains(&format!("Created provider {provider}")), + "sandbox create did not report creating provider {provider}:\n{}{}", + create.stdout, + create.stderr + ); + + let sandbox = cx.adopt(&sandbox_name).await?; + sandbox.wait_ready().await?; + let output = sandbox + .exec(["printenv", AUTO_CREATE_ENV]) + .await? + .success()?; + let value = output.stdout.trim(); + testkit::ensure!( + !value.contains(AUTO_CREATE_SECRET), + "the sandbox has the raw credential in {AUTO_CREATE_ENV}" + ); + testkit::ensure!( + is_placeholder_for(value, AUTO_CREATE_ENV), + "expected a placeholder for {AUTO_CREATE_ENV} in the sandbox; got {value:?}" + ); + Ok(()) +} + +async fn expect_project(sandbox: &testkit::Sandbox, path: &str, project: &str) -> testkit::Result { + let content = sandbox.exec(["cat", path]).await?.success()?.stdout; + testkit::ensure!( + content.contains(&format!("project = \"{project}\"")), + "expected project {project} in the provider file:\n{content}" + ); + Ok(()) +} + +/// A profile `id` whose credential the CLI discovers in [`AUTO_CREATE_ENV`]. +/// Its endpoint makes the gateway inject the credential without a policy +/// binding; nothing connects to it. +fn auto_create_profile(id: &str) -> String { + format!( + "id: {id} +display_name: E2E Auto-created Provider +category: other +credentials: + - name: api_key + env_vars: [{AUTO_CREATE_ENV}] + required: true + auth_style: header + header_name: x-api-key +discovery: + credentials: [api_key] +endpoints: + - host: api.auto-provider.example + port: 443 + protocol: rest + access: read-write + enforcement: enforce +binaries: [/usr/bin/curl] +" + ) +} + +/// Whether `value` is the placeholder for the credential in `key`, in the +/// unversioned or the revisioned form. +fn is_placeholder_for(value: &str, key: &str) -> bool { + value == format!("{PLACEHOLDER_PREFIX}{key}") + || (value.starts_with(&format!("{PLACEHOLDER_PREFIX}v")) + && value.ends_with(&format!("_{key}"))) +} diff --git a/tests/tmachine/artifacts.nix b/tests/tmachine/artifacts.nix index ef1be7ed98..02e0f549cc 100644 --- a/tests/tmachine/artifacts.nix +++ b/tests/tmachine/artifacts.nix @@ -97,9 +97,6 @@ let "local_driver_token_restart" "podman_corporate_proxy" "podman_gateway_start" - # Its move to conformance is deferred until the isolation work in #4120; see - # #4246. - "provider_auto_create" # Needs the SPIFFE fixtures that the wrapper sets up. Wrapper lane. "provider_token_exchange" # One test times out reaching support-container IPs from the sandbox in the From 9bac4beac52be5da7a3aa8ab44c7364afdfd89be Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:33:55 -0800 Subject: [PATCH 3/6] test(testkit): add service_get for exposed sandbox services Requests go to the gateway's endpoint with the service URL's host as the Host header, which is how the gateway routes service traffic. Over TLS the client trusts the gateway CA and presents the registered gateway's client certificate, through openshell-crypto's rustls configuration. Signed-off-by: Matthew Grossman --- tests/suites/conformance/Cargo.lock | 3 + tests/suites/drivers/podman/Cargo.lock | 3 + tests/suites/features/Cargo.lock | 3 + tests/support/Cargo.lock | 3 + tests/testkit/Cargo.lock | 3 + tests/testkit/README.md | 4 + tests/testkit/testkit/Cargo.toml | 5 + tests/testkit/testkit/src/cli.rs | 4 + tests/testkit/testkit/src/error.rs | 6 + tests/testkit/testkit/src/gateway.rs | 6 + tests/testkit/testkit/src/lib.rs | 2 + tests/testkit/testkit/src/service.rs | 190 +++++++++++++++++++++++++ 12 files changed, 232 insertions(+) create mode 100644 tests/testkit/testkit/src/service.rs diff --git a/tests/suites/conformance/Cargo.lock b/tests/suites/conformance/Cargo.lock index a71ec82570..aed0d59e47 100644 --- a/tests/suites/conformance/Cargo.lock +++ b/tests/suites/conformance/Cargo.lock @@ -2270,11 +2270,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/suites/drivers/podman/Cargo.lock b/tests/suites/drivers/podman/Cargo.lock index a4720466d4..c437dcfa64 100644 --- a/tests/suites/drivers/podman/Cargo.lock +++ b/tests/suites/drivers/podman/Cargo.lock @@ -2258,11 +2258,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index b8fcdca5bd..0e273a670f 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -2322,11 +2322,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/support/Cargo.lock b/tests/support/Cargo.lock index 0ea856894c..c1571bee43 100644 --- a/tests/support/Cargo.lock +++ b/tests/support/Cargo.lock @@ -2276,11 +2276,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/testkit/Cargo.lock b/tests/testkit/Cargo.lock index e0ccc98bc0..b6c66d01ee 100644 --- a/tests/testkit/Cargo.lock +++ b/tests/testkit/Cargo.lock @@ -2249,11 +2249,14 @@ dependencies = [ "noyalib", "openshell-bootstrap", "openshell-core", + "openshell-crypto", "openshell-policy", "openshell-sdk", "openshell-testkit-macros", "quick-junit", "rand 0.9.4", + "reqwest", + "rustls", "serde", "serde_json", "tempfile", diff --git a/tests/testkit/README.md b/tests/testkit/README.md index 59bbd491d7..93f6c7675c 100644 --- a/tests/testkit/README.md +++ b/tests/testkit/README.md @@ -108,6 +108,10 @@ fails. A killed test process skips cleanup, so names are unique to the run. credentials)` does both. Chain `.config(key, value)` before the `.await` to set provider config. - `testkit::YamlFile` for YAML a CLI command reads from a file. +- `cx.service_get(service_url, headers)` to request an exposed sandbox + service. It connects to the gateway's endpoint and sends the service URL's + host as the Host header, so the test machine doesn't need to resolve + service hosts. It returns the status and body whatever the status. - `cx.client()` for SDK calls testkit does not wrap. - `?` and `.context(..)` on I/O, JSON, YAML, SDK and gRPC results, which convert into `testkit::Error`. diff --git a/tests/testkit/testkit/Cargo.toml b/tests/testkit/testkit/Cargo.toml index abfa8cf97b..8a050d12cd 100644 --- a/tests/testkit/testkit/Cargo.toml +++ b/tests/testkit/testkit/Cargo.toml @@ -17,6 +17,7 @@ name = "testkit" [dependencies] openshell-bootstrap = { path = "../../../crates/openshell-bootstrap" } openshell-core = { path = "../../../crates/openshell-core", default-features = false } +openshell-crypto = { path = "../../../crates/openshell-crypto" } openshell-policy = { path = "../../../crates/openshell-policy" } openshell-sdk = { path = "../../../crates/openshell-sdk" } openshell-testkit-macros = { path = "../testkit-macros" } @@ -26,6 +27,10 @@ libtest-mimic = "0.8" linkme = "0.3" quick-junit = "0.8" rand = "0.9" +# Service requests use a rustls configuration from openshell-crypto, so +# reqwest brings no crypto provider or root store of its own. +reqwest = { version = "0.12.28", default-features = false, features = ["rustls-tls-no-provider"] } +rustls = { version = "0.23", default-features = false, features = ["std"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } diff --git a/tests/testkit/testkit/src/cli.rs b/tests/testkit/testkit/src/cli.rs index ad597482db..911c9fa0dc 100644 --- a/tests/testkit/testkit/src/cli.rs +++ b/tests/testkit/testkit/src/cli.rs @@ -52,6 +52,10 @@ impl Cli { } } + pub(crate) fn gateway(&self) -> &Gateway { + &self.gateway + } + /// Check that the CLI reaches the gateway: `openshell status` must report /// it connected. pub async fn check_connected(&self) -> Result { diff --git a/tests/testkit/testkit/src/error.rs b/tests/testkit/testkit/src/error.rs index 18ce6138c7..574979c1b8 100644 --- a/tests/testkit/testkit/src/error.rs +++ b/tests/testkit/testkit/src/error.rs @@ -99,6 +99,12 @@ impl From for Error { } } +impl From for Error { + fn from(error: reqwest::Error) -> Self { + Self::from_error(&error) + } +} + impl From for Error { fn from(error: std::io::Error) -> Self { Self::from_error(&error) diff --git a/tests/testkit/testkit/src/gateway.rs b/tests/testkit/testkit/src/gateway.rs index fef05833b7..79bd5fe827 100644 --- a/tests/testkit/testkit/src/gateway.rs +++ b/tests/testkit/testkit/src/gateway.rs @@ -121,6 +121,12 @@ impl Gateway { }) } + /// The directory holding the gateway's CA and client certificate, for + /// an mTLS gateway. + pub(crate) fn mtls_dir(&self) -> Option<&Path> { + self.mtls_dir.as_deref() + } + /// One line naming the gateway, for test logs. pub fn describe(&self) -> String { let auth = match (&self.mtls_dir, self.endpoint.starts_with("https://")) { diff --git a/tests/testkit/testkit/src/lib.rs b/tests/testkit/testkit/src/lib.rs index a8e782993f..0480b40b6a 100644 --- a/tests/testkit/testkit/src/lib.rs +++ b/tests/testkit/testkit/src/lib.rs @@ -41,6 +41,7 @@ mod provider; mod ready; mod registry; mod run; +pub mod service; mod yaml_file; pub use cleanup::CLEANUP_TIMEOUT; @@ -60,6 +61,7 @@ pub use provider::ProviderBuilder; pub use ready::check as check_ready; pub use registry::Test; pub use run::{TestInfo, run}; +pub use service::{SERVICE_REQUEST_TIMEOUT, ServiceResponse}; pub use testkit_macros::test; pub use tonic::Code as GrpcCode; pub use yaml_file::YamlFile; diff --git a/tests/testkit/testkit/src/service.rs b/tests/testkit/testkit/src/service.rs new file mode 100644 index 0000000000..f914554bf3 --- /dev/null +++ b/tests/testkit/testkit/src/service.rs @@ -0,0 +1,190 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Requests to sandbox services that the gateway exposes. +//! +//! A service URL names the sandbox and service in its host, such as +//! `http://default--my-sandbox.openshell.localhost:8080/`. The gateway routes +//! service requests by their Host header on its own listener, so testkit +//! connects to the gateway's endpoint and sends the service URL's host as the +//! Host header. The machine that runs the tests doesn't need to resolve +//! service hosts. +//! +//! The request keeps the service URL's scheme. A gateway listening on +//! loopback reports `http` service URLs and accepts plaintext service +//! requests on its TLS listener. Over TLS, testkit verifies the gateway's +//! certificate against the endpoint's host and presents the registered +//! gateway's client certificate. + +use std::path::Path; +use std::sync::Arc; +use std::time::Duration; + +use reqwest::Url; +use reqwest::header::HOST; +use rustls::RootCertStore; +use rustls::pki_types::pem::PemObject as _; +use rustls::pki_types::{CertificateDer, PrivateKeyDer}; + +use crate::context::Context; +use crate::error::{Error, Result, ResultExt as _}; +use crate::gateway::Gateway; + +/// How long one service request may take. +pub const SERVICE_REQUEST_TIMEOUT: Duration = Duration::from_secs(30); + +/// The response to a service request. +#[derive(Debug)] +pub struct ServiceResponse { + pub status: u16, + pub body: String, +} + +impl Context { + /// Send `GET service_url` with `headers` through the test's gateway and + /// return the response, whatever its status. See [the module + /// docs](crate::service) for how the request reaches the gateway. + pub async fn service_get( + &self, + service_url: &str, + headers: &[(&str, &str)], + ) -> Result { + let cli = self.cli_handle(); + let request = ServiceRequest::new(cli.gateway(), service_url)?; + let mut builder = client(cli.gateway())? + .get(request.target) + .header(HOST, &request.host); + for (name, value) in headers { + builder = builder.header(*name, *value); + } + let what = format!("GET {service_url} through {}", cli.gateway().endpoint); + let response = builder.send().await.context(&what)?; + let status = response.status().as_u16(); + let body = response.text().await.context(&what)?; + Ok(ServiceResponse { status, body }) + } +} + +/// Where a service request goes, and the Host header that selects the +/// service. +#[derive(Debug, PartialEq, Eq)] +struct ServiceRequest { + target: Url, + host: String, +} + +impl ServiceRequest { + fn new(gateway: &Gateway, service_url: &str) -> Result { + let service = parse(service_url, "service URL")?; + let endpoint = parse(&gateway.endpoint, "gateway endpoint")?; + let service_host = service + .host_str() + .ok_or_else(|| Error::msg(format!("service URL {service_url} has no host")))?; + let host = match service.port() { + Some(port) => format!("{service_host}:{port}"), + None => service_host.to_string(), + }; + let mut target = service.clone(); + target + .set_host(endpoint.host_str()) + .map_err(Error::from_display) + .context(format!("send {service_url} to {}", gateway.endpoint))?; + target + .set_port(endpoint.port_or_known_default()) + .map_err(|()| Error::msg(format!("send {service_url} to {}", gateway.endpoint)))?; + Ok(Self { target, host }) + } +} + +fn parse(url: &str, what: &str) -> Result { + Url::parse(url) + .map_err(Error::from_display) + .context(format!("parse {what} {url}")) +} + +/// An HTTP client that trusts the gateway's CA and presents its client +/// certificate. Without a registered mTLS gateway it trusts no certificate, +/// so only plaintext requests succeed. +fn client(gateway: &Gateway) -> Result { + let builder = openshell_crypto::tls::client_builder(); + let tls = match gateway.mtls_dir() { + Some(dir) => { + let mut roots = RootCertStore::empty(); + for certificate in certificates(&dir.join("ca.crt"))? { + roots + .add(certificate) + .map_err(Error::from_display) + .context("trust the gateway CA")?; + } + builder + .with_root_certificates(roots) + .with_client_auth_cert( + certificates(&dir.join("tls.crt"))?, + private_key(&dir.join("tls.key"))?, + ) + .map_err(Error::from_display) + .context("use the gateway client certificate")? + } + None => builder + .with_root_certificates(RootCertStore::empty()) + .with_no_client_auth(), + }; + reqwest::Client::builder() + .use_preconfigured_tls(Arc::new(tls)) + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + .timeout(SERVICE_REQUEST_TIMEOUT) + .build() + .context("build the service HTTP client") +} + +fn certificates(path: &Path) -> Result>> { + CertificateDer::pem_file_iter(path) + .and_then(Iterator::collect) + .map_err(Error::from_display) + .context(format!("read {}", path.display())) +} + +fn private_key(path: &Path) -> Result> { + PrivateKeyDer::from_pem_file(path) + .map_err(Error::from_display) + .context(format!("read {}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::ServiceRequest; + use crate::gateway::Gateway; + + fn request(endpoint: &str, service_url: &str) -> ServiceRequest { + let gateway = Gateway::from_endpoint(endpoint.to_string(), "default".to_string()); + ServiceRequest::new(&gateway, service_url).expect("service request") + } + + /// A loopback gateway with TLS reports `http` service URLs; the request + /// goes to the gateway's address in plaintext, with the service as Host. + #[test] + fn plaintext_service_on_a_tls_gateway() { + let request = request( + "https://127.0.0.1:17670", + "http://default--sb.openshell.localhost:17670/path?q=1", + ); + assert_eq!(request.target.as_str(), "http://127.0.0.1:17670/path?q=1"); + assert_eq!(request.host, "default--sb.openshell.localhost:17670"); + } + + /// The request goes to the gateway's port even when the service URL + /// names another one, such as the gateway's port inside a cluster. + #[test] + fn service_port_differs_from_the_endpoint() { + let request = request( + "https://openshell.openshell.svc.cluster.local:8443", + "https://default--sb--web.openshell.localhost/", + ); + assert_eq!( + request.target.as_str(), + "https://openshell.openshell.svc.cluster.local:8443/" + ); + assert_eq!(request.host, "default--sb--web.openshell.localhost"); + } +} From b70615fcb0b2b14ba22747bb571dce1861e4d90a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 11:37:08 -0800 Subject: [PATCH 4/6] test(features): port service bearer passthrough to a service-exposure suite Signed-off-by: Matthew Grossman --- .github/workflows/e2e-suites.yml | 5 + TESTING.md | 6 + e2e/rust/Cargo.lock | 2 - e2e/rust/Cargo.toml | 7 - e2e/rust/tests/service_bearer_passthrough.rs | 350 ------------------ tests/suites/features/Cargo.lock | 8 + tests/suites/features/Cargo.toml | 2 +- .../features/service-exposure/Cargo.toml | 16 + .../tests/service_exposure.rs | 119 ++++++ 9 files changed, 155 insertions(+), 360 deletions(-) delete mode 100644 e2e/rust/tests/service_bearer_passthrough.rs create mode 100644 tests/suites/features/service-exposure/Cargo.toml create mode 100644 tests/suites/features/service-exposure/tests/service_exposure.rs diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index 232d3234a1..4add698aa9 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -104,6 +104,11 @@ jobs: {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"host-gateway","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"host-gateway","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"service-exposure","gateway":"openshell","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s","family":"features","package":"service-exposure","gateway":"tmachine","runner":"ubuntu-24.04"}, + {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"features","package":"service-exposure","gateway":"tmachine","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}, {"environment":"fedora-podman-rootful-userns-default","installer":"rpm","family":"drivers/podman","gateway":"openshell","runner":"ubuntu-24.04"}, diff --git a/TESTING.md b/TESTING.md index 084535630a..79caca15ac 100644 --- a/TESTING.md +++ b/TESTING.md @@ -650,6 +650,12 @@ rootful and rootless, and the `k3s` and `k3s-ha` installers. The tests run on the K3s node, so the K3s installers set the chart's `server.hostGatewayIP` to the node's address. +The `service-exposure` feature suite (`tests/suites/features/service-exposure`) +exposes a service in a sandbox and checks how the gateway forwards the client's +`Authorization` header. The test sends requests to the gateway's endpoint with +the service URL's host as the Host header, so the test machine doesn't resolve +service hosts. CI runs it on the same environments as `host-gateway`. + Run the VM-backed Rust CLI e2e suite: ```shell diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index 4782ce04bc..96feec5898 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -2039,7 +2039,6 @@ dependencies = [ "opentelemetry-proto", "prost", "rand 0.9.5", - "rustls", "serde", "serde_json", "serial_test", @@ -2047,7 +2046,6 @@ dependencies = [ "sha2 0.10.9", "tempfile", "tokio", - "tokio-rustls", "tokio-stream", "tonic", "tonic-prost", diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 65982ed939..a0aa0949c3 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -71,11 +71,6 @@ name = "vm_overlay" path = "tests/vm_overlay.rs" required-features = ["e2e-vm"] -[[test]] -name = "service_bearer_passthrough" -path = "tests/service_bearer_passthrough.rs" -required-features = ["e2e-docker"] - [[test]] name = "rootfs_tar" path = "tests/rootfs_tar.rs" @@ -276,13 +271,11 @@ sha1 = "0.10" sha2 = "0.10" hex = "0.4" rand = "0.9" -rustls = { version = "0.23", default-features = false, features = ["std", "logging", "tls12"] } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } tonic = { version = "0.14", features = ["transport"] } tonic-prost = "0.14" -tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12"] } tower = "0.5" url = "2" nix = { version = "0.29", features = ["process", "signal", "term", "user"] } diff --git a/e2e/rust/tests/service_bearer_passthrough.rs b/e2e/rust/tests/service_bearer_passthrough.rs deleted file mode 100644 index 75415c7df2..0000000000 --- a/e2e/rust/tests/service_bearer_passthrough.rs +++ /dev/null @@ -1,350 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -#![cfg(feature = "e2e-docker")] - -//! Verifies application bearer authorization behavior through an exposed -//! `OpenShell` service. - -use std::fs::File; -use std::io::BufReader; -use std::net::Ipv4Addr; -use std::path::{Path, PathBuf}; -use std::process::Stdio; -use std::sync::Arc; -use std::time::Duration; - -use bytes::Bytes; -use http_body_util::{BodyExt as _, Empty}; -use hyper::client::conn::http1; -use hyper::{Request, StatusCode, header}; -use hyper_util::rt::TokioIo; -use openshell_e2e::harness::binary::openshell_cmd; -use openshell_e2e::harness::sandbox::{E2E_WORKLOAD_IMAGE, SandboxGuard}; -use rustls::RootCertStore; -use rustls::pki_types::{CertificateDer, PrivateKeyDer, ServerName, pem::PemObject}; -use serde_json::Value; -use tokio::io::{AsyncRead, AsyncWrite}; -use tokio::net::TcpStream; -use tokio::time::{sleep, timeout}; -use tokio_rustls::TlsConnector; -use url::Position; - -const SERVICE_PORT: &str = "4500"; -const BEARER_TOKEN: &str = "Bearer openshell-e2e-application-token"; -const READY_TIMEOUT: Duration = Duration::from_mins(1); -const HEADER_ECHO_SERVER: &str = r#" -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer - -class Handler(BaseHTTPRequestHandler): - def do_GET(self): - body = self.headers.get("Authorization", "").encode() - self.send_response(200) - self.send_header("Content-Type", "text/plain") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, _format, *_args): - pass - -ThreadingHTTPServer(("127.0.0.1", 4500), Handler).serve_forever() -"#; - -async fn run_cli(args: &[&str]) -> Result { - let mut command = openshell_cmd(); - command - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - command - .output() - .await - .map_err(|error| format!("failed to run openshell: {error}")) -} - -enum ServiceTransport { - Http, - Https { - connector: TlsConnector, - server_name: ServerName<'static>, - }, -} - -struct ServiceTarget { - port: u16, - authority: String, - path: String, - transport: ServiceTransport, -} - -impl ServiceTarget { - fn from_url(url: &str) -> Result { - let url = url::Url::parse(url).map_err(|error| format!("invalid service URL: {error}"))?; - let host = url - .host_str() - .ok_or_else(|| "service URL omitted its host".to_string())?; - let port = url - .port_or_known_default() - .ok_or_else(|| "service URL omitted its port".to_string())?; - let transport = match url.scheme() { - "http" => ServiceTransport::Http, - "https" => ServiceTransport::Https { - connector: e2e_tls_connector()?, - server_name: ServerName::try_from(host.to_string()) - .map_err(|error| format!("invalid service TLS server name: {error}"))?, - }, - scheme => return Err(format!("unsupported service URL scheme {scheme:?}")), - }; - let authority = url[Position::BeforeHost..Position::AfterPort].to_string(); - let path = url.query().map_or_else( - || url.path().to_string(), - |query| format!("{}?{query}", url.path()), - ); - - Ok(Self { - port, - authority, - path, - transport, - }) - } -} - -fn e2e_mtls_dir() -> Result { - let config_home = std::env::var_os("XDG_CONFIG_HOME") - .ok_or_else(|| "XDG_CONFIG_HOME is required for an HTTPS service URL".to_string())?; - let gateway = std::env::var("OPENSHELL_GATEWAY") - .map_err(|_| "OPENSHELL_GATEWAY is required for an HTTPS service URL".to_string())?; - Ok(PathBuf::from(config_home) - .join("openshell/gateways") - .join(gateway) - .join("mtls")) -} - -fn load_certificates( - path: &Path, - description: &str, -) -> Result>, String> { - let file = File::open(path) - .map_err(|error| format!("open {description} '{}': {error}", path.display()))?; - let mut reader = BufReader::new(file); - CertificateDer::pem_reader_iter(&mut reader) - .collect::, _>>() - .map_err(|error| format!("parse {description} '{}': {error}", path.display())) -} - -fn load_private_key(path: &Path) -> Result, String> { - let file = File::open(path) - .map_err(|error| format!("open client TLS key '{}': {error}", path.display()))?; - let mut reader = BufReader::new(file); - PrivateKeyDer::pem_reader_iter(&mut reader) - .next() - .transpose() - .map_err(|error| format!("parse client TLS key '{}': {error}", path.display()))? - .ok_or_else(|| format!("client TLS key '{}' is empty", path.display())) -} - -fn e2e_tls_connector() -> Result { - let mtls_dir = e2e_mtls_dir()?; - let ca_path = mtls_dir.join("ca.crt"); - let mut roots = RootCertStore::empty(); - for certificate in load_certificates(&ca_path, "gateway CA certificate")? { - roots.add(certificate).map_err(|error| { - format!( - "add gateway CA certificate '{}': {error}", - ca_path.display() - ) - })?; - } - let client_certificates = - load_certificates(&mtls_dir.join("tls.crt"), "client TLS certificate")?; - let client_key = load_private_key(&mtls_dir.join("tls.key"))?; - let config = openshell_crypto::tls::client_builder() - .with_root_certificates(roots) - .with_client_auth_cert(client_certificates, client_key) - .map_err(|error| format!("build e2e mTLS client configuration: {error}"))?; - Ok(TlsConnector::from(Arc::new(config))) -} - -async fn request_over_stream( - stream: S, - target: &ServiceTarget, - authorization: &str, -) -> Result<(StatusCode, String), String> -where - S: AsyncRead + AsyncWrite + Unpin + Send + 'static, -{ - let (mut sender, connection) = http1::handshake(TokioIo::new(stream)) - .await - .map_err(|error| format!("start HTTP connection: {error}"))?; - tokio::spawn(async move { - let _ = connection.await; - }); - - let request = Request::builder() - .uri(&target.path) - .header(header::HOST, &target.authority) - .header(header::AUTHORIZATION, authorization) - .body(Empty::::new()) - .map_err(|error| format!("build service request: {error}"))?; - let response = sender - .send_request(request) - .await - .map_err(|error| format!("send service request: {error}"))?; - let status = response.status(); - let body = response - .into_body() - .collect() - .await - .map_err(|error| format!("read service response: {error}"))? - .to_bytes(); - let body = String::from_utf8(body.to_vec()) - .map_err(|error| format!("service returned non-UTF-8 data: {error}"))?; - Ok((status, body)) -} - -#[allow( - clippy::disallowed_methods, - reason = "the e2e workspace does not depend on openshell-core" -)] -async fn request_service( - target: &ServiceTarget, - authorization: &str, -) -> Result<(StatusCode, String), String> { - // The service hostname is a virtual routing authority. Dial the loopback - // gateway directly so the test does not depend on the host resolver - // recognizing arbitrary subdomains of `.localhost`. - let stream = TcpStream::connect((Ipv4Addr::LOCALHOST, target.port)) - .await - .map_err(|error| format!("connect to loopback service gateway: {error}"))?; - let _ = stream.set_nodelay(true); - match &target.transport { - ServiceTransport::Http => request_over_stream(stream, target, authorization).await, - ServiceTransport::Https { - connector, - server_name, - } => { - let stream = connector - .connect(server_name.clone(), stream) - .await - .map_err(|error| format!("start service TLS connection: {error}"))?; - request_over_stream(stream, target, authorization).await - } - } -} - -async fn wait_for_authorization(url: &str, expected: &str) -> Result<(), String> { - // Parse the URL and load TLS material before the retry loop so permanent - // test-configuration errors fail immediately instead of looking like - // service-readiness timeouts. - let target = ServiceTarget::from_url(url)?; - let mut last_observation = "no request attempted".to_string(); - let result = timeout(READY_TIMEOUT, async { - loop { - match request_service(&target, BEARER_TOKEN).await { - Ok((StatusCode::OK, body)) if body == expected => return Ok(()), - Ok((StatusCode::OK, body)) => { - return Err(format!( - "service received unexpected Authorization value: {body:?}" - )); - } - Ok((status, body)) - if matches!( - status, - StatusCode::BAD_GATEWAY - | StatusCode::PRECONDITION_FAILED - | StatusCode::SERVICE_UNAVAILABLE - ) => - { - last_observation = - format!("service returned retryable status {status} with body {body:?}"); - sleep(Duration::from_millis(250)).await; - } - Err(error) => { - last_observation = error; - sleep(Duration::from_millis(250)).await; - } - Ok((status, body)) => { - return Err(format!( - "service returned unexpected status {status} with body {body:?}" - )); - } - } - } - }) - .await; - - match result { - Ok(result) => result, - Err(_) => Err(format!( - "timed out waiting for the exposed service; last observation: {last_observation}" - )), - } -} - -#[tokio::test] -async fn service_bearer_passthrough_preserves_authorization_header() { - let sandbox_name = format!("svc-auth-{}", std::process::id()); - let create = run_cli(&[ - "sandbox", - "create", - "--name", - &sandbox_name, - "--from", - E2E_WORKLOAD_IMAGE, - "--expose", - SERVICE_PORT, - "--output", - "json", - "--detach", - "--no-tty", - "--", - "python3", - "-c", - HEADER_ECHO_SERVER, - ]) - .await - .expect("run sandbox create"); - assert!( - create.status.success(), - "sandbox create failed with exit {:?}: {}", - create.status.code(), - String::from_utf8_lossy(&create.stderr) - ); - let mut sandbox = SandboxGuard::manage_existing(sandbox_name.clone()); - - let created: Value = serde_json::from_slice(&create.stdout).expect("parse sandbox create JSON"); - let service_url = created - .get("service_urls") - .and_then(|urls| urls.get("")) - .and_then(Value::as_str) - .expect("unnamed service URL in create response"); - - wait_for_authorization(service_url, "") - .await - .expect("default mode should strip Authorization"); - - let expose = run_cli(&[ - "service", - "expose", - &sandbox_name, - SERVICE_PORT, - "--authorization-mode", - "bearer-passthrough", - ]) - .await - .expect("run service re-expose"); - assert!( - expose.status.success(), - "service re-expose failed with exit {:?}: {}", - expose.status.code(), - String::from_utf8_lossy(&expose.stderr) - ); - - wait_for_authorization(service_url, BEARER_TOKEN) - .await - .expect("passthrough mode should preserve Authorization"); - - sandbox.cleanup().await; -} diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index 0e273a670f..dcf779f1d7 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -3209,6 +3209,14 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "service-exposure" +version = "0.0.0" +dependencies = [ + "openshell-testkit", + "serde_json", +] + [[package]] name = "sha1" version = "0.10.6" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index a592b6f028..b418ae40f6 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["cdi", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image"] +members = ["cdi", "host-gateway", "keycloak-provider-refresh", "kubernetes-updates", "oci-image", "service-exposure"] diff --git a/tests/suites/features/service-exposure/Cargo.toml b/tests/suites/features/service-exposure/Cargo.toml new file mode 100644 index 0000000000..73b970a33d --- /dev/null +++ b/tests/suites/features/service-exposure/Cargo.toml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "service-exposure" +description = "Features suite: requests reach a sandbox service the gateway exposes" +version = "0.0.0" +edition = "2024" +rust-version = "1.94" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA/OpenShell" +publish = false + +[dependencies] +openshell-testkit = { path = "../../../testkit/testkit" } +serde_json = "1" diff --git a/tests/suites/features/service-exposure/tests/service_exposure.rs b/tests/suites/features/service-exposure/tests/service_exposure.rs new file mode 100644 index 0000000000..8b78e49f80 --- /dev/null +++ b/tests/suites/features/service-exposure/tests/service_exposure.rs @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Exposed sandbox services and the application's `Authorization` header. +//! +//! The sandbox runs a server that echoes the request's `Authorization` +//! header, and the test requests it through the gateway with +//! `cx.service_get`. + +use std::time::Duration; + +use testkit::{Context, Poll, ResultExt as _, poll_until}; + +const SERVICE_PORT: &str = "4500"; +const BEARER_TOKEN: &str = "Bearer openshell-e2e-application-token"; +/// How long an exposed service may take to start answering, or to apply a +/// changed authorization mode. +const SERVICE_TIMEOUT: Duration = Duration::from_mins(1); +const HEADER_ECHO_SERVER: &str = r#" +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +class Handler(BaseHTTPRequestHandler): + def do_GET(self): + body = self.headers.get("Authorization", "").encode() + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, _format, *_args): + pass + +ThreadingHTTPServer(("127.0.0.1", 4500), Handler).serve_forever() +"#; + +/// An exposed service doesn't receive the client's `Authorization` header by +/// default. Exposing it again with `--authorization-mode bearer-passthrough` +/// forwards the header unchanged. +#[testkit::test] +async fn bearer_passthrough_preserves_authorization_header(cx: &mut Context) -> testkit::Result { + let builder = cx + .cli_sandbox("svc") + .image(testkit::python_image()) + .args(["--expose", SERVICE_PORT, "--output", "json", "--detach"]) + .command(["python3", "-c", HEADER_ECHO_SERVER]); + let name = builder.name().to_string(); + let created: serde_json::Value = builder + .run() + .await? + .success() + .context(format!("create sandbox {name}"))? + .json()?; + let sandbox = cx.adopt(&name).await?; + let service_url = created["service_urls"][""] + .as_str() + .ok_or_else(|| testkit::Error::msg(format!("no unnamed service URL in {created}")))? + .to_string(); + sandbox.wait_ready().await?; + + wait_for_authorization(cx, &service_url, "") + .await + .context("the default mode strips Authorization")?; + + cx.cli([ + "service", + "expose", + &name, + SERVICE_PORT, + "--authorization-mode", + "bearer-passthrough", + ]) + .await? + .success()?; + wait_for_authorization(cx, &service_url, BEARER_TOKEN) + .await + .context("bearer-passthrough mode preserves Authorization") +} + +/// Request `service_url` with [`BEARER_TOKEN`] until the service echoes +/// `expected`. Connection errors and the gateway's statuses for a service +/// that isn't reachable yet are retried; any other answer fails. +async fn wait_for_authorization( + cx: &Context, + service_url: &str, + expected: &str, +) -> testkit::Result { + poll_until( + &format!("{service_url} to receive Authorization {expected:?}"), + SERVICE_TIMEOUT, + Duration::from_millis(250), + async || { + let response = match cx + .service_get(service_url, &[("authorization", BEARER_TOKEN)]) + .await + { + Ok(response) => response, + Err(error) => return Ok(Poll::Pending(error.to_string())), + }; + match response.status { + 200 if response.body == expected => Ok(Poll::Ready(())), + 200 => Err(testkit::Error::msg(format!( + "the service received Authorization {:?}", + response.body + ))), + // Bad gateway, precondition failed, service unavailable. + 502 | 412 | 503 => Ok(Poll::Pending(format!( + "status {} with body {:?}", + response.status, response.body + ))), + status => Err(testkit::Error::msg(format!( + "the service answered status {status} with body {:?}", + response.body + ))), + } + }, + ) + .await +} From 182481d97171c5cefe503289b31a56a2ce13c71a Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 13:46:10 -0800 Subject: [PATCH 5/6] fix(testkit): pass the service client's rustls config by value Signed-off-by: Matthew Grossman --- tests/testkit/testkit/src/service.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/testkit/testkit/src/service.rs b/tests/testkit/testkit/src/service.rs index f914554bf3..8e2ee3102a 100644 --- a/tests/testkit/testkit/src/service.rs +++ b/tests/testkit/testkit/src/service.rs @@ -17,7 +17,6 @@ //! gateway's client certificate. use std::path::Path; -use std::sync::Arc; use std::time::Duration; use reqwest::Url; @@ -130,7 +129,7 @@ fn client(gateway: &Gateway) -> Result { .with_no_client_auth(), }; reqwest::Client::builder() - .use_preconfigured_tls(Arc::new(tls)) + .use_preconfigured_tls(tls) .no_proxy() .redirect(reqwest::redirect::Policy::none()) .timeout(SERVICE_REQUEST_TIMEOUT) @@ -153,9 +152,18 @@ fn private_key(path: &Path) -> Result> { #[cfg(test)] mod tests { - use super::ServiceRequest; + use super::{ServiceRequest, client}; use crate::gateway::Gateway; + /// reqwest accepts the rustls configuration only as a plain + /// `ClientConfig`; anything else fails when the client is built. + #[test] + fn client_accepts_the_tls_configuration() { + let gateway = + Gateway::from_endpoint("http://127.0.0.1:8080".to_string(), "default".to_string()); + client(&gateway).expect("service HTTP client"); + } + fn request(endpoint: &str, service_url: &str) -> ServiceRequest { let gateway = Gateway::from_endpoint(endpoint.to_string(), "default".to_string()); ServiceRequest::new(&gateway, service_url).expect("service request") From c7af9e2cfaa18e641b24d1dcebb6742fd4bf37d4 Mon Sep 17 00:00:00 2001 From: Matthew Grossman Date: Tue, 17 Nov 2026 13:46:10 -0800 Subject: [PATCH 6/6] test(conformance): create the auto-create sandbox from the workload image Signed-off-by: Matthew Grossman --- tests/suites/conformance/src/providers.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/suites/conformance/src/providers.rs b/tests/suites/conformance/src/providers.rs index e67a349e3a..c5b0cdfec8 100644 --- a/tests/suites/conformance/src/providers.rs +++ b/tests/suites/conformance/src/providers.rs @@ -110,6 +110,7 @@ async fn auto_created_provider_credential_available_in_sandbox( // keeps the default policy's rules from conflicting with the provider's. let policy = YamlFile::new(&policy::to_yaml(&policy::base())?)?; let sandbox_name = cx.name("ac"); + let image = testkit::python_image(); cx.track(&sandbox_name); let create = cx .cli_with_env( @@ -121,6 +122,8 @@ async fn auto_created_provider_credential_available_in_sandbox( &sandbox_name, "--no-tty", "--detach", + "--from", + &image, "--policy", policy.path()?, "--provider",