From 7d65206eb4a7763d381911671a9ed220df0c7538 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 18 Aug 2026 00:47:59 -0700 Subject: [PATCH 01/13] refactor(compute): register compiled drivers Signed-off-by: Drew Newberry --- .../skills/debug-openshell-cluster/SKILL.md | 7 + architecture/compute-runtimes.md | 17 + crates/openshell-core/src/config.rs | 8 +- crates/openshell-server/src/cli.rs | 20 +- .../src/compute/driver_config.rs | 11 +- crates/openshell-server/src/compute/mod.rs | 7 +- crates/openshell-server/src/lib.rs | 696 ++++++++++++++---- crates/openshell-server/src/main.rs | 5 +- 8 files changed, 620 insertions(+), 151 deletions(-) diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index bc35e0d351..d7723fd8d7 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -19,6 +19,13 @@ The target deployment flow is: 4. The CLI registers a reachable gateway endpoint with `openshell gateway add`. 5. The gateway creates sandboxes through the selected compute driver. +The standard gateway binary explicitly installs its compiled Docker, Podman, +Kubernetes, and VM registrations at startup. With no configured driver, the +gateway probes only installed registrations in priority order (Kubernetes, +Podman, then Docker); VM has no probe and remains opt-in. A custom gateway +binary may install a different set, so confirm the binary's registered drivers +when auto-detection reports that no suitable driver is available. + For local evaluation only, TLS may be disabled and the gateway can be reached through `http://127.0.0.1:`. ## Prerequisites diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 33a9b7c2c4..3a8a5b709b 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -91,6 +91,23 @@ The gateway records driver identity and version from the startup capability response. Elevated gateway info reports that initialized driver snapshot instead of re-querying drivers on each request. +## Compiled Driver Selection + +The gateway binary explicitly installs the compute drivers compiled into that +binary before entering server startup. The server selects a configured driver +by normalized registry name. When no driver is configured, it evaluates only +the installed drivers' probes and chooses the lowest registered priority. +Drivers without a probe, including VM, remain opt-in. + +This follows the same composition model as SQLx's `Any` drivers: the binary +defines the available implementation set, while the runtime consumes a generic +registry. Adding or removing a compiled driver therefore changes registration +rather than the server's selection flow. Alternate gateway binaries can install +their own `ComputeDriverFactory` registrations and hand the completed registry +to `run_cli_with_compute_drivers`; factories receive merged driver config and +finish through the same in-process runtime adapter. A configured UDS endpoint +still takes precedence over a compiled registration with the same name. + ## Stop and Start Lifecycle The gateway persists lifecycle intent before mutating compute: diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index fcbdeb73b6..62411e20b5 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -208,7 +208,9 @@ pub fn detect_driver() -> Option { None } -fn is_podman_available() -> bool { +/// Return whether a responsive local Podman API socket is available. +#[must_use] +pub fn is_podman_available() -> bool { detect_podman_socket().is_some() } @@ -266,7 +268,9 @@ fn podman_socket_candidates_from_env( candidates } -fn is_docker_available() -> bool { +/// Return whether a responsive local Docker API socket is available. +#[must_use] +pub fn is_docker_available() -> bool { detect_docker_socket().is_some() } diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index fc70284721..ae831d2016 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -17,7 +17,10 @@ use crate::certgen; use crate::compute::driver_config::GuestTlsPaths; use crate::config_file::{self, ConfigFile, GatewayFileSection}; use crate::defaults::{self, LocalTlsPaths}; -use crate::{ServerStartupConfig, run_server, tracing_bus::TracingLogBus}; +use crate::{ + ComputeDriverRegistry, ServerStartupConfig, install_default_compute_drivers, run_server, + tracing_bus::TracingLogBus, +}; /// `OpenShell` gateway process - gRPC and HTTP server with protocol multiplexing. /// @@ -219,6 +222,11 @@ pub fn command() -> Command { } pub async fn run_cli() -> Result<()> { + run_cli_with_compute_drivers(install_default_compute_drivers()).await +} + +/// Run the gateway CLI with the compute drivers linked by the binary. +pub async fn run_cli_with_compute_drivers(compute_drivers: ComputeDriverRegistry) -> Result<()> { rustls::crypto::ring::default_provider() .install_default() .map_err(|e| miette::miette!("failed to install rustls crypto provider: {e:?}"))?; @@ -228,7 +236,7 @@ pub async fn run_cli() -> Result<()> { match cli.command { Some(Commands::GenerateCerts(args)) => certgen::run(args).await, - None => Box::pin(run_from_args(cli.run, matches)).await, + None => Box::pin(run_from_args(cli.run, matches, compute_drivers)).await, } } @@ -468,7 +476,11 @@ fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result Result<()> { +async fn run_from_args( + mut args: RunArgs, + matches: ArgMatches, + compute_drivers: ComputeDriverRegistry, +) -> Result<()> { let prepared = prepare_server_config(&mut args, &matches)?; let tracing_log_bus = TracingLogBus::new(); @@ -537,7 +549,7 @@ async fn run_from_args(mut args: RunArgs, matches: ArgMatches) -> Result<()> { info!(bind = %prepared.config.bind_address, "Starting OpenShell server"); - let result = Box::pin(run_server(prepared, tracing_log_bus)).await; + let result = Box::pin(run_server(prepared, tracing_log_bus, compute_drivers)).await; tracing_handle.shutdown(); diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index 9f4cac9a01..f0eb6f98b4 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -24,6 +24,12 @@ pub struct GuestTlsPaths { key: PathBuf, } +impl GuestTlsPaths { + pub(crate) fn as_paths(&self) -> (&std::path::Path, &std::path::Path, &std::path::Path) { + (&self.ca, &self.cert, &self.key) + } +} + impl From<&LocalTlsPaths> for GuestTlsPaths { fn from(paths: &LocalTlsPaths) -> Self { Self { @@ -60,7 +66,10 @@ pub struct RemoteDriverConfig { pub socket_path: PathBuf, } -fn driver_config_from_context(context: DriverStartupContext<'_>, driver_name: &str) -> Result +pub fn driver_config_from_context( + context: DriverStartupContext<'_>, + driver_name: &str, +) -> Result where T: Default + serde::de::DeserializeOwned, { diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 0c3a48205b..0261520f9a 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -77,8 +77,9 @@ use tonic::{Code, Request, Status}; use tower::service_fn; use tracing::{Instrument as _, debug, info, warn}; -type DriverWatchStream = Pin> + Send>>; -type SharedComputeDriver = +pub type DriverWatchStream = + Pin> + Send>>; +pub type SharedComputeDriver = Arc + Send + Sync>; use traced_driver::TracedDriver; @@ -582,7 +583,7 @@ impl ComputeRuntime { driver.name = %driver_name, ) )] - async fn from_driver( + pub(crate) async fn from_driver( driver_name: String, driver: SharedComputeDriver, driver_process: Option>, diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 6ff455bea2..e181eeae7e 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -9,19 +9,9 @@ //! - Protocol multiplexing (gRPC + HTTP on same port) //! - mTLS support //! -//! TODO(driver-abstraction): `build_compute_runtime` still switches on -//! built-in driver names and calls driver-specific constructors -//! ([`ComputeRuntime::new_kubernetes`], [`ComputeRuntime::new_docker`], -//! [`compute::vm::spawn`] + [`ComputeRuntime::new_remote_driver`], -//! [`ComputeRuntime::new_podman`]). Endpoint-backed drivers now share the -//! remote `compute_driver.proto` path, so new remote drivers should enter -//! through named endpoint acquisition rather than gateway-wide socket side -//! channels. Once we have a generalized compute-driver registry, the remaining -//! per-arm wiring here should collapse to driver construction records that -//! produce either an in-process `SharedComputeDriver` or an acquired remote -//! endpoint, then hand the rest of the gateway a uniform [`ComputeRuntime`]. -//! The VM launch plumbing now lives in [`compute::vm`]; keep this file limited -//! to selecting and acquiring drivers. +//! Compiled-in compute drivers are installed into a registry at gateway +//! startup. Runtime selection only consults that registry or a configured +//! external endpoint; it does not switch on driver names. mod auth; pub mod certgen; @@ -58,8 +48,10 @@ mod tracing_setup; mod ws_tunnel; use metrics_exporter_prometheus::PrometheusBuilder; +#[cfg(target_os = "windows")] +use openshell_core::ComputeDriverKind; use openshell_core::net::set_tcp_nodelay_best_effort; -use openshell_core::{ComputeDriverKind, Config, Error, ObjectLabels, Result}; +use openshell_core::{Config, Error, ObjectLabels, Result}; use openshell_extension_core::{ BearerTokenSlot, ExtensionAudience, ExtensionCallerKind, ExtensionKind, MAX_EXTENSION_TOKEN_TTL, }; @@ -67,6 +59,7 @@ use openshell_supervisor_middleware::MiddlewareRegistry; use std::collections::{BTreeMap, HashMap}; use std::io::ErrorKind; use std::net::SocketAddr; +use std::path::Path; #[cfg(test)] use std::sync::LazyLock; use std::sync::{Arc, Mutex}; @@ -438,6 +431,7 @@ impl ServerState { pub(crate) async fn run_server( startup: ServerStartupConfig, tracing_log_bus: TracingLogBus, + compute_drivers: ComputeDriverRegistry, ) -> Result<()> { let ServerStartupConfig { config, @@ -591,6 +585,7 @@ pub(crate) async fn run_server( endpoint_overrides: &config.compute_driver_endpoints, }; let (compute, operator_allowlist) = build_compute_runtime( + &compute_drivers, &config, driver_startup, store.clone(), @@ -1079,13 +1074,405 @@ fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::Com )) } -// Internal wiring helper: each argument is a distinct piece of runtime state -// that must be passed through, so the count is justified. -#[allow(clippy::too_many_arguments)] type OperatorAllowlistArc = Option; +pub use compute::{DriverWatchStream, SharedComputeDriver}; + +/// Opaque result returned by a compiled compute-driver factory. +pub struct ComputeDriverBuildOutput { + runtime: ComputeRuntime, + operator_allowlist: OperatorAllowlistArc, +} + +/// Factory for a compute driver linked into a gateway binary. +#[async_trait::async_trait] +pub trait ComputeDriverFactory: Send + Sync { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result; +} + +/// One named compiled-driver registration. +#[derive(Clone)] +pub struct ComputeDriverRegistration { + name: String, + detection_priority: u16, + detect: Option bool>, + factory: Arc, +} + +impl std::fmt::Debug for ComputeDriverRegistration { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ComputeDriverRegistration") + .field("name", &self.name) + .field("detection_priority", &self.detection_priority) + .field("has_detection_probe", &self.detect.is_some()) + .finish_non_exhaustive() + } +} + +impl ComputeDriverRegistration { + /// Define a compiled driver. Lower detection priorities are preferred. + pub fn new( + name: impl Into, + detection_priority: u16, + detect: Option bool>, + factory: impl ComputeDriverFactory + 'static, + ) -> Result { + let name = openshell_core::config::normalize_compute_driver_name(&name.into()) + .map_err(Error::config)?; + Ok(Self { + name, + detection_priority, + detect, + factory: Arc::new(factory), + }) + } +} + +/// Registry of compute drivers compiled into this gateway binary. +/// +/// Like `SQLx`'s `Any` driver registry, installation is explicit at the binary +/// composition boundary while runtime selection is generic. +#[derive(Clone, Default)] +pub struct ComputeDriverRegistry { + drivers: BTreeMap, +} + +impl ComputeDriverRegistry { + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Install a compiled driver factory. + pub fn install(&mut self, registration: ComputeDriverRegistration) -> Result<()> { + let name = registration.name.clone(); + match self.drivers.entry(name.clone()) { + std::collections::btree_map::Entry::Vacant(entry) => { + entry.insert(registration); + Ok(()) + } + std::collections::btree_map::Entry::Occupied(_) => Err(Error::config(format!( + "compute driver '{name}' registered twice" + ))), + } + } + + /// Names installed into this gateway binary, in lexical order. + pub fn installed_driver_names(&self) -> impl Iterator { + self.drivers.keys().map(String::as_str) + } + + fn get(&self, name: &str) -> Option<&ComputeDriverRegistration> { + self.drivers.get(name) + } + + fn detect(&self) -> Option<&ComputeDriverRegistration> { + self.drivers + .values() + .filter(|registration| registration.detect.is_some_and(|detect| detect())) + .min_by_key(|registration| registration.detection_priority) + } +} + +/// Install every first-party compute driver linked into the standard gateway. +#[must_use] +pub fn install_default_compute_drivers() -> ComputeDriverRegistry { + let mut registry = ComputeDriverRegistry::new(); + #[cfg(not(target_os = "windows"))] + { + registry + .install( + ComputeDriverRegistration::new( + "kubernetes", + 100, + Some(|| std::env::var_os("KUBERNETES_SERVICE_HOST").is_some()), + KubernetesComputeDriverFactory, + ) + .expect("valid kubernetes registration"), + ) + .expect("unique kubernetes registration"); + registry + .install( + ComputeDriverRegistration::new( + "podman", + 200, + Some(openshell_core::config::is_podman_available), + PodmanComputeDriverFactory, + ) + .expect("valid podman registration"), + ) + .expect("unique podman registration"); + registry + .install( + ComputeDriverRegistration::new( + "docker", + 300, + Some(openshell_core::config::is_docker_available), + DockerComputeDriverFactory, + ) + .expect("valid docker registration"), + ) + .expect("unique docker registration"); + registry + .install( + ComputeDriverRegistration::new("vm", u16::MAX, None, VmComputeDriverFactory) + .expect("valid vm registration"), + ) + .expect("unique vm registration"); + } + #[cfg(target_os = "windows")] + for name in ["kubernetes", "podman", "docker", "vm"] { + registry + .install( + ComputeDriverRegistration::new( + name, + u16::MAX, + None, + UnsupportedComputeDriverFactory, + ) + .expect("valid unsupported registration"), + ) + .expect("unique unsupported registration"); + } + registry +} + +pub struct ComputeDriverBuildContext<'a> { + driver_name: String, + config: &'a Config, + driver_startup: compute::driver_config::DriverStartupContext<'a>, + store: Arc, + sandbox_index: SandboxIndex, + sandbox_watch_bus: SandboxWatchBus, + tracing_log_bus: TracingLogBus, + supervisor_sessions: Arc, + shutdown_rx: watch::Receiver, +} + +impl ComputeDriverBuildContext<'_> { + #[must_use] + pub fn driver_name(&self) -> &str { + &self.driver_name + } + + #[must_use] + pub fn gateway_config(&self) -> &Config { + self.config + } + + #[must_use] + pub fn gateway_port(&self) -> u16 { + self.driver_startup.gateway_port + } + + #[must_use] + pub fn gateway_tls_enabled(&self) -> bool { + self.driver_startup.gateway_tls_enabled + } + + /// Gateway client credentials that a local driver may mount into guests. + #[must_use] + pub fn guest_tls_paths(&self) -> Option<(&Path, &Path, &Path)> { + self.driver_startup + .guest_tls + .map(compute::driver_config::GuestTlsPaths::as_paths) + } + + /// Deserialize the selected driver's merged TOML table. + pub fn driver_config(&self) -> Result + where + T: Default + serde::de::DeserializeOwned, + { + compute::driver_config::driver_config_from_context(self.driver_startup, &self.driver_name) + } + + #[must_use] + pub fn shutdown_receiver(&self) -> watch::Receiver { + self.shutdown_rx.clone() + } + + /// Finish construction of an in-process driver through the common runtime path. + pub async fn finish_in_process( + self, + driver: SharedComputeDriver, + ) -> Result { + let runtime = ComputeRuntime::from_driver( + self.driver_name, + driver, + None, + self.store, + self.sandbox_index, + self.sandbox_watch_bus, + self.tracing_log_bus, + self.supervisor_sessions, + ) + .await + .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; + Ok(ComputeDriverBuildOutput { + runtime, + operator_allowlist: None, + }) + } +} + +#[cfg(target_os = "windows")] +#[derive(Clone, Copy)] +struct UnsupportedComputeDriverFactory; + +#[cfg(target_os = "windows")] +#[async_trait::async_trait] +impl ComputeDriverFactory for UnsupportedComputeDriverFactory { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result { + Err(Error::execution( + unsupported_builtin_compute_driver( + context + .driver_name + .parse() + .expect("default driver names are valid"), + ) + .to_string(), + )) + } +} + +#[cfg(not(target_os = "windows"))] +#[derive(Clone, Copy)] +struct KubernetesComputeDriverFactory; + +#[cfg(not(target_os = "windows"))] +#[async_trait::async_trait] +impl ComputeDriverFactory for KubernetesComputeDriverFactory { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result { + warn_if_kubernetes_sandbox_jwt_expiry_disabled(context.config); + let config = compute::driver_config::builtin::kubernetes_config_from_context( + context.driver_startup, + )?; + let (runtime, operator_allowlist) = ComputeRuntime::new_kubernetes( + config, + context.store, + context.sandbox_index, + context.sandbox_watch_bus, + context.tracing_log_bus, + context.supervisor_sessions, + context.shutdown_rx, + ) + .await + .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; + Ok(ComputeDriverBuildOutput { + runtime, + operator_allowlist, + }) + } +} + +#[cfg(not(target_os = "windows"))] +#[derive(Clone, Copy)] +struct DockerComputeDriverFactory; + +#[cfg(not(target_os = "windows"))] +#[async_trait::async_trait] +impl ComputeDriverFactory for DockerComputeDriverFactory { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result { + let driver_config = + compute::driver_config::builtin::docker_config_from_context(context.driver_startup)?; + let runtime = ComputeRuntime::new_docker( + context.config.clone(), + driver_config, + context.store, + context.sandbox_index, + context.sandbox_watch_bus, + context.tracing_log_bus, + context.supervisor_sessions, + ) + .await + .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; + Ok(ComputeDriverBuildOutput { + runtime, + operator_allowlist: None, + }) + } +} + +#[cfg(not(target_os = "windows"))] +#[derive(Clone, Copy)] +struct PodmanComputeDriverFactory; + +#[cfg(not(target_os = "windows"))] +#[async_trait::async_trait] +impl ComputeDriverFactory for PodmanComputeDriverFactory { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result { + let driver_config = + compute::driver_config::builtin::podman_config_from_context(context.driver_startup)?; + let runtime = ComputeRuntime::new_podman( + driver_config, + context.store, + context.sandbox_index, + context.sandbox_watch_bus, + context.tracing_log_bus, + context.supervisor_sessions, + ) + .await + .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; + Ok(ComputeDriverBuildOutput { + runtime, + operator_allowlist: None, + }) + } +} + +#[cfg(not(target_os = "windows"))] +#[derive(Clone, Copy)] +struct VmComputeDriverFactory; + +#[cfg(not(target_os = "windows"))] +#[async_trait::async_trait] +impl ComputeDriverFactory for VmComputeDriverFactory { + async fn build( + &self, + context: ComputeDriverBuildContext<'_>, + ) -> Result { + let driver_config = + compute::driver_config::builtin::vm_config_from_context(context.driver_startup)?; + let otlp_config = context + .driver_startup + .file + .and_then(|file| file.openshell.gateway.otlp.as_ref()); + let endpoint = compute::vm::spawn(context.config, &driver_config, otlp_config).await?; + let runtime = ComputeRuntime::new_remote_driver( + endpoint, + context.store, + context.sandbox_index, + context.sandbox_watch_bus, + context.tracing_log_bus, + context.supervisor_sessions, + ) + .await + .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; + Ok(ComputeDriverBuildOutput { + runtime, + operator_allowlist: None, + }) + } +} #[allow(clippy::too_many_arguments)] async fn build_compute_runtime( + registry: &ComputeDriverRegistry, config: &Config, driver_startup: compute::driver_config::DriverStartupContext<'_>, store: Arc, @@ -1095,86 +1482,26 @@ async fn build_compute_runtime( supervisor_sessions: Arc, shutdown_rx: watch::Receiver, ) -> Result<(ComputeRuntime, OperatorAllowlistArc)> { - let driver = configured_compute_driver(config, driver_startup)?; + let driver = configured_compute_driver(registry, config, driver_startup)?; info!(driver = %driver.name(), "Using compute driver"); let (runtime, operator_allowlist) = match driver { - #[cfg(target_os = "windows")] - ConfiguredComputeDriver::Builtin(driver) => { - return Err(Error::execution( - unsupported_builtin_compute_driver(driver).to_string(), - )); - } - #[cfg(not(target_os = "windows"))] - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Kubernetes) => { - warn_if_kubernetes_sandbox_jwt_expiry_disabled(config); - let k8s_config = - compute::driver_config::builtin::kubernetes_config_from_context(driver_startup)?; - let (rt, allowlist) = ComputeRuntime::new_kubernetes( - k8s_config, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions.clone(), - shutdown_rx, - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - (rt, allowlist) - } - #[cfg(not(target_os = "windows"))] - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Docker) => { - let docker_config = - compute::driver_config::builtin::docker_config_from_context(driver_startup)?; - let rt = ComputeRuntime::new_docker( - config.clone(), - docker_config, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - (rt, None) - } - #[cfg(not(target_os = "windows"))] - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Podman) => { - let podman_config = - compute::driver_config::builtin::podman_config_from_context(driver_startup)?; - let rt = ComputeRuntime::new_podman( - podman_config, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - (rt, None) - } - #[cfg(not(target_os = "windows"))] - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Vm) => { - let vm_config = - compute::driver_config::builtin::vm_config_from_context(driver_startup)?; - let otlp_config = driver_startup - .file - .and_then(|file| file.openshell.gateway.otlp.as_ref()); - let endpoint = compute::vm::spawn(config, &vm_config, otlp_config).await?; - let rt = ComputeRuntime::new_remote_driver( - endpoint, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - (rt, None) + ConfiguredComputeDriver::Registered(registration) => { + let output = registration + .factory + .build(ComputeDriverBuildContext { + driver_name: registration.name, + config, + driver_startup, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions, + shutdown_rx, + }) + .await?; + (output.runtime, output.operator_allowlist) } ConfiguredComputeDriver::Remote { name } => { let remote_config = @@ -1206,35 +1533,35 @@ async fn build_compute_runtime( #[derive(Debug, Clone)] enum ConfiguredComputeDriver { - Builtin(ComputeDriverKind), + Registered(ComputeDriverRegistration), Remote { name: String }, } impl ConfiguredComputeDriver { fn name(&self) -> &str { match self { - Self::Builtin(kind) => kind.as_str(), + Self::Registered(registration) => ®istration.name, Self::Remote { name } => name, } } } fn configured_compute_driver( + registry: &ComputeDriverRegistry, config: &Config, driver_startup: compute::driver_config::DriverStartupContext<'_>, ) -> Result { match config.compute_drivers.as_slice() { - [] => match openshell_core::config::detect_driver() { - Some(ComputeDriverKind::Vm) => Err(Error::config( - "vm compute driver is opt-in only; set --drivers vm or OPENSHELL_DRIVERS=vm", - )), - Some(driver) => Ok(ConfiguredComputeDriver::Builtin(driver)), - None => Err(Error::config( - "no compute driver configured and auto-detection found no suitable driver; \ - set --drivers or OPENSHELL_DRIVERS to kubernetes, podman, docker, or vm", - )), - }, - [driver] => resolve_configured_compute_driver(driver, driver_startup), + [] => registry.detect().map_or_else( + || { + Err(Error::config( + "no compute driver configured and auto-detection found no suitable driver; \ + set --drivers or OPENSHELL_DRIVERS to kubernetes, podman, docker, or vm", + )) + }, + |registration| Ok(ConfiguredComputeDriver::Registered(registration.clone())), + ), + [driver] => resolve_configured_compute_driver(registry, driver, driver_startup), drivers => Err(Error::config(format!( "multiple compute drivers are not supported yet; configured drivers: {}", drivers.join(",") @@ -1243,27 +1570,23 @@ fn configured_compute_driver( } fn resolve_configured_compute_driver( + registry: &ComputeDriverRegistry, driver_name: &str, driver_startup: compute::driver_config::DriverStartupContext<'_>, ) -> Result { let name = openshell_core::config::normalize_compute_driver_name(driver_name) .map_err(Error::config)?; - let driver_kind = builtin_compute_driver(&name); if driver_startup.endpoint_overrides.contains_key(&name) { return Ok(ConfiguredComputeDriver::Remote { name }); } - if let Some(kind) = driver_kind { - return Ok(ConfiguredComputeDriver::Builtin(kind)); + if let Some(registration) = registry.get(&name) { + return Ok(ConfiguredComputeDriver::Registered(registration.clone())); } Ok(ConfiguredComputeDriver::Remote { name }) } -fn builtin_compute_driver(name: &str) -> Option { - name.parse().ok() -} - fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config) -> bool { config .gateway_jwt @@ -1471,6 +1794,23 @@ mod tests { } } + fn test_compute_drivers() -> super::ComputeDriverRegistry { + super::install_default_compute_drivers() + } + + #[derive(Clone, Copy)] + struct TestComputeDriverFactory; + + #[async_trait::async_trait] + impl super::ComputeDriverFactory for TestComputeDriverFactory { + async fn build( + &self, + _context: super::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + unreachable!("selection tests do not construct the driver") + } + } + fn test_tls_acceptor() -> (TempDir, TlsAcceptor) { install_rustls_provider(); @@ -1784,18 +2124,21 @@ mod tests { // Empty drivers triggers auto-detection, which may return Some or None // depending on the environment. This test verifies the auto-detection path // is taken rather than immediately returning an error. - let result = configured_compute_driver(&config, test_driver_startup(&config, None)); + let result = configured_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ); // Either we get a detected driver or an error about none being detected. match result { - Ok(ConfiguredComputeDriver::Builtin(driver)) => { + Ok(ConfiguredComputeDriver::Registered(registration)) => { assert!( matches!( - driver, - ComputeDriverKind::Kubernetes - | ComputeDriverKind::Docker - | ComputeDriverKind::Podman + registration.name.as_str(), + "kubernetes" | "docker" | "podman" ), - "auto-detected unexpected driver: {driver:?}" + "auto-detected unexpected driver: {}", + registration.name ); } Ok(ConfiguredComputeDriver::Remote { name }) => { @@ -1811,12 +2154,58 @@ mod tests { } } + #[test] + fn registry_detection_uses_registered_priorities() { + fn available() -> bool { + true + } + + let mut registry = super::ComputeDriverRegistry::new(); + registry + .install( + super::ComputeDriverRegistration::new( + "later", + 200, + Some(available), + TestComputeDriverFactory, + ) + .unwrap(), + ) + .unwrap(); + registry + .install( + super::ComputeDriverRegistration::new( + "earlier", + 100, + Some(available), + TestComputeDriverFactory, + ) + .unwrap(), + ) + .unwrap(); + + assert_eq!( + registry + .detect() + .map(|registration| registration.name.as_str()), + Some("earlier") + ); + assert_eq!( + registry.installed_driver_names().collect::>(), + vec!["earlier", "later"] + ); + } + #[test] fn configured_compute_driver_rejects_multiple_entries() { let config = Config::new(None) .with_compute_drivers([ComputeDriverKind::Kubernetes, ComputeDriverKind::Podman]); - let err = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap_err(); + let err = configured_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap_err(); assert!( err.to_string() .contains("multiple compute drivers are not supported yet") @@ -1827,33 +2216,45 @@ mod tests { #[test] fn configured_compute_driver_accepts_podman() { let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Podman]); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = configured_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Podman) + ConfiguredComputeDriver::Registered(registration) if registration.name == "podman" )); } #[test] fn configured_compute_driver_accepts_vm() { let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Vm]); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = configured_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Vm) + ConfiguredComputeDriver::Registered(registration) if registration.name == "vm" )); } #[test] fn configured_compute_driver_accepts_docker() { let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Docker]); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = configured_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Builtin(ComputeDriverKind::Docker) + ConfiguredComputeDriver::Registered(registration) if registration.name == "docker" )); } @@ -1861,15 +2262,22 @@ mod tests { fn configured_compute_driver_resolves_named_remote() { let config = Config::new(None).with_compute_drivers(["kyma"]); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = configured_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); match driver { ConfiguredComputeDriver::Remote { name } => { assert_eq!(name, "kyma"); } - ConfiguredComputeDriver::Builtin(other) => { - panic!("expected remote driver, got builtin driver {other:?}") + ConfiguredComputeDriver::Registered(other) => { + panic!( + "expected remote driver, got registered driver {}", + other.name + ) } } } @@ -1880,8 +2288,12 @@ mod tests { .with_compute_drivers([ComputeDriverKind::Vm]) .with_compute_driver_endpoint("vm", "/run/openshell/vm.sock"); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = configured_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); assert!(matches!( driver, ConfiguredComputeDriver::Remote { name } if name == "vm" @@ -1894,8 +2306,12 @@ mod tests { .with_compute_drivers([ComputeDriverKind::Docker]) .with_compute_driver_endpoint("docker", "/run/openshell/docker.sock"); - let driver = - configured_compute_driver(&config, test_driver_startup(&config, None)).unwrap(); + let driver = configured_compute_driver( + &test_compute_drivers(), + &config, + test_driver_startup(&config, None), + ) + .unwrap(); assert!(matches!( driver, ConfiguredComputeDriver::Remote { name } if name == "docker" diff --git a/crates/openshell-server/src/main.rs b/crates/openshell-server/src/main.rs index 0f33c685f4..c76761016d 100644 --- a/crates/openshell-server/src/main.rs +++ b/crates/openshell-server/src/main.rs @@ -7,5 +7,8 @@ use miette::Result; #[tokio::main] async fn main() -> Result<()> { - openshell_server::cli::run_cli().await + openshell_server::cli::run_cli_with_compute_drivers( + openshell_server::install_default_compute_drivers(), + ) + .await } From d8784b3948804df3275ca3b2c383f0b046bfaa36 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 18 Aug 2026 01:30:29 -0700 Subject: [PATCH 02/13] test(compute): exercise drivers as external binaries Signed-off-by: Drew Newberry --- .github/workflows/branch-e2e.yml | 18 ++- .github/workflows/e2e-test.yml | 7 +- Cargo.lock | 5 + architecture/compute-runtimes.md | 6 + crates/openshell-core/Cargo.toml | 1 + .../src/external_driver_socket.rs | 125 ++++++++++++++++++ crates/openshell-core/src/lib.rs | 4 + .../src/operator_namespace_allowlist.rs | 78 +++++++++++ crates/openshell-driver-docker/Cargo.toml | 10 +- crates/openshell-driver-docker/src/main.rs | 83 ++++++++++++ .../openshell-driver-kubernetes/src/config.rs | 89 +------------ crates/openshell-driver-kubernetes/src/lib.rs | 5 +- .../openshell-driver-kubernetes/src/main.rs | 41 ++++-- crates/openshell-driver-podman/src/main.rs | 62 +++++++-- crates/openshell-server/Cargo.toml | 15 ++- crates/openshell-server/src/auth/k8s_sa.rs | 2 +- .../src/compute/driver_config.rs | 106 ++++++++++++++- .../src/compute/driver_config/builtin.rs | 52 +------- crates/openshell-server/src/compute/mod.rs | 67 ++++++---- crates/openshell-server/src/lib.rs | 59 +++++---- .../openshell/templates/_gateway-workload.tpl | 47 +++++++ .../openshell/templates/gateway-config.yaml | 5 + .../Dockerfile.external-kubernetes-gateway | 16 +++ e2e/no-compute-driver-gateway.sh | 28 ++++ e2e/rust/e2e-vm.sh | 39 +++++- e2e/support/gateway-common.sh | 63 ++++++++- e2e/with-docker-gateway.sh | 69 ++++++++-- e2e/with-kube-gateway.sh | 46 ++++++- e2e/with-podman-gateway.sh | 37 ++++++ examples/governance-interceptor/Cargo.lock | 1 + tasks/test.toml | 24 ++++ 31 files changed, 972 insertions(+), 238 deletions(-) create mode 100644 crates/openshell-core/src/external_driver_socket.rs create mode 100644 crates/openshell-core/src/operator_namespace_allowlist.rs create mode 100644 crates/openshell-driver-docker/src/main.rs create mode 100644 e2e/docker/Dockerfile.external-kubernetes-gateway create mode 100755 e2e/no-compute-driver-gateway.sh diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index 37154c75df..2b4d9d5d46 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -196,6 +196,20 @@ jobs: e2e-task: e2e:kubernetes:workspace-managed cli-artifact-prefix: rust-binary-cli + kubernetes-external-driver-e2e: + needs: [pr_metadata, build-gateway, build-supervisor, build-cli] + if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' + permissions: + actions: read + contents: read + packages: read + uses: ./.github/workflows/e2e-kubernetes-test.yml + with: + image-tag: ${{ github.sha }} + job-name: Kubernetes E2E (external compute driver) + e2e-task: e2e:kubernetes:external-driver + cli-artifact-prefix: rust-binary-cli + kubernetes-workspace-operator-e2e: needs: [pr_metadata, build-gateway, build-supervisor, build-cli] if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' @@ -240,7 +254,7 @@ jobs: core-e2e-result: name: Core E2E result - needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] + needs: [pr_metadata, build-gateway, build-supervisor, build-cli, build-driver-vm-linux, e2e, kubernetes-e2e, kubernetes-external-driver-e2e, kubernetes-workspace-managed-e2e, kubernetes-workspace-operator-e2e] if: always() && needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true' runs-on: ubuntu-latest steps: @@ -252,6 +266,7 @@ jobs: BUILD_DRIVER_VM_RESULT: ${{ needs.build-driver-vm-linux.result }} E2E_RESULT: ${{ needs.e2e.result }} KUBERNETES_E2E_RESULT: ${{ needs.kubernetes-e2e.result }} + KUBERNETES_EXTERNAL_DRIVER_E2E_RESULT: ${{ needs.kubernetes-external-driver-e2e.result }} KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT: ${{ needs.kubernetes-workspace-managed-e2e.result }} KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT: ${{ needs.kubernetes-workspace-operator-e2e.result }} run: | @@ -264,6 +279,7 @@ jobs: "build-driver-vm-linux:$BUILD_DRIVER_VM_RESULT" \ "e2e:$E2E_RESULT" \ "kubernetes-e2e:$KUBERNETES_E2E_RESULT" \ + "kubernetes-external-driver-e2e:$KUBERNETES_EXTERNAL_DRIVER_E2E_RESULT" \ "kubernetes-workspace-managed-e2e:$KUBERNETES_WORKSPACE_MANAGED_E2E_RESULT" \ "kubernetes-workspace-operator-e2e:$KUBERNETES_WORKSPACE_OPERATOR_E2E_RESULT"; do name="${item%%:*}" diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c123790580..9fdc5d698b 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -59,6 +59,9 @@ jobs: - suite: rust-docker cmd: "mise run --no-deps --skip-deps e2e:rust" apt_packages: "openssh-client" + - suite: rust-docker-external-driver + cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:docker:external-driver" + apt_packages: "openssh-client" - suite: mcp cmd: "mise run --no-deps --skip-deps e2e:mcp" apt_packages: "" @@ -262,7 +265,7 @@ jobs: run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin - name: Run rootless Podman E2E - run: mise run --no-deps --skip-deps e2e:podman:rootless + run: env -u OPENSHELL_GATEWAY_BIN OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER=1 mise run --no-deps --skip-deps e2e:podman:rootless - name: Print AppArmor denials if: always() @@ -362,4 +365,4 @@ jobs: cache-on-failure: "true" - name: Run VM E2E - run: mise run --no-deps --skip-deps e2e:vm + run: env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:vm:external-driver diff --git a/Cargo.lock b/Cargo.lock index c30f890914..40291d8cf5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3717,6 +3717,7 @@ dependencies = [ "prost-types", "protoc-bin-vendored", "reqwest 0.12.28", + "rustix 1.1.4", "serde", "serde_json", "tar", @@ -3755,7 +3756,9 @@ version = "0.0.0" dependencies = [ "bollard", "bytes", + "clap", "futures", + "miette", "openshell-core", "prost-types", "serde", @@ -3765,8 +3768,10 @@ dependencies = [ "tempfile", "tokio", "tokio-stream", + "toml", "tonic", "tracing", + "tracing-subscriber", "url", ] diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 3a8a5b709b..022db31a1f 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -108,6 +108,12 @@ to `run_cli_with_compute_drivers`; factories receive merged driver config and finish through the same in-process runtime adapter. A configured UDS endpoint still takes precedence over a compiled registration with the same name. +The standard server crate groups first-party registrations behind the +`in-tree-compute-drivers` feature. Protocol-only gateway builds disable that +feature and link no compute-driver crates. E2E lanes compose that gateway with +Docker, Podman, Kubernetes, and VM driver executables over the public UDS gRPC +contract so an in-tree driver cannot silently depend on a server-only API. + ## Stop and Start Lifecycle The gateway persists lifecycle intent before mutating compute: diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index 8b28fafa23..0e2ad24ff0 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -35,6 +35,7 @@ tempfile = { version = "3", optional = true } [target.'cfg(unix)'.dependencies] nix = { workspace = true } +rustix = { workspace = true } [features] default = ["telemetry"] diff --git a/crates/openshell-core/src/external_driver_socket.rs b/crates/openshell-core/src/external_driver_socket.rs new file mode 100644 index 0000000000..d554d5cd34 --- /dev/null +++ b/crates/openshell-core/src/external_driver_socket.rs @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Public Unix-socket transport helpers for out-of-process drivers. + +use std::io; +use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use tokio::net::{UnixListener, UnixStream}; +use tokio_stream::Stream; + +/// Prepare and bind a private Unix socket owned by the current effective UID. +pub fn bind_private(path: &Path) -> Result { + let parent = path + .parent() + .ok_or_else(|| format!("driver socket path '{}' has no parent", path.display()))?; + let expected_uid = rustix::process::geteuid().as_raw(); + std::fs::create_dir_all(parent) + .map_err(|err| format!("create socket directory {}: {err}", parent.display()))?; + let parent_metadata = std::fs::symlink_metadata(parent) + .map_err(|err| format!("stat socket directory {}: {err}", parent.display()))?; + if parent_metadata.file_type().is_symlink() || !parent_metadata.file_type().is_dir() { + return Err(format!( + "driver socket parent '{}' must be a directory, not a symlink", + parent.display() + )); + } + if parent_metadata.uid() != expected_uid { + return Err(format!( + "driver socket parent '{}' is owned by uid {}, expected {}", + parent.display(), + parent_metadata.uid(), + expected_uid + )); + } + std::fs::set_permissions(parent, std::fs::Permissions::from_mode(0o700)) + .map_err(|err| format!("chmod socket directory {}: {err}", parent.display()))?; + + match std::fs::symlink_metadata(path) { + Ok(metadata) + if metadata.file_type().is_socket() + && !metadata.file_type().is_symlink() + && metadata.uid() == expected_uid => + { + std::fs::remove_file(path) + .map_err(|err| format!("remove stale socket {}: {err}", path.display()))?; + } + Ok(_) => { + return Err(format!( + "driver socket path '{}' exists but is not an owned Unix socket", + path.display() + )); + } + Err(err) if err.kind() == io::ErrorKind::NotFound => {} + Err(err) => return Err(format!("stat driver socket {}: {err}", path.display())), + } + + let listener = UnixListener::bind(path) + .map_err(|err| format!("bind driver socket {}: {err}", path.display()))?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .map_err(|err| format!("chmod driver socket {}: {err}", path.display()))?; + Ok(listener) +} + +/// Remove a socket created by [`bind_private`]. +pub struct SocketCleanup(PathBuf); + +impl SocketCleanup { + #[must_use] + pub fn new(path: PathBuf) -> Self { + Self(path) + } +} + +impl Drop for SocketCleanup { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +/// Incoming UDS connections restricted to the driver's effective UID. +pub struct SameUidUnixIncoming { + listener: UnixListener, + expected_uid: u32, +} + +impl SameUidUnixIncoming { + #[must_use] + pub fn new(listener: UnixListener) -> Self { + Self { + listener, + expected_uid: rustix::process::geteuid().as_raw(), + } + } +} + +impl Stream for SameUidUnixIncoming { + type Item = io::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + match this.listener.poll_accept(cx) { + Poll::Ready(Ok((stream, _))) => match stream.peer_cred() { + Ok(credentials) if credentials.uid() == this.expected_uid => { + return Poll::Ready(Some(Ok(stream))); + } + Ok(credentials) => tracing::warn!( + peer_uid = credentials.uid(), + expected_uid = this.expected_uid, + "rejected external driver socket client" + ), + Err(err) => { + tracing::warn!(error = %err, "failed to authenticate driver socket client"); + } + }, + Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))), + Poll::Pending => return Poll::Pending, + } + } + } +} diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index d373d656ed..96be19e1e2 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -18,6 +18,8 @@ pub mod driver_mounts; pub mod driver_utils; pub mod endpoint_path; pub mod error; +#[cfg(unix)] +pub mod external_driver_socket; pub mod forward; pub mod google_cloud; pub mod gpu; @@ -29,6 +31,7 @@ pub mod jwt; pub mod metadata; pub mod middleware; pub mod net; +pub mod operator_namespace_allowlist; pub mod paths; pub mod policy; pub mod progress; @@ -53,6 +56,7 @@ pub use error::{ComputeDriverError, Error, Result}; pub use metadata::{ GetResourceVersion, ObjectId, ObjectLabels, ObjectName, ObjectWorkspace, SetResourceVersion, }; +pub use operator_namespace_allowlist::OperatorNamespaceAllowlist; /// Build version string derived from git metadata. /// diff --git a/crates/openshell-core/src/operator_namespace_allowlist.rs b/crates/openshell-core/src/operator_namespace_allowlist.rs new file mode 100644 index 0000000000..c8f0f7f3de --- /dev/null +++ b/crates/openshell-core/src/operator_namespace_allowlist.rs @@ -0,0 +1,78 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::BTreeSet; +use std::sync::{Arc, RwLock}; + +/// Thread-safe dynamic allowlist of Kubernetes operator-mode namespaces. +/// +/// This type lives in the public core API because both the Kubernetes driver +/// and gateway authentication boundary consume it. +#[derive(Debug, Clone)] +pub struct OperatorNamespaceAllowlist { + inner: Arc>>, +} + +impl OperatorNamespaceAllowlist { + fn read_guard(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { + self.inner + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn write_guard(&self) -> std::sync::RwLockWriteGuard<'_, BTreeSet> { + self.inner + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + #[must_use] + pub fn new() -> Self { + Self { + inner: Arc::new(RwLock::new(BTreeSet::new())), + } + } + + #[must_use] + pub fn from_set(set: BTreeSet) -> Self { + Self { + inner: Arc::new(RwLock::new(set)), + } + } + + pub fn replace(&self, new_set: BTreeSet) { + *self.write_guard() = new_set; + } + + pub fn merge(&self, additional: &BTreeSet) { + self.write_guard().extend(additional.iter().cloned()); + } + + pub fn read(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { + self.read_guard() + } + + #[must_use] + pub fn contains(&self, namespace: &str) -> bool { + self.read_guard().contains(namespace) + } + + pub fn insert(&self, name: String) -> bool { + self.write_guard().insert(name) + } + + pub fn remove(&self, name: &str) -> bool { + self.write_guard().remove(name) + } + + #[must_use] + pub fn shared(&self) -> Arc>> { + Arc::clone(&self.inner) + } +} + +impl Default for OperatorNamespaceAllowlist { + fn default() -> Self { + Self::new() + } +} diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 92a32c6d20..065b3ff4b6 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -10,11 +10,15 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[[bin]] +name = "openshell-driver-docker" +path = "src/main.rs" + [dependencies] openshell-core = { path = "../openshell-core", default-features = false, features = ["driver-extraction"] } tokio = { workspace = true } -tonic = { workspace = true } +tonic = { workspace = true, features = ["transport"] } futures = { workspace = true } tokio-stream = { workspace = true } tracing = { workspace = true } @@ -24,6 +28,10 @@ serde_json = { workspace = true } prost-types = { workspace = true } bollard = { version = "0.20" } url = { workspace = true } +clap = { workspace = true } +miette = { workspace = true } +toml = { workspace = true } +tracing-subscriber = { workspace = true } [dev-dependencies] prost-types = { workspace = true } diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs new file mode 100644 index 0000000000..3c539ba11b --- /dev/null +++ b/crates/openshell-driver-docker/src/main.rs @@ -0,0 +1,83 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::net::SocketAddr; +use std::path::PathBuf; + +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_core::{Config, VERSION}; +use openshell_driver_docker::{DockerComputeConfig, DockerComputeDriver}; +use tracing::info; +use tracing_subscriber::EnvFilter; + +#[derive(Debug, Parser)] +#[command(name = "openshell-driver-docker", version = VERSION)] +struct Args { + /// Public compute-driver Unix socket used by the gateway. + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: PathBuf, + + /// TOML file containing a serialized `DockerComputeConfig` table. + #[arg(long, env = "OPENSHELL_DOCKER_DRIVER_CONFIG")] + config: PathBuf, + + /// Gateway listener address used to derive sandbox callback routing. + #[arg( + long, + env = "OPENSHELL_GATEWAY_BIND", + default_value = "127.0.0.1:50051" + )] + gateway_bind: SocketAddr, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, +} + +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let config_source = std::fs::read_to_string(&args.config).into_diagnostic()?; + let docker_config: DockerComputeConfig = toml::from_str(&config_source).into_diagnostic()?; + let gateway_config = Config::new(None).with_bind_address(args.gateway_bind); + let driver = DockerComputeDriver::new(&gateway_config, &docker_config) + .await + .into_diagnostic()?; + + let listener = openshell_core::external_driver_socket::bind_private(&args.bind_socket) + .map_err(|err| miette::miette!("{err}"))?; + let _cleanup = + openshell_core::external_driver_socket::SocketCleanup::new(args.bind_socket.clone()); + info!(socket = %args.bind_socket.display(), "Starting Docker compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(driver)) + .serve_with_incoming_shutdown( + openshell_core::external_driver_socket::SameUidUnixIncoming::new(listener), + shutdown_signal(), + ) + .await + .into_diagnostic() +} + +async fn shutdown_signal() { + let terminate = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut signal) => { + signal.recv().await; + } + Err(_) => std::future::pending::<()>().await, + } + }; + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + () = terminate => {} + } + info!("Received shutdown signal, draining in-flight requests"); +} diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index bfa08a6642..217fec09ec 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1,12 +1,14 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +pub use openshell_core::OperatorNamespaceAllowlist; use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::BTreeMap; +#[cfg(test)] +use std::collections::BTreeSet; use std::path::Path; use std::str::FromStr; -use std::sync::{Arc, RwLock}; /// Default gateway identity used in managed-mode namespace naming. pub const DEFAULT_GATEWAY_ID: &str = "openshell"; @@ -838,89 +840,6 @@ pub fn validate_managed_namespace_name(gateway_id: &str, workspace: &str) -> Res Ok(()) } -/// Thread-safe dynamic allowlist of valid operator-mode namespaces. -/// -/// Backed by an `Arc>>` that is updated by background -/// tasks (label selector watcher, drop-in file watcher) and read by the SA -/// authenticator and namespace resolver. -#[derive(Debug, Clone)] -pub struct OperatorNamespaceAllowlist { - inner: Arc>>, -} - -impl OperatorNamespaceAllowlist { - fn read_guard(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { - self.inner - .read() - .unwrap_or_else(std::sync::PoisonError::into_inner) - } - - fn write_guard(&self) -> std::sync::RwLockWriteGuard<'_, BTreeSet> { - self.inner - .write() - .unwrap_or_else(std::sync::PoisonError::into_inner) - } - - #[must_use] - pub fn new() -> Self { - Self { - inner: Arc::new(RwLock::new(BTreeSet::new())), - } - } - - #[must_use] - pub fn from_set(set: BTreeSet) -> Self { - Self { - inner: Arc::new(RwLock::new(set)), - } - } - - /// Replace the entire allowlist (used by background watchers on refresh). - pub fn replace(&self, new_set: BTreeSet) { - let mut guard = self.write_guard(); - *guard = new_set; - } - - /// Merge additional namespaces into the allowlist. - pub fn merge(&self, additional: &BTreeSet) { - let mut guard = self.write_guard(); - guard.extend(additional.iter().cloned()); - } - - /// Read the current allowlist snapshot. - pub fn read(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { - self.read_guard() - } - - /// Check whether a namespace is in the allowlist. - #[must_use] - pub fn contains(&self, namespace: &str) -> bool { - self.read_guard().contains(namespace) - } - - /// Insert a namespace into the allowlist. Returns `true` if it was new. - pub fn insert(&self, name: String) -> bool { - self.write_guard().insert(name) - } - - /// Remove a namespace from the allowlist. Returns `true` if it was present. - pub fn remove(&self, name: &str) -> bool { - self.write_guard().remove(name) - } - - /// Return a clone of the inner `Arc` for sharing with background tasks. - #[must_use] - pub fn shared(&self) -> Arc>> { - Arc::clone(&self.inner) - } -} - -impl Default for OperatorNamespaceAllowlist { - fn default() -> Self { - Self::new() - } -} - fn is_dns1123_subdomain(value: &str) -> bool { !value.is_empty() && value.len() <= 253 diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index 1a234385c6..d69f9749a1 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -8,8 +8,9 @@ pub mod grpc; pub use config::{ AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig, - ManagedSshIngressConfig, OperatorNamespaceAllowlist, SupervisorSideloadMethod, - SupervisorTopology, WorkspaceMode, managed_namespace_prefix, + ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode, + managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; +pub use openshell_core::OperatorNamespaceAllowlist; diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 30b4fcada9..3a805c8685 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -5,6 +5,7 @@ use clap::{ArgAction, Parser}; use miette::{IntoDiagnostic, Result}; use std::collections::BTreeMap; use std::net::SocketAddr; +use std::path::PathBuf; use tracing::info; use tracing_subscriber::EnvFilter; @@ -22,6 +23,10 @@ use openshell_driver_kubernetes::{ #[command(version = VERSION)] #[allow(clippy::struct_excessive_bools)] struct Args { + /// Public compute-driver Unix socket used by an external gateway. + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: Option, + #[arg( long, env = "OPENSHELL_COMPUTE_DRIVER_BIND", @@ -286,13 +291,31 @@ async fn main() -> Result<()> { .await .into_diagnostic()?; - info!(address = %args.bind_address, "Starting Kubernetes compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve_with_shutdown(args.bind_address, async move { - shutdown_signal().await; - let _ = shutdown_tx.send(true); - }) - .await - .into_diagnostic() + let service = ComputeDriverServer::new(ComputeDriverService::new(driver)); + let shutdown = async move { + shutdown_signal().await; + let _ = shutdown_tx.send(true); + }; + if let Some(socket_path) = args.bind_socket { + let listener = openshell_core::external_driver_socket::bind_private(&socket_path) + .map_err(|err| miette::miette!("{err}"))?; + let _cleanup = + openshell_core::external_driver_socket::SocketCleanup::new(socket_path.clone()); + info!(socket = %socket_path.display(), "Starting Kubernetes compute driver"); + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming_shutdown( + openshell_core::external_driver_socket::SameUidUnixIncoming::new(listener), + shutdown, + ) + .await + .into_diagnostic() + } else { + info!(address = %args.bind_address, "Starting Kubernetes compute driver"); + tonic::transport::Server::builder() + .add_service(service) + .serve_with_shutdown(args.bind_address, shutdown) + .await + .into_diagnostic() + } } diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 405deb93a6..07aa09df3a 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -20,6 +20,10 @@ use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanC #[command(name = "openshell-driver-podman")] #[command(version = VERSION)] struct Args { + /// Public compute-driver Unix socket used by an external gateway. + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: Option, + #[arg( long, env = "OPENSHELL_COMPUTE_DRIVER_BIND", @@ -148,6 +152,10 @@ struct Args { /// Each entry is `"container_id:host_id:size"`. #[arg(long = "gidmap")] gidmap: Vec, + + /// Allow sandbox requests to attach host bind mounts. + #[arg(long, env = "OPENSHELL_ENABLE_BIND_MOUNTS", default_value_t = false)] + enable_bind_mounts: bool, } #[tokio::main] @@ -186,18 +194,54 @@ async fn main() -> Result<()> { userns: args.userns, uidmap: args.uidmap, gidmap: args.gidmap, + enable_bind_mounts: args.enable_bind_mounts, ..PodmanComputeConfig::default() }) .await .into_diagnostic()?; - info!(address = %args.bind_address, "Starting Podman compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve_with_shutdown(args.bind_address, async { - tokio::signal::ctrl_c().await.ok(); - info!("Received shutdown signal, draining in-flight requests"); - }) - .await - .into_diagnostic() + let service = ComputeDriverServer::new(ComputeDriverService::new(driver)); + if let Some(socket_path) = args.bind_socket { + let listener = openshell_core::external_driver_socket::bind_private(&socket_path) + .map_err(|err| miette::miette!("{err}"))?; + let _cleanup = + openshell_core::external_driver_socket::SocketCleanup::new(socket_path.clone()); + info!(socket = %socket_path.display(), "Starting Podman compute driver"); + tonic::transport::Server::builder() + .add_service(service) + .serve_with_incoming_shutdown( + openshell_core::external_driver_socket::SameUidUnixIncoming::new(listener), + shutdown_signal(), + ) + .await + .into_diagnostic() + } else { + info!(address = %args.bind_address, "Starting Podman compute driver"); + tonic::transport::Server::builder() + .add_service(service) + .serve_with_shutdown(args.bind_address, shutdown_signal()) + .await + .into_diagnostic() + } +} + +async fn shutdown_signal() { + #[cfg(unix)] + { + let terminate = async { + match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) { + Ok(mut signal) => { + signal.recv().await; + } + Err(_) => std::future::pending::<()>().await, + } + }; + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + () = terminate => {} + } + } + #[cfg(not(unix))] + let _ = tokio::signal::ctrl_c().await; + info!("Received shutdown signal, draining in-flight requests"); } diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 772590d1b0..215f6abfa4 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -115,12 +115,19 @@ arc-swap = "1" notify = "8" [target.'cfg(not(target_os = "windows"))'.dependencies] -openshell-driver-docker = { path = "../openshell-driver-docker" } -openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes" } -openshell-driver-podman = { path = "../openshell-driver-podman" } +openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } +openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes", optional = true } +openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } [features] -default = ["telemetry"] +default = ["telemetry", "in-tree-compute-drivers"] +## Link the first-party compute drivers into the standard gateway binary. +## Disable this feature for a protocol-only gateway that uses external drivers. +in-tree-compute-drivers = [ + "dep:openshell-driver-docker", + "dep:openshell-driver-kubernetes", + "dep:openshell-driver-podman", +] ## Compile in anonymous telemetry emission (forwards to openshell-core/telemetry). ## On by default; build with `--no-default-features` for a telemetry-free gateway ## that contains no telemetry endpoint, HTTP client, or emission code. diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index 32cb2e119c..131dbaba47 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -26,7 +26,7 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; -use openshell_driver_kubernetes::OperatorNamespaceAllowlist; +use openshell_core::OperatorNamespaceAllowlist; use std::sync::Arc; use tonic::Status; use tracing::{debug, info, warn}; diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index f0eb6f98b4..fad71efd36 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -7,7 +7,7 @@ //! driver-specific environment overrides, and applying gateway startup defaults. //! It does not acquire, connect to, or start compute drivers. -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub mod builtin; use crate::config_file; @@ -53,19 +53,73 @@ pub fn remote_driver_config_from_context( context: DriverStartupContext<'_>, name: &str, ) -> Result { - let mut cfg = driver_config_from_context(context, name)?; + let mut cfg = RemoteDriverConfig::default(); + if let Some(file) = context.file { + let merged = config_file::driver_table( + name, + &file.openshell.gateway, + file.openshell.drivers.get(name), + ); + if let Some(socket_path) = merged.get("socket_path").and_then(toml::Value::as_str) { + cfg.socket_path = PathBuf::from(socket_path); + } + } apply_remote_driver_overrides(&mut cfg, context, name); validate_remote_driver_config(&cfg, name)?; Ok(cfg) } #[derive(Debug, Clone, Default, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] pub struct RemoteDriverConfig { #[serde(default)] pub socket_path: PathBuf, } +#[derive(Debug, Clone, Deserialize)] +#[serde(default)] +pub struct KubernetesSaBootstrapConfig { + pub namespace: String, + pub service_account_name: String, + pub workspace_mode: String, + pub gateway_id: String, +} + +impl Default for KubernetesSaBootstrapConfig { + fn default() -> Self { + Self { + namespace: "openshell".to_string(), + service_account_name: "default".to_string(), + workspace_mode: "shared".to_string(), + gateway_id: "openshell".to_string(), + } + } +} + +pub fn kubernetes_sa_bootstrap_config( + file: Option<&config_file::ConfigFile>, +) -> Result { + let Some(file) = file else { + return Err(Error::config( + "K8s ServiceAccount bootstrap requires [openshell.drivers.kubernetes] when sandbox JWT issuing is enabled in-cluster", + )); + }; + if !file.openshell.drivers.contains_key("kubernetes") { + return Err(Error::config( + "K8s ServiceAccount bootstrap requires [openshell.drivers.kubernetes] when sandbox JWT issuing is enabled in-cluster", + )); + } + let merged = config_file::driver_table( + "kubernetes", + &file.openshell.gateway, + file.openshell.drivers.get("kubernetes"), + ); + merged.try_into().map_err(|error| { + Error::config(format!( + "invalid Kubernetes ServiceAccount bootstrap config: {error}" + )) + }) +} + pub fn driver_config_from_context( context: DriverStartupContext<'_>, driver_name: &str, @@ -157,6 +211,52 @@ socket_path = "/run/openshell/kyma.sock" assert_eq!(cfg.socket_path, PathBuf::from("/run/openshell/kyma.sock")); } + #[test] + fn remote_driver_config_ignores_in_process_driver_fields() { + let file: config_file::ConfigFile = toml::from_str( + r#" +[openshell.gateway] +sandbox_namespace = "sandboxes" + +[openshell.drivers.kubernetes] +socket_path = "/run/openshell/kubernetes.sock" +workspace_mode = "shared" +service_account_name = "sandbox-sa" +"#, + ) + .expect("valid config"); + + let cfg = remote_driver_config_from_context(test_context(Some(&file)), "kubernetes") + .expect("remote config"); + assert_eq!( + cfg.socket_path, + PathBuf::from("/run/openshell/kubernetes.sock") + ); + } + + #[test] + fn kubernetes_sa_bootstrap_uses_public_gateway_config() { + let file: config_file::ConfigFile = toml::from_str( + r#" +[openshell.gateway] +sandbox_namespace = "sandboxes" + +[openshell.drivers.kubernetes] +socket_path = "/run/openshell/kubernetes.sock" +workspace_mode = "managed" +gateway_id = "gateway-a" +service_account_name = "sandbox-sa" +"#, + ) + .expect("valid config"); + + let cfg = kubernetes_sa_bootstrap_config(Some(&file)).expect("bootstrap config"); + assert_eq!(cfg.namespace, "sandboxes"); + assert_eq!(cfg.workspace_mode, "managed"); + assert_eq!(cfg.gateway_id, "gateway-a"); + assert_eq!(cfg.service_account_name, "sandbox-sa"); + } + #[test] fn remote_driver_config_uses_endpoint_override_without_file() { let endpoint_overrides = diff --git a/crates/openshell-server/src/compute/driver_config/builtin.rs b/crates/openshell-server/src/compute/driver_config/builtin.rs index 948c982dd2..dea867237d 100644 --- a/crates/openshell-server/src/compute/driver_config/builtin.rs +++ b/crates/openshell-server/src/compute/driver_config/builtin.rs @@ -3,12 +3,11 @@ //! Configuration construction for built-in compute drivers. -use super::{ - DriverStartupContext, GuestTlsPaths, driver_config_from_context, driver_config_from_file, -}; +use super::{DriverStartupContext, GuestTlsPaths, driver_config_from_context}; use crate::compute::VmComputeConfig; +#[cfg(test)] use crate::config_file; -use openshell_core::{ComputeDriverKind, Error, Result}; +use openshell_core::{ComputeDriverKind, Result}; use openshell_driver_docker::DockerComputeConfig; use openshell_driver_kubernetes::KubernetesComputeConfig; use openshell_driver_podman::PodmanComputeConfig; @@ -23,22 +22,6 @@ pub fn kubernetes_config_from_context( Ok(cfg) } -pub fn kubernetes_config_for_k8s_sa_bootstrap( - file: Option<&config_file::ConfigFile>, -) -> Result { - let Some(file) = file else { - return Err(Error::config( - "K8s ServiceAccount bootstrap requires [openshell.drivers.kubernetes] when sandbox JWT issuing is enabled in-cluster", - )); - }; - if !file.openshell.drivers.contains_key("kubernetes") { - return Err(Error::config( - "K8s ServiceAccount bootstrap requires [openshell.drivers.kubernetes] when sandbox JWT issuing is enabled in-cluster", - )); - } - driver_config_from_file(Some(file), ComputeDriverKind::Kubernetes.as_str()) -} - /// Build the selected Podman config from TOML plus runtime defaults. pub fn podman_config_from_context( context: DriverStartupContext<'_>, @@ -165,35 +148,6 @@ mod tests { } } - #[test] - fn k8s_sa_bootstrap_rejects_missing_kubernetes_driver_config() { - let err = kubernetes_config_for_k8s_sa_bootstrap(None).unwrap_err(); - assert!(err.to_string().contains("[openshell.drivers.kubernetes]")); - - let file: config_file::ConfigFile = - toml::from_str("[openshell.gateway]\n").expect("valid config"); - let err = kubernetes_config_for_k8s_sa_bootstrap(Some(&file)).unwrap_err(); - assert!(err.to_string().contains("[openshell.drivers.kubernetes]")); - } - - #[test] - fn k8s_sa_bootstrap_uses_configured_namespace_and_service_account() { - let file: config_file::ConfigFile = toml::from_str( - r#" -[openshell.gateway] - -[openshell.drivers.kubernetes] -namespace = "sandboxes" -service_account_name = "sandbox-sa" -"#, - ) - .expect("valid config"); - - let cfg = kubernetes_config_for_k8s_sa_bootstrap(Some(&file)).unwrap(); - assert_eq!(cfg.namespace, "sandboxes"); - assert_eq!(cfg.service_account_name, "sandbox-sa"); - } - #[test] fn podman_config_reads_bind_mount_opt_in_from_driver_table() { let file: config_file::ConfigFile = toml::from_str( diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 0261520f9a..4d5c9447b5 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -5,16 +5,16 @@ pub mod driver_config; pub mod lease; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub mod vm; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub use openshell_driver_docker::DockerComputeConfig; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub use openshell_driver_kubernetes::KubernetesComputeConfig; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub use openshell_driver_podman::PodmanComputeConfig; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub use vm::VmComputeConfig; use crate::grpc::policy::SANDBOX_SETTINGS_OBJECT_TYPE; @@ -49,14 +49,14 @@ use openshell_core::proto::{ SandboxTemplate, ServiceEndpoint, SshSession, }; use openshell_core::{ObjectLabels, ObjectWorkspace}; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] use openshell_driver_docker::DockerComputeDriver; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] use openshell_driver_kubernetes::{ ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, OperatorNamespaceAllowlist, }; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; use prost::Message; use std::collections::HashMap; @@ -296,7 +296,7 @@ pub struct ManagedDriverProcess { } impl ManagedDriverProcess { - #[cfg(unix)] + #[cfg(all(unix, any(test, feature = "in-tree-compute-drivers")))] pub(crate) fn new(child: tokio::process::Child, socket_path: PathBuf) -> Self { Self { child: std::sync::Mutex::new(Some(child)), @@ -394,6 +394,7 @@ pub struct AcquiredRemoteDriverEndpoint { } impl AcquiredRemoteDriverEndpoint { + #[cfg(any(test, feature = "in-tree-compute-drivers"))] pub(crate) fn managed_builtin( driver_kind: ComputeDriverKind, channel: Channel, @@ -710,7 +711,7 @@ impl ComputeRuntime { self.lifecycle_gates.entry_count() } - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub async fn new_docker( config: openshell_core::Config, docker_config: DockerComputeConfig, @@ -738,7 +739,7 @@ impl ComputeRuntime { .await } - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub async fn new_kubernetes( config: KubernetesComputeConfig, store: Arc, @@ -789,7 +790,7 @@ impl ComputeRuntime { .await } - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] pub async fn new_podman( config: PodmanComputeConfig, store: Arc, @@ -3198,20 +3199,34 @@ pub async fn connect_remote_compute_driver( name: impl Into, socket_path: &Path, ) -> Result { - let socket_path: PathBuf = socket_path.to_path_buf(); - let display_path = socket_path.clone(); - let channel = Endpoint::from_static("http://[::]:50051") - .connect_with_connector(service_fn(move |_: tonic::transport::Uri| { - let socket_path = socket_path.clone(); - async move { UnixStream::connect(socket_path).await.map(TokioIo::new) } - })) - .await - .map_err(|e| { - ComputeError::Message(format!( - "failed to connect to remote compute driver socket '{}': {e}", - display_path.display() - )) - })?; + let socket_path = socket_path.to_path_buf(); + let deadline = tokio::time::Instant::now() + Duration::from_secs(30); + let channel = loop { + let connector_path = socket_path.clone(); + match Endpoint::from_static("http://[::]:50051") + .connect_with_connector(service_fn(move |_: tonic::transport::Uri| { + let connector_path = connector_path.clone(); + async move { UnixStream::connect(connector_path).await.map(TokioIo::new) } + })) + .await + { + Ok(channel) => break channel, + Err(error) if tokio::time::Instant::now() < deadline => { + tracing::debug!( + socket = %socket_path.display(), + %error, + "waiting for remote compute driver socket" + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } + Err(error) => { + return Err(ComputeError::Message(format!( + "failed to connect to remote compute driver socket '{}' within 30s: {error}", + socket_path.display() + ))); + } + } + }; Ok(AcquiredRemoteDriverEndpoint::unmanaged(name, channel)) } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index e181eeae7e..a2f26f6ae8 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -664,26 +664,24 @@ pub(crate) async fn run_server( // Pod lookups and TokenReview identity checks must match the sandbox // namespace and service account used by the Kubernetes driver. let kubernetes_config = - compute::driver_config::builtin::kubernetes_config_for_k8s_sa_bootstrap( - config_file.as_ref(), - )?; + compute::driver_config::kubernetes_sa_bootstrap_config(config_file.as_ref())?; let sandbox_namespace = kubernetes_config.namespace.clone(); let sandbox_service_account = kubernetes_config.service_account_name.clone(); - let namespace_validator = match kubernetes_config.workspace_mode { - openshell_driver_kubernetes::WorkspaceMode::Shared => { - auth::k8s_sa::NamespaceValidator::Exact(kubernetes_config.namespace) - } - openshell_driver_kubernetes::WorkspaceMode::Managed => { - auth::k8s_sa::NamespaceValidator::Prefix( - openshell_driver_kubernetes::managed_namespace_prefix( - &kubernetes_config.gateway_id, - ), - ) - } - openshell_driver_kubernetes::WorkspaceMode::Operator => { + let namespace_validator = match kubernetes_config.workspace_mode.as_str() { + "shared" => auth::k8s_sa::NamespaceValidator::Exact(kubernetes_config.namespace), + "managed" => auth::k8s_sa::NamespaceValidator::Prefix(format!( + "openshell-{}-", + kubernetes_config.gateway_id + )), + "operator" => { let allowlist = operator_allowlist.clone().unwrap_or_default(); auth::k8s_sa::NamespaceValidator::Allowlist(allowlist) } + mode => { + return Err(Error::config(format!( + "invalid Kubernetes workspace_mode '{mode}' for ServiceAccount bootstrap" + ))); + } }; match kube::Client::try_default().await { Ok(client) => { @@ -1066,7 +1064,7 @@ async fn terminate_signal() { let _ = signal.recv().await; } -#[cfg(target_os = "windows")] +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::ComputeError { compute::ComputeError::Message(format!( "{} compute driver is unsupported on Windows", @@ -1074,7 +1072,7 @@ fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::Com )) } -type OperatorAllowlistArc = Option; +type OperatorAllowlistArc = Option; pub use compute::{DriverWatchStream, SharedComputeDriver}; /// Opaque result returned by a compiled compute-driver factory. @@ -1180,8 +1178,9 @@ impl ComputeDriverRegistry { /// Install every first-party compute driver linked into the standard gateway. #[must_use] pub fn install_default_compute_drivers() -> ComputeDriverRegistry { + #[allow(unused_mut)] let mut registry = ComputeDriverRegistry::new(); - #[cfg(not(target_os = "windows"))] + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] { registry .install( @@ -1223,7 +1222,7 @@ pub fn install_default_compute_drivers() -> ComputeDriverRegistry { ) .expect("unique vm registration"); } - #[cfg(target_os = "windows")] + #[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] for name in ["kubernetes", "podman", "docker", "vm"] { registry .install( @@ -1318,11 +1317,11 @@ impl ComputeDriverBuildContext<'_> { } } -#[cfg(target_os = "windows")] +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct UnsupportedComputeDriverFactory; -#[cfg(target_os = "windows")] +#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl ComputeDriverFactory for UnsupportedComputeDriverFactory { async fn build( @@ -1341,11 +1340,11 @@ impl ComputeDriverFactory for UnsupportedComputeDriverFactory { } } -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct KubernetesComputeDriverFactory; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl ComputeDriverFactory for KubernetesComputeDriverFactory { async fn build( @@ -1374,11 +1373,11 @@ impl ComputeDriverFactory for KubernetesComputeDriverFactory { } } -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct DockerComputeDriverFactory; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl ComputeDriverFactory for DockerComputeDriverFactory { async fn build( @@ -1405,11 +1404,11 @@ impl ComputeDriverFactory for DockerComputeDriverFactory { } } -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct PodmanComputeDriverFactory; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl ComputeDriverFactory for PodmanComputeDriverFactory { async fn build( @@ -1435,11 +1434,11 @@ impl ComputeDriverFactory for PodmanComputeDriverFactory { } } -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[derive(Clone, Copy)] struct VmComputeDriverFactory; -#[cfg(not(target_os = "windows"))] +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] #[async_trait::async_trait] impl ComputeDriverFactory for VmComputeDriverFactory { async fn build( @@ -1587,6 +1586,7 @@ fn resolve_configured_compute_driver( Ok(ConfiguredComputeDriver::Remote { name }) } +#[cfg(any(test, feature = "in-tree-compute-drivers"))] fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config) -> bool { config .gateway_jwt @@ -1594,6 +1594,7 @@ fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config) -> bool { .is_some_and(|jwt| jwt.ttl_secs == 0) } +#[cfg(feature = "in-tree-compute-drivers")] fn warn_if_kubernetes_sandbox_jwt_expiry_disabled(config: &Config) { if kubernetes_sandbox_jwt_expiry_disabled(config) { warn!( diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 5ff608ae59..4acbe2ee45 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -5,6 +5,9 @@ Gateway pod template shared by the StatefulSet and Deployment workload shapes. */}} {{- define "openshell.gatewayPodTemplate" -}} +{{- $testing := index .Values "testing" | default (dict) -}} +{{- $externalKubernetesComputeDriver := index $testing "externalKubernetesComputeDriver" | default (dict) -}} +{{- $externalKubernetesComputeDriverSocket := $externalKubernetesComputeDriver.socketPath | default "/var/run/openshell-compute/driver.sock" -}} metadata: annotations: # Roll the gateway workload when the rendered gateway TOML changes - the @@ -78,6 +81,10 @@ spec: - name: OPENSHELL_TELEMETRY_ENABLED value: {{ .Values.server.telemetryEnabled | quote }} volumeMounts: + {{- if $externalKubernetesComputeDriver.enabled }} + - name: compute-driver-socket + mountPath: {{ dir $externalKubernetesComputeDriverSocket | quote }} + {{- end }} {{- if eq (include "openshell.workloadKind" .) "statefulset" }} - name: openshell-data mountPath: /var/openshell @@ -145,7 +152,47 @@ spec: failureThreshold: {{ .Values.probes.readiness.failureThreshold }} resources: {{- toYaml .Values.resources | nindent 8 }} + {{- if $externalKubernetesComputeDriver.enabled }} + - name: kubernetes-compute-driver + securityContext: + {{- toYaml .Values.securityContext | nindent 8 }} + image: {{ include "openshell.image" . | quote }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: ["/usr/local/bin/openshell-driver-kubernetes"] + args: + - --bind-socket + - {{ $externalKubernetesComputeDriverSocket | quote }} + - --workspace-mode + - {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} + - --gateway-id + - {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} + - --sandbox-namespace + - {{ include "openshell.sandboxNamespace" . | quote }} + - --sandbox-service-account + - {{ include "openshell.sandboxServiceAccountName" . | quote }} + - --grpc-endpoint + - {{ include "openshell.grpcEndpoint" . | quote }} + - --sandbox-image + - {{ .Values.server.sandboxImage | quote }} + - --supervisor-image + - {{ include "openshell.supervisorImage" . | quote }} + - --supervisor-sideload-method + - {{ include "openshell.supervisorSideloadMethod" . | quote }} + - --topology + - {{ .Values.supervisor.topology | default "combined" | quote }} + - --client-tls-secret-name + - {{ .Values.server.tls.clientTlsSecretName | quote }} + volumeMounts: + - name: compute-driver-socket + mountPath: {{ dir $externalKubernetesComputeDriverSocket | quote }} + resources: + {{- toYaml .Values.resources | nindent 8 }} + {{- end }} volumes: + {{- if $externalKubernetesComputeDriver.enabled }} + - name: compute-driver-socket + emptyDir: {} + {{- end }} - name: gateway-config configMap: name: {{ include "openshell.fullname" . }}-config diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 9d24dbd917..00e38215c0 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -13,6 +13,8 @@ One value is intentionally NOT rendered here: --db-url arg for SQLite */}} {{- $credentialDrivers := list -}} +{{- $testing := index .Values "testing" | default (dict) -}} +{{- $externalKubernetesComputeDriver := index $testing "externalKubernetesComputeDriver" | default (dict) -}} {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled -}} {{- $credentialDrivers = append $credentialDrivers "kubernetes-secrets" -}} {{- end -}} @@ -135,6 +137,9 @@ data: {{- end }} [openshell.drivers.kubernetes] + {{- if $externalKubernetesComputeDriver.enabled }} + socket_path = {{ $externalKubernetesComputeDriver.socketPath | default "/var/run/openshell-compute/driver.sock" | quote }} + {{- end }} workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} diff --git a/e2e/docker/Dockerfile.external-kubernetes-gateway b/e2e/docker/Dockerfile.external-kubernetes-gateway new file mode 100644 index 0000000000..15e43b9e6b --- /dev/null +++ b/e2e/docker/Dockerfile.external-kubernetes-gateway @@ -0,0 +1,16 @@ +# syntax=docker/dockerfile:1.4 + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +ARG GATEWAY_BASE_IMAGE=gcr.io/distroless/cc-debian13:nonroot@sha256:d97bc0a941b8d4be647dc0ee75b264ddbb772f1ac5ba690a4309c00723b23775 +FROM ${GATEWAY_BASE_IMAGE} + +ARG TARGETARCH +COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-gateway /usr/local/bin/openshell-gateway +COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-driver-kubernetes /usr/local/bin/openshell-driver-kubernetes + +USER 1000:1000 +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/openshell-gateway"] +CMD ["--bind-address", "0.0.0.0", "--port", "8080"] diff --git a/e2e/no-compute-driver-gateway.sh b/e2e/no-compute-driver-gateway.sh new file mode 100755 index 0000000000..7ad3896ca1 --- /dev/null +++ b/e2e/no-compute-driver-gateway.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT}" + +echo "Building gateway without compiled compute drivers..." +cargo build -p openshell-server --bin openshell-gateway \ + --no-default-features --features telemetry + +dependency_tree="$(cargo tree -p openshell-server \ + --no-default-features --features telemetry --edges normal)" +for driver in \ + openshell-driver-docker \ + openshell-driver-kubernetes \ + openshell-driver-podman \ + openshell-driver-vm; do + if grep -q "${driver} v" <<<"${dependency_tree}"; then + echo "ERROR: driver-free gateway dependency graph contains ${driver}" >&2 + exit 1 + fi +done + +"${ROOT}/target/debug/openshell-gateway" --version +echo "Driver-free gateway build passed." diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 26671ccb86..a1ff90f48d 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -97,7 +97,14 @@ fi build_packages=() if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then - build_packages+=(-p openshell-server) + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + echo "==> Building driver-free openshell-gateway" + cargo build \ + -p openshell-server --bin openshell-gateway \ + --no-default-features --features telemetry + else + build_packages+=(-p openshell-server) + fi else echo "==> Using prebuilt openshell-gateway at ${GATEWAY_BIN}" fi @@ -165,6 +172,9 @@ GATEWAY_DB="${RUN_STATE_DIR}/gateway.db" JWT_DIR="${RUN_STATE_DIR}/jwt" PKI_DIR="${RUN_STATE_DIR}/pki" GATEWAY_NAME="openshell-e2e-vm-${HOST_PORT}" +DRIVER_PID="" +DRIVER_LOG="${RUN_STATE_DIR}/vm-driver.log" +DRIVER_SOCKET="${RUN_STATE_DIR}/compute-driver.sock" # ── Cleanup (trap) ─────────────────────────────────────────────────── @@ -188,6 +198,7 @@ cleanup() { kill -KILL "${gateway_pid}" 2>/dev/null || true wait "${gateway_pid}" 2>/dev/null || true fi + e2e_stop_process "${DRIVER_PID}" "external VM compute driver" # On failure, keep the VM console log for debugging. We deliberately # print it instead of leaving it on disk because the state dir gets @@ -196,6 +207,11 @@ cleanup() { echo "=== gateway log (preserved for debugging) ===" cat "${GATEWAY_LOG}" 2>/dev/null || true echo "=== end gateway log ===" + if [ -f "${DRIVER_LOG}" ]; then + echo "=== external VM compute driver log ===" + cat "${DRIVER_LOG}" 2>/dev/null || true + echo "=== end external VM compute driver log ===" + fi local console while IFS= read -r -d '' console; do @@ -261,6 +277,11 @@ gateway_id = "${GATEWAY_NAME}" ttl_secs = 0 [openshell.drivers.vm] +EOF +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + printf 'socket_path = "%s"\n' "${DRIVER_SOCKET}" >>"${GATEWAY_CONFIG}" +else + cat >>"${GATEWAY_CONFIG}" <"${DRIVER_LOG}" 2>&1 & + DRIVER_PID=$! + e2e_wait_for_socket \ + "${DRIVER_SOCKET}" "${DRIVER_PID}" "external VM compute driver" 60 +fi GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index d9f336b411..34412408a9 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -217,8 +217,14 @@ e2e_build_gateway_binaries() { if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then echo "Building openshell-gateway..." - cargo build "${jobs[@]}" \ - -p openshell-server --bin openshell-gateway + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + cargo build "${jobs[@]}" \ + -p openshell-server --bin openshell-gateway \ + --no-default-features --features telemetry + else + cargo build "${jobs[@]}" \ + -p openshell-server --bin openshell-gateway + fi else echo "Using prebuilt openshell gateway at ${OPENSHELL_GATEWAY_BIN}" fi @@ -241,6 +247,59 @@ e2e_build_gateway_binaries() { fi } +e2e_build_external_driver() { + local root=$1 + local package=$2 + local binary=$3 + local output_var=$4 + local target_dir + local jobs=() + + if [ -n "${CARGO_BUILD_JOBS:-}" ]; then + jobs=(-j "${CARGO_BUILD_JOBS}") + fi + target_dir="$(e2e_cargo_target_dir "${root}")" + printf -v "${output_var}" '%s' "${target_dir}/debug/${binary}" + echo "Building external ${binary}..." + cargo build "${jobs[@]}" -p "${package}" --bin "${binary}" + if [ ! -x "${!output_var}" ]; then + echo "ERROR: expected external driver binary at ${!output_var}" >&2 + exit 1 + fi +} + +e2e_wait_for_socket() { + local socket_path=$1 + local process_pid=$2 + local process_label=$3 + local timeout="${4:-30}" + local elapsed=0 + + while [ "${elapsed}" -lt "${timeout}" ]; do + if [ -S "${socket_path}" ]; then + return 0 + fi + if ! kill -0 "${process_pid}" 2>/dev/null; then + echo "ERROR: ${process_label} exited before creating ${socket_path}" >&2 + return 1 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + echo "ERROR: ${process_label} did not create ${socket_path} within ${timeout}s" >&2 + return 1 +} + +e2e_stop_process() { + local process_pid=$1 + local process_label=$2 + if [ -n "${process_pid}" ] && kill -0 "${process_pid}" 2>/dev/null; then + echo "Stopping ${process_label} (pid ${process_pid})..." + kill "${process_pid}" 2>/dev/null || true + wait "${process_pid}" 2>/dev/null || true + fi +} + e2e_write_gateway_args_file() { local args_file=$1 shift diff --git a/e2e/with-docker-gateway.sh b/e2e/with-docker-gateway.sh index 15e9d3466e..958ea1b0eb 100755 --- a/e2e/with-docker-gateway.sh +++ b/e2e/with-docker-gateway.sh @@ -109,6 +109,11 @@ GATEWAY_PID="" GATEWAY_LOG="${WORKDIR}/gateway.log" GATEWAY_PID_FILE="${WORKDIR}/gateway.pid" GATEWAY_ARGS_FILE="${WORKDIR}/gateway.args" +DRIVER_BIN="" +DRIVER_PID="" +DRIVER_LOG="${WORKDIR}/docker-driver.log" +DRIVER_SOCKET="${WORKDIR}/compute-driver.sock" +DRIVER_CONFIG="${WORKDIR}/docker-driver.toml" E2E_NAMESPACE="" DOCKER_NETWORK_NAME="" DOCKER_NETWORK_CONNECTED_CONTAINER="" @@ -134,6 +139,7 @@ cleanup() { local exit_code=$? e2e_stop_gateway "${GATEWAY_PID}" "${GATEWAY_PID_FILE}" + e2e_stop_process "${DRIVER_PID}" "external Docker compute driver" if [ "${exit_code}" -ne 0 ] \ && [ -n "${E2E_NAMESPACE}" ] \ @@ -182,6 +188,11 @@ cleanup() { fi e2e_print_gateway_log_on_failure "${exit_code}" "${GATEWAY_LOG}" + if [ "${exit_code}" -ne 0 ] && [ -f "${DRIVER_LOG}" ]; then + echo "=== external Docker compute driver log ===" + cat "${DRIVER_LOG}" || true + echo "=== end external Docker compute driver log ===" + fi rm -rf "${WORKDIR}" 2>/dev/null || true } @@ -426,6 +437,10 @@ ensure_sandbox_image_available() { } e2e_build_gateway_binaries "${ROOT}" TARGET_DIR GATEWAY_BIN CLI_BIN +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + e2e_build_external_driver \ + "${ROOT}" openshell-driver-docker openshell-driver-docker DRIVER_BIN +fi SUPERVISOR_IMAGE="$(resolve_docker_supervisor_image)" build_local_docker_supervisor_image_if_required "${SUPERVISOR_IMAGE}" @@ -495,21 +510,51 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml" fi fi printf '[openshell.drivers.docker]\n' - printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" - printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" - printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" - printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" - printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" - printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" - printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" - printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" - printf 'enable_bind_mounts = true\n' - printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" - if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then - printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" + else + printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" + printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" + printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" + printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" + printf 'enable_bind_mounts = true\n' + printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" + if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then + printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" + fi fi } > "${GATEWAY_CONFIG}" +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + { + printf 'sandbox_namespace = %s\n' "$(toml_string "${E2E_NAMESPACE}")" + printf 'network_name = %s\n' "$(toml_string "${DOCKER_NETWORK_NAME}")" + printf 'grpc_endpoint = %s\n' "$(toml_string "${GATEWAY_ENDPOINT}")" + printf 'default_image = %s\n' "$(toml_string "${SANDBOX_IMAGE}")" + printf 'image_pull_policy = %s\n' "$(toml_string "${SANDBOX_IMAGE_PULL_POLICY}")" + printf 'guest_tls_ca = %s\n' "$(toml_string "${PKI_DIR}/ca.crt")" + printf 'guest_tls_cert = %s\n' "$(toml_string "${PKI_DIR}/client/tls.crt")" + printf 'guest_tls_key = %s\n' "$(toml_string "${PKI_DIR}/client/tls.key")" + printf 'enable_bind_mounts = true\n' + printf 'supervisor_image = %s\n' "$(toml_string "${SUPERVISOR_IMAGE}")" + if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then + printf 'host_gateway_ip = %s\n' "$(toml_string "${GATEWAY_HOST_ALIAS_IP}")" + fi + } >"${DRIVER_CONFIG}" + "${DRIVER_BIN}" \ + --bind-socket "${DRIVER_SOCKET}" \ + --config "${DRIVER_CONFIG}" \ + --gateway-bind "127.0.0.1:${HOST_PORT}" \ + >"${DRIVER_LOG}" 2>&1 & + DRIVER_PID=$! + e2e_wait_for_socket \ + "${DRIVER_SOCKET}" "${DRIVER_PID}" "external Docker compute driver" +fi + GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" --port "${HOST_PORT}" diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index f6d0efc4ec..8a2ceff63f 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -655,8 +655,33 @@ fi if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then require_cmd docker echo "Building local Kubernetes e2e images (${REGISTRY_VALUE}/{gateway,supervisor}:${IMAGE_TAG_VALUE})..." - CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ - bash "${ROOT}/tasks/scripts/docker-build-image.sh" gateway + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + if [ "$(uname -s)" != "Linux" ]; then + echo "ERROR: external Kubernetes driver image composition currently requires a Linux build host." >&2 + exit 2 + fi + cargo build -p openshell-server --bin openshell-gateway \ + --no-default-features --features telemetry + cargo build -p openshell-driver-kubernetes --bin openshell-driver-kubernetes + case "$(uname -m)" in + x86_64) external_arch=amd64 ;; + aarch64|arm64) external_arch=arm64 ;; + *) echo "ERROR: unsupported external Kubernetes driver architecture: $(uname -m)" >&2; exit 2 ;; + esac + external_stage="${ROOT}/deploy/docker/.build/prebuilt-binaries/${external_arch}" + mkdir -p "${external_stage}" + cp "${ROOT}/target/debug/openshell-gateway" "${external_stage}/openshell-gateway" + cp "${ROOT}/target/debug/openshell-driver-kubernetes" \ + "${external_stage}/openshell-driver-kubernetes" + docker build \ + --build-arg "TARGETARCH=${external_arch}" \ + --tag "${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \ + --file "${ROOT}/e2e/docker/Dockerfile.external-kubernetes-gateway" \ + "${ROOT}" + else + CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ + bash "${ROOT}/tasks/scripts/docker-build-image.sh" gateway + fi CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ bash "${ROOT}/tasks/scripts/docker-build-image.sh" supervisor fi @@ -671,6 +696,16 @@ if [ -n "${import_cluster_name}" ]; then --mode direct >/dev/null fi done +elif [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ] \ + && [[ "${KUBE_CONTEXT}" == kind-* ]] \ + && command -v kind >/dev/null 2>&1; then + kind_cluster_name="${KUBE_CONTEXT#kind-}" + for image in \ + "${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \ + "${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}"; do + echo "Loading ${image} into kind cluster ${kind_cluster_name}..." + kind load docker-image "${image}" --name "${kind_cluster_name}" + done fi # The Kubernetes compute driver creates and watches Sandbox CRs reconciled @@ -690,6 +725,13 @@ fi helm_extra_args=() helm_extra_args+=(--set "server.telemetryEnabled=${OPENSHELL_TELEMETRY_ENABLED}") +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" != "1" ]; then + echo "ERROR: external Kubernetes driver e2e requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=1." >&2 + exit 2 + fi + helm_extra_args+=(--set "testing.externalKubernetesComputeDriver.enabled=true") +fi if [ -n "${HOST_GATEWAY_IP}" ]; then helm_extra_args+=(--set "server.hostGatewayIP=${HOST_GATEWAY_IP}") fi diff --git a/e2e/with-podman-gateway.sh b/e2e/with-podman-gateway.sh index cd52e007ab..bb22ee7376 100755 --- a/e2e/with-podman-gateway.sh +++ b/e2e/with-podman-gateway.sh @@ -92,6 +92,10 @@ GATEWAY_PID="" GATEWAY_LOG="${WORKDIR}/gateway.log" GATEWAY_PID_FILE="${WORKDIR}/gateway.pid" GATEWAY_ARGS_FILE="${WORKDIR}/gateway.args" +DRIVER_BIN="" +DRIVER_PID="" +DRIVER_LOG="${WORKDIR}/podman-driver.log" +DRIVER_SOCKET="${WORKDIR}/compute-driver.sock" E2E_NAMESPACE="" PODMAN_NETWORK_NAME="" PODMAN_NETWORK_MANAGED=0 @@ -114,6 +118,7 @@ cleanup() { local exit_code=$? e2e_stop_gateway "${GATEWAY_PID}" "${GATEWAY_PID_FILE}" + e2e_stop_process "${DRIVER_PID}" "external Podman compute driver" local sandbox_ids="" if command -v podman >/dev/null 2>&1; then @@ -159,6 +164,11 @@ cleanup() { fi e2e_print_gateway_log_on_failure "${exit_code}" "${GATEWAY_LOG}" + if [ "${exit_code}" -ne 0 ] && [ -f "${DRIVER_LOG}" ]; then + echo "=== external Podman compute driver log ===" + cat "${DRIVER_LOG}" || true + echo "=== end external Podman compute driver log ===" + fi if [ "${exit_code}" -ne 0 ] && [ -f "${PODMAN_SERVICE_LOG}" ]; then echo "=== podman service log (preserved for debugging) ===" cat "${PODMAN_SERVICE_LOG}" || true @@ -363,6 +373,10 @@ fi ensure_podman_api_socket e2e_build_gateway_binaries "${ROOT}" TARGET_DIR GATEWAY_BIN CLI_BIN +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + e2e_build_external_driver \ + "${ROOT}" openshell-driver-podman openshell-driver-podman DRIVER_BIN +fi SUPERVISOR_IMAGE="$(resolve_podman_supervisor_image)" ensure_podman_supervisor_image "${SUPERVISOR_IMAGE}" @@ -443,6 +457,9 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" fi fi printf '\n[openshell.drivers.podman]\n' + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")" + else # The Podman driver scopes isolation by network rather than namespace. printf 'network_name = %s\n' "$(toml_string "${PODMAN_NETWORK_NAME}")" printf 'gateway_port = %s\n' "${HOST_PORT}" @@ -464,8 +481,28 @@ cp "${ROOT}/deploy/rpm/gateway.toml.default" "${GATEWAY_CONFIG}" if [ -n "${OPENSHELL_PODMAN_SOCKET:-}" ]; then printf 'socket_path = %s\n' "$(toml_string "${OPENSHELL_PODMAN_SOCKET}")" fi + fi } >> "${GATEWAY_CONFIG}" +if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then + OPENSHELL_COMPUTE_DRIVER_SOCKET="${DRIVER_SOCKET}" \ + OPENSHELL_PODMAN_SOCKET="${OPENSHELL_PODMAN_SOCKET:-}" \ + OPENSHELL_SANDBOX_IMAGE="${SANDBOX_IMAGE}" \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY="missing" \ + OPENSHELL_GATEWAY_PORT="${HOST_PORT}" \ + OPENSHELL_NETWORK_NAME="${PODMAN_NETWORK_NAME}" \ + OPENSHELL_STOP_TIMEOUT="${PODMAN_STOP_TIMEOUT_SECS}" \ + OPENSHELL_SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE}" \ + OPENSHELL_PODMAN_TLS_CA="${PKI_DIR}/ca.crt" \ + OPENSHELL_PODMAN_TLS_CERT="${PKI_DIR}/client/tls.crt" \ + OPENSHELL_PODMAN_TLS_KEY="${PKI_DIR}/client/tls.key" \ + OPENSHELL_ENABLE_BIND_MOUNTS=true \ + "${DRIVER_BIN}" >"${DRIVER_LOG}" 2>&1 & + DRIVER_PID=$! + e2e_wait_for_socket \ + "${DRIVER_SOCKET}" "${DRIVER_PID}" "external Podman compute driver" +fi + GATEWAY_ARGS=( --config "${GATEWAY_CONFIG}" # compute_drivers comes from the RPM template. Override the loopback address diff --git a/examples/governance-interceptor/Cargo.lock b/examples/governance-interceptor/Cargo.lock index 02aaefe8a7..8cbb4bafcc 100644 --- a/examples/governance-interceptor/Cargo.lock +++ b/examples/governance-interceptor/Cargo.lock @@ -883,6 +883,7 @@ dependencies = [ "prost", "prost-types", "protoc-bin-vendored", + "rustix", "serde", "serde_json", "thiserror", diff --git a/tasks/test.toml b/tasks/test.toml index bef8baf2b5..9f47d0dcda 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -184,6 +184,30 @@ run = "e2e/rust/e2e-kubernetes.sh" description = "Start openshell-gateway with the VM compute driver and run VM e2e tests" run = "e2e/rust/e2e-vm.sh" +["e2e:gateway:no-compute-drivers"] +description = "Build and launch-check openshell-gateway without compiled compute drivers" +run = "bash e2e/no-compute-driver-gateway.sh" + +["e2e:docker:external-driver"] +description = "Run Docker smoke E2E with a driver-free gateway and external Docker driver binary" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1" } +run = "e2e/rust/e2e-docker.sh" + +["e2e:podman:external-driver"] +description = "Run Podman E2E with a driver-free gateway and external Podman driver binary" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_PODMAN_TEST = "smoke" } +run = "e2e/rust/e2e-podman.sh" + +["e2e:vm:external-driver"] +description = "Run VM E2E with a driver-free gateway and external VM driver binary" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1" } +run = "e2e/rust/e2e-vm.sh" + +["e2e:kubernetes:external-driver"] +description = "Run Kubernetes smoke E2E with a driver-free gateway and external Kubernetes driver sidecar" +env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBE_TEST = "smoke" } +run = "e2e/rust/e2e-kubernetes.sh" + ["e2e:docker"] description = "Run smoke e2e against a standalone gateway with the Docker compute driver" run = "e2e/rust/e2e-docker.sh" From 0cf505c3cb590d1b36f215ab5bf297deb34e77d9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 18 Aug 2026 02:18:59 -0700 Subject: [PATCH 03/13] refactor(compute): isolate gateway driver composition Signed-off-by: Drew Newberry --- .../skills/debug-openshell-cluster/SKILL.md | 5 +- .github/workflows/release-dev.yml | 2 +- .github/workflows/release-tag.yml | 2 +- .github/workflows/rust-native-build.yml | 2 +- AGENTS.md | 1 + Cargo.lock | 26 +- architecture/compute-runtimes.md | 12 +- crates/openshell-core/src/config.rs | 558 +----------------- ...lowlist.rs => dynamic_string_allowlist.rs} | 11 +- crates/openshell-core/src/lib.rs | 13 +- crates/openshell-core/src/local_api_socket.rs | 81 +++ crates/openshell-core/src/telemetry.rs | 65 +- crates/openshell-driver-docker/src/lib.rs | 36 +- .../openshell-driver-kubernetes/src/config.rs | 2 +- crates/openshell-driver-kubernetes/src/lib.rs | 2 +- crates/openshell-driver-podman/README.md | 10 +- crates/openshell-driver-podman/src/driver.rs | 40 +- crates/openshell-driver-vm/src/driver.rs | 27 +- crates/openshell-gateway/Cargo.toml | 59 ++ crates/openshell-gateway/src/lib.rs | 260 ++++++++ .../src/main.rs | 8 +- .../compute => openshell-gateway/src}/vm.rs | 60 +- crates/openshell-server/Cargo.toml | 18 +- crates/openshell-server/src/auth/k8s_sa.rs | 2 +- crates/openshell-server/src/cli.rs | 192 +++--- .../src/compute/driver_config.rs | 19 +- .../src/compute/driver_config/builtin.rs | 231 -------- crates/openshell-server/src/compute/mod.rs | 121 +--- crates/openshell-server/src/config_file.rs | 87 +-- crates/openshell-server/src/grpc/sandbox.rs | 35 +- crates/openshell-server/src/lib.rs | 439 ++++---------- deploy/docker/Dockerfile.gateway-macos | 16 +- e2e/no-compute-driver-gateway.sh | 16 +- e2e/run.sh | 4 +- e2e/rust/e2e-vm.sh | 4 +- e2e/support/gateway-common.sh | 4 +- e2e/with-kube-gateway.sh | 2 +- tasks/ci.toml | 2 +- tasks/gateway.toml | 2 +- tasks/rust.toml | 4 +- tasks/scripts/gateway-docker.sh | 2 +- tasks/scripts/gateway-vm.sh | 2 +- tasks/scripts/package-deb-install.sh | 2 +- tasks/scripts/stage-prebuilt-binaries.sh | 2 +- tasks/scripts/vm/smoke-orphan-cleanup.sh | 2 +- 45 files changed, 894 insertions(+), 1596 deletions(-) rename crates/openshell-core/src/{operator_namespace_allowlist.rs => dynamic_string_allowlist.rs} (84%) create mode 100644 crates/openshell-core/src/local_api_socket.rs create mode 100644 crates/openshell-gateway/Cargo.toml create mode 100644 crates/openshell-gateway/src/lib.rs rename crates/{openshell-server => openshell-gateway}/src/main.rs (59%) rename crates/{openshell-server/src/compute => openshell-gateway/src}/vm.rs (94%) delete mode 100644 crates/openshell-server/src/compute/driver_config/builtin.rs diff --git a/.agents/skills/debug-openshell-cluster/SKILL.md b/.agents/skills/debug-openshell-cluster/SKILL.md index d7723fd8d7..4c4576398d 100644 --- a/.agents/skills/debug-openshell-cluster/SKILL.md +++ b/.agents/skills/debug-openshell-cluster/SKILL.md @@ -19,8 +19,9 @@ The target deployment flow is: 4. The CLI registers a reachable gateway endpoint with `openshell gateway add`. 5. The gateway creates sandboxes through the selected compute driver. -The standard gateway binary explicitly installs its compiled Docker, Podman, -Kubernetes, and VM registrations at startup. With no configured driver, the +The `openshell-gateway` composition crate explicitly installs its compiled +Docker, Podman, Kubernetes, and VM registrations at startup; `openshell-server` +does not link compute-driver crates. With no configured driver, the gateway probes only installed registrations in priority order (Kubernetes, Podman, then Docker); VM has no probe and remains opt-in. A custom gateway binary may install a different set, so confirm the binary's registered drivers diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index 768f5bd5a6..c1d5afd70a 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -442,7 +442,7 @@ jobs: run: | set -euo pipefail mise x -- rustup target add ${{ matrix.target }} - mise x -- cargo zigbuild --release --target ${{ matrix.zig_target }} -p openshell-server --bin openshell-gateway --features bundled-z3 + mise x -- cargo zigbuild --release --target ${{ matrix.zig_target }} -p openshell-gateway --bin openshell-gateway --features bundled-z3 mkdir -p artifacts/bin install -m 0755 target/${{ matrix.target }}/release/openshell-gateway artifacts/bin/openshell-gateway diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 35792bcb63..269144c913 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -478,7 +478,7 @@ jobs: run: | set -euo pipefail mise x -- rustup target add ${{ matrix.target }} - mise x -- cargo zigbuild --release --target ${{ matrix.zig_target }} -p openshell-server --bin openshell-gateway --features bundled-z3 + mise x -- cargo zigbuild --release --target ${{ matrix.zig_target }} -p openshell-gateway --bin openshell-gateway --features bundled-z3 mkdir -p artifacts/bin install -m 0755 target/${{ matrix.target }}/release/openshell-gateway artifacts/bin/openshell-gateway diff --git a/.github/workflows/rust-native-build.yml b/.github/workflows/rust-native-build.yml index f024cacc50..4d036a8583 100644 --- a/.github/workflows/rust-native-build.yml +++ b/.github/workflows/rust-native-build.yml @@ -123,7 +123,7 @@ jobs: case "$COMPONENT" in gateway) - crate=openshell-server + crate=openshell-gateway binary=openshell-gateway zig_target= ;; diff --git a/AGENTS.md b/AGENTS.md index bd532c1971..9e0f53dd22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files | `crates/openshell-otel/` | OpenTelemetry support | Shared OTLP trace provider, resource, and tracing-layer construction | | `crates/openshell-core/` | Shared core | Common types, configuration, error handling | | `crates/openshell-extension-core/` | Extension core | Shared extension identity, JWT claims, bearer-token rotation, and TLS transport primitives | +| `crates/openshell-gateway/` | Gateway binary composition | Links selected first-party compute drivers into the backend-agnostic server registry | | `crates/openshell-sdk/` | Shared client SDK | Async Rust gateway client (gRPC transport, TLS, OIDC refresh, edge tunnel); consumed by CLI, TUI, and `@openshell/sdk` | | `crates/openshell-providers/` | Provider management | Credential provider backends | | `crates/openshell-tui/` | Terminal UI | Ratatui-based dashboard for monitoring | diff --git a/Cargo.lock b/Cargo.lock index 40291d8cf5..e6d1af2cea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3927,6 +3927,29 @@ dependencies = [ "tower 0.5.3", ] +[[package]] +name = "openshell-gateway" +version = "0.0.0" +dependencies = [ + "async-trait", + "hyper-util", + "miette", + "nix 0.29.0", + "openshell-core", + "openshell-driver-docker", + "openshell-driver-kubernetes", + "openshell-driver-podman", + "openshell-otel", + "openshell-server", + "rustix 1.1.4", + "serde", + "tempfile", + "tokio", + "tonic", + "tower 0.5.3", + "tracing", +] + [[package]] name = "openshell-gateway-interceptors" version = "0.0.0" @@ -4126,10 +4149,7 @@ dependencies = [ "openshell-bootstrap", "openshell-core", "openshell-driver-db-credstore", - "openshell-driver-docker", - "openshell-driver-kubernetes", "openshell-driver-kubernetes-secrets", - "openshell-driver-podman", "openshell-driver-vault", "openshell-extension-core", "openshell-gateway-interceptors", diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index 022db31a1f..f0e0554b45 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -108,11 +108,13 @@ to `run_cli_with_compute_drivers`; factories receive merged driver config and finish through the same in-process runtime adapter. A configured UDS endpoint still takes precedence over a compiled registration with the same name. -The standard server crate groups first-party registrations behind the -`in-tree-compute-drivers` feature. Protocol-only gateway builds disable that -feature and link no compute-driver crates. E2E lanes compose that gateway with -Docker, Podman, Kubernetes, and VM driver executables over the public UDS gRPC -contract so an in-tree driver cannot silently depend on a server-only API. +The `openshell-gateway` composition crate groups first-party registrations +behind the `in-tree-compute-drivers` feature. `openshell-server` has no compute +driver dependencies or backend-name dispatch. Protocol-only gateway builds +disable the composition feature and link no compute-driver crates. E2E lanes +compose that gateway with Docker, Podman, Kubernetes, and VM driver executables +over the public UDS gRPC contract so an in-tree driver cannot silently depend +on a server-only API. ## Stop and Start Lifecycle diff --git a/crates/openshell-core/src/config.rs b/crates/openshell-core/src/config.rs index 62411e20b5..a481997b8e 100644 --- a/crates/openshell-core/src/config.rs +++ b/crates/openshell-core/src/config.rs @@ -6,13 +6,8 @@ use serde::{Deserialize, Serialize}; use std::borrow::Cow; use std::collections::BTreeMap; -use std::fmt; -#[cfg(unix)] -use std::io::{Read, Write}; use std::net::SocketAddr; -#[cfg(unix)] -use std::os::unix::fs::FileTypeExt; -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::str::FromStr; use std::time::Duration; @@ -31,9 +26,6 @@ pub const DEFAULT_SERVER_PORT: u16 = 17670; /// Default container stop timeout in seconds (SIGTERM → SIGKILL). pub const DEFAULT_STOP_TIMEOUT_SECS: u32 = 10; -/// Default Docker bridge network name for local sandboxes. -pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; - /// Default domain used for browser-facing sandbox service URLs. pub const DEFAULT_SERVICE_ROUTING_DOMAIN: &str = "openshell.localhost"; @@ -114,31 +106,9 @@ pub const CDI_GPU_DEVICE_ALL: &str = "nvidia.com/gpu=all"; /// Default maximum number of processes (PIDs) allowed inside a sandbox container. /// -/// Shared by the Docker and Podman drivers; override via driver config. +/// Compute drivers may override this through backend configuration. pub const DEFAULT_SANDBOX_PIDS_LIMIT: i64 = 2048; -/// Compute backends the gateway can orchestrate sandboxes through. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum ComputeDriverKind { - Kubernetes, - Vm, - Docker, - Podman, -} - -impl ComputeDriverKind { - #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::Kubernetes => "kubernetes", - Self::Vm => "vm", - Self::Docker => "docker", - Self::Podman => "podman", - } - } -} - /// Normalize a configured compute driver name. /// /// Built-in driver names and custom remote driver names share the same @@ -160,253 +130,6 @@ pub fn normalize_compute_driver_name(value: &str) -> Result { Ok(value.to_ascii_lowercase()) } -impl fmt::Display for ComputeDriverKind { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -impl FromStr for ComputeDriverKind { - type Err = String; - - fn from_str(value: &str) -> Result { - match value.trim().to_ascii_lowercase().as_str() { - "kubernetes" => Ok(Self::Kubernetes), - "vm" => Ok(Self::Vm), - "docker" => Ok(Self::Docker), - "podman" => Ok(Self::Podman), - other => Err(format!( - "unsupported compute driver '{other}'. expected one of: kubernetes, vm, docker, podman" - )), - } - } -} - -/// Auto-detect the appropriate compute driver based on the runtime environment. -/// -/// Priority order: Kubernetes → Podman → Docker. -/// VM is never auto-detected (requires explicit `--drivers vm`). -/// -/// Returns the first driver where the environment check passes. -/// Returns `None` if no compatible driver is found. -pub fn detect_driver() -> Option { - // Kubernetes: check for KUBERNETES_SERVICE_HOST env var (set inside pods) - if std::env::var_os("KUBERNETES_SERVICE_HOST").is_some() { - return Some(ComputeDriverKind::Kubernetes); - } - - // Podman: check for a reachable local API socket. - if is_podman_available() { - return Some(ComputeDriverKind::Podman); - } - - // Docker: check for a reachable local API socket. - if is_docker_available() { - return Some(ComputeDriverKind::Docker); - } - - None -} - -/// Return whether a responsive local Podman API socket is available. -#[must_use] -pub fn is_podman_available() -> bool { - detect_podman_socket().is_some() -} - -/// Return the first responsive Podman API socket, or `None` if none respond. -pub fn detect_podman_socket() -> Option { - detect_podman_socket_from_candidates(&podman_socket_candidates()) -} - -fn detect_podman_socket_from_candidates(candidates: &[PathBuf]) -> Option { - candidates - .iter() - .find(|path| podman_socket_responds(path)) - .cloned() -} - -fn podman_socket_candidates() -> Vec { - let socket = std::env::var("OPENSHELL_PODMAN_SOCKET") - .ok() - .filter(|path| !path.trim().is_empty()) - .map(PathBuf::from); - podman_socket_candidates_from_env( - socket, - std::env::var_os("XDG_RUNTIME_DIR").map(PathBuf::from), - std::env::var_os("HOME").map(PathBuf::from), - ) -} - -fn podman_socket_candidates_from_env( - socket: Option, - runtime_dir: Option, - home: Option, -) -> Vec { - let mut candidates = Vec::new(); - - if let Some(path) = socket { - candidates.push(path); - } - - if let Some(runtime_dir) = runtime_dir { - candidates.push(runtime_dir.join("podman/podman.sock")); - } - - #[cfg(target_os = "linux")] - { - candidates.push(PathBuf::from(format!( - "/run/user/{}/podman/podman.sock", - current_uid() - ))); - } - - if let Some(home) = home { - candidates.push(home.join(".local/share/containers/podman/machine/podman.sock")); - } - - candidates -} - -/// Return whether a responsive local Docker API socket is available. -#[must_use] -pub fn is_docker_available() -> bool { - detect_docker_socket().is_some() -} - -pub fn detect_docker_socket() -> Option { - detect_docker_socket_from_candidates(&docker_socket_candidates()) -} - -fn detect_docker_socket_from_candidates(candidates: &[PathBuf]) -> Option { - candidates - .iter() - .find(|path| docker_socket_responds(path)) - .cloned() -} - -fn docker_socket_candidates() -> Vec { - let mut candidates = Vec::new(); - - if let Ok(host) = std::env::var("DOCKER_HOST") - && let Some(path) = docker_host_unix_socket_path(&host) - { - candidates.push(path); - } - - candidates.push(PathBuf::from("/var/run/docker.sock")); - - if let Some(home) = std::env::var_os("HOME") { - candidates.push(PathBuf::from(home).join(".docker/run/docker.sock")); - } - - if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { - candidates.push(PathBuf::from(runtime_dir).join("docker.sock")); - } - - candidates -} - -fn docker_host_unix_socket_path(host: &str) -> Option { - let path = host.trim().strip_prefix("unix://")?; - (!path.is_empty()).then(|| PathBuf::from(path)) -} - -#[cfg(unix)] -fn is_unix_socket(path: &Path) -> bool { - path.metadata() - .is_ok_and(|metadata| metadata.file_type().is_socket()) -} - -#[cfg(unix)] -fn podman_socket_responds(path: &Path) -> bool { - unix_socket_http_ping(path, |response| { - http_response_is_success(response) && contains_ascii(response, b"Libpod-Api-Version:") - }) -} - -#[cfg(unix)] -fn docker_socket_responds(path: &Path) -> bool { - unix_socket_http_ping(path, |response| { - http_response_is_success(response) - && contains_ascii(response, b"Api-Version:") - && !contains_ascii(response, b"Libpod-Api-Version:") - }) -} - -#[cfg(unix)] -fn unix_socket_http_ping(path: &Path, accepts_response: impl FnOnce(&[u8]) -> bool) -> bool { - const PROBE_TIMEOUT: Duration = Duration::from_secs(1); - const PING_REQUEST: &[u8] = - b"GET /_ping HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; - - if !is_unix_socket(path) { - return false; - } - - let Ok(mut stream) = std::os::unix::net::UnixStream::connect(path) else { - return false; - }; - if stream.set_read_timeout(Some(PROBE_TIMEOUT)).is_err() - || stream.set_write_timeout(Some(PROBE_TIMEOUT)).is_err() - || stream.write_all(PING_REQUEST).is_err() - { - return false; - } - - let mut response = [0_u8; 512]; - let mut total = 0; - while total < response.len() { - let Ok(n) = stream.read(&mut response[total..]) else { - return false; - }; - if n == 0 { - break; - } - total += n; - if contains_ascii(&response[..total], b"\r\n\r\n") { - break; - } - } - total > 0 && accepts_response(&response[..total]) -} - -#[cfg(unix)] -fn http_response_is_success(response: &[u8]) -> bool { - response.starts_with(b"HTTP/1.1 200") || response.starts_with(b"HTTP/1.0 200") -} - -#[cfg(unix)] -fn contains_ascii(haystack: &[u8], needle: &[u8]) -> bool { - haystack - .windows(needle.len()) - .any(|window| window.eq_ignore_ascii_case(needle)) -} - -#[cfg(all(unix, test))] -fn is_reachable_unix_socket(path: &Path) -> bool { - is_unix_socket(path) && std::os::unix::net::UnixStream::connect(path).is_ok() -} - -#[cfg(all(unix, target_os = "linux"))] -fn current_uid() -> u32 { - use std::os::unix::fs::MetadataExt; - - std::fs::metadata("/proc/self").map_or(0, |metadata| metadata.uid()) -} - -#[cfg(not(unix))] -fn podman_socket_responds(path: &Path) -> bool { - let _ = path; - false -} - -#[cfg(not(unix))] -fn docker_socket_responds(path: &Path) -> bool { - let _ = path; - false -} - /// Server configuration. /// /// Built programmatically in [`crate::Config::new`] and the gateway CLI from @@ -1088,49 +811,14 @@ const fn default_ssh_session_ttl_secs() -> u64 { #[cfg(test)] mod tests { use super::{ - ComputeDriverKind, Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, + Config, DEFAULT_SERVICE_ROUTING_DOMAIN, GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayJwtConfig, GatewayProviderProfileSourceConfig, PolicyValidationFailureMode, - detect_docker_socket_from_candidates, detect_driver, detect_podman_socket_from_candidates, - docker_host_unix_socket_path, docker_socket_responds, normalize_compute_driver_name, - podman_socket_candidates_from_env, podman_socket_responds, + normalize_compute_driver_name, }; - #[cfg(unix)] - use super::{is_reachable_unix_socket, is_unix_socket}; - #[cfg(unix)] - use std::io::{Read as _, Write as _}; use std::net::SocketAddr; - #[cfg(unix)] - use std::os::unix::net::UnixListener; - use std::path::PathBuf; use std::time::Duration; - #[test] - fn compute_driver_kind_parses_supported_values() { - assert_eq!( - "kubernetes".parse::().unwrap(), - ComputeDriverKind::Kubernetes - ); - assert_eq!( - "vm".parse::().unwrap(), - ComputeDriverKind::Vm - ); - assert_eq!( - "podman".parse::().unwrap(), - ComputeDriverKind::Podman - ); - assert_eq!( - "docker".parse::().unwrap(), - ComputeDriverKind::Docker - ); - } - - #[test] - fn compute_driver_kind_rejects_unknown_values() { - let err = "firecracker".parse::().unwrap_err(); - assert!(err.contains("unsupported compute driver 'firecracker'")); - } - #[test] fn policy_validation_failure_mode_is_secure_by_default() { assert_eq!( @@ -1341,244 +1029,6 @@ mod tests { assert_eq!(cfg.health_bind_address, Some(addr)); } - #[test] - fn detect_driver_returns_none_without_k8s_env_or_local_runtime() { - // When KUBERNETES_SERVICE_HOST is not set, no Docker binary/socket is - // available, and no Podman API socket is available, detect_driver - // should return None. - // This test may pass or fail depending on the test environment, - // but it documents the expected behavior. - let _ = detect_driver(); // Returns Some or None based on environment - } - - #[test] - fn docker_host_unix_socket_path_parses_unix_hosts() { - assert_eq!( - docker_host_unix_socket_path("unix:///var/run/docker.sock"), - Some(PathBuf::from("/var/run/docker.sock")) - ); - assert_eq!(docker_host_unix_socket_path("tcp://127.0.0.1:2375"), None); - assert_eq!(docker_host_unix_socket_path("unix://"), None); - } - - #[cfg(unix)] - #[test] - fn is_unix_socket_detects_socket_files() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("docker.sock"); - let _listener = UnixListener::bind(&socket_path).expect("bind unix socket"); - - assert!(is_unix_socket(&socket_path)); - assert!(is_reachable_unix_socket(&socket_path)); - - let regular_file = temp_dir.path().join("not-a-socket"); - std::fs::write(®ular_file, b"not a socket").expect("write regular file"); - assert!(!is_unix_socket(®ular_file)); - assert!(!is_reachable_unix_socket(®ular_file)); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn podman_socket_probe_accepts_successful_ping_response() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("podman.sock"); - let listener = UnixListener::bind(&socket_path).expect("bind podman socket"); - - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept podman probe"); - let mut request = [0_u8; 128]; - let n = stream.read(&mut request).expect("read podman probe"); - assert!(request[..n].starts_with(b"GET /_ping HTTP/1.1\r\n")); - stream - .write_all( - b"HTTP/1.1 200 OK\r\nLibpod-Api-Version: 5.8.2\r\nContent-Length: 2\r\n\r\nOK", - ) - .expect("write podman ping response"); - }); - - assert!(podman_socket_responds(&socket_path)); - handle.join().expect("probe server exits"); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn podman_socket_probe_rejects_docker_ping_response() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("podman.sock"); - let listener = UnixListener::bind(&socket_path).expect("bind podman socket"); - - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept podman probe"); - let mut request = [0_u8; 128]; - let n = stream.read(&mut request).expect("read podman probe"); - assert!(request[..n].starts_with(b"GET /_ping HTTP/1.1\r\n")); - stream - .write_all( - b"HTTP/1.1 200 OK\r\nServer: Docker/29.2.1\r\nContent-Length: 2\r\n\r\nOK", - ) - .expect("write docker ping response"); - }); - - assert!(!podman_socket_responds(&socket_path)); - handle.join().expect("probe server exits"); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn docker_socket_probe_accepts_successful_ping_response() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("docker.sock"); - let listener = UnixListener::bind(&socket_path).expect("bind docker socket"); - - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept docker probe"); - let mut request = [0_u8; 128]; - let n = stream.read(&mut request).expect("read docker probe"); - assert!(request[..n].starts_with(b"GET /_ping HTTP/1.1\r\n")); - stream - .write_all( - b"HTTP/1.1 200 OK\r\nApi-Version: 1.51\r\nDocker-Experimental: false\r\nContent-Length: 2\r\n\r\nOK", - ) - .expect("write docker ping response"); - }); - - assert!(docker_socket_responds(&socket_path)); - handle.join().expect("probe server exits"); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn docker_socket_probe_rejects_podman_ping_response() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("podman.sock"); - let listener = UnixListener::bind(&socket_path).expect("bind podman socket"); - - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept docker probe"); - let mut request = [0_u8; 128]; - let n = stream.read(&mut request).expect("read docker probe"); - assert!(request[..n].starts_with(b"GET /_ping HTTP/1.1\r\n")); - stream - .write_all( - b"HTTP/1.1 200 OK\r\nLibpod-Api-Version: 5.8.2\r\nContent-Length: 2\r\n\r\nOK", - ) - .expect("write podman ping response"); - }); - - assert!(!docker_socket_responds(&socket_path)); - handle.join().expect("probe server exits"); - } - - #[cfg(unix)] - #[test] - fn docker_socket_probe_rejects_inactive_socket() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let socket_path = temp_dir.path().join("docker.sock"); - let listener = UnixListener::bind(&socket_path).expect("bind docker socket"); - drop(listener); - - assert!(is_unix_socket(&socket_path)); - assert!(!docker_socket_responds(&socket_path)); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn docker_socket_detection_returns_the_responsive_candidate() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let inactive_path = temp_dir.path().join("inactive.sock"); - let inactive_listener = UnixListener::bind(&inactive_path).expect("bind inactive socket"); - drop(inactive_listener); - - let responsive_path = temp_dir.path().join("responsive.sock"); - let listener = UnixListener::bind(&responsive_path).expect("bind responsive socket"); - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept docker probe"); - let mut request = [0_u8; 128]; - let _ = stream.read(&mut request).expect("read docker probe"); - stream - .write_all(b"HTTP/1.1 200 OK\r\nApi-Version: 1.51\r\nContent-Length: 2\r\n\r\nOK") - .expect("write docker ping response"); - }); - - assert_eq!( - detect_docker_socket_from_candidates(&[inactive_path, responsive_path.clone(),]), - Some(responsive_path) - ); - handle.join().expect("probe server exits"); - } - - #[test] - fn podman_socket_candidates_include_env_runtime_and_home_paths() { - let candidates = podman_socket_candidates_from_env( - Some(PathBuf::from("/tmp/custom-podman.sock")), - Some(PathBuf::from("/tmp/runtime")), - Some(PathBuf::from("/tmp/home")), - ); - - assert!(candidates.contains(&PathBuf::from("/tmp/custom-podman.sock"))); - assert!(candidates.contains(&PathBuf::from("/tmp/runtime/podman/podman.sock"))); - assert!(candidates.contains(&PathBuf::from( - "/tmp/home/.local/share/containers/podman/machine/podman.sock" - ))); - } - - #[cfg(unix)] - #[test] - #[ignore = "flaky under concurrent test execution"] - fn podman_socket_detection_returns_the_responsive_candidate() { - let temp_dir = tempfile::tempdir().expect("create temp dir"); - let inactive_path = temp_dir.path().join("inactive.sock"); - let inactive_listener = UnixListener::bind(&inactive_path).expect("bind inactive socket"); - drop(inactive_listener); - - let responsive_path = temp_dir.path().join("responsive.sock"); - let listener = UnixListener::bind(&responsive_path).expect("bind responsive socket"); - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().expect("accept podman probe"); - let mut request = [0_u8; 128]; - let _ = stream.read(&mut request).expect("read podman probe"); - stream - .write_all( - b"HTTP/1.1 200 OK\r\nLibpod-Api-Version: 5.8.2\r\nContent-Length: 2\r\n\r\nOK", - ) - .expect("write podman ping response"); - }); - - assert_eq!( - detect_podman_socket_from_candidates(&[inactive_path, responsive_path.clone(),]), - Some(responsive_path) - ); - handle.join().expect("probe server exits"); - } - - #[test] - #[allow(unsafe_code)] // std::env::set_var/remove_var require unsafe in Rust 2024 - fn detect_driver_prefers_kubernetes_when_k8s_env_is_set() { - // Save the original env var - let original = std::env::var("KUBERNETES_SERVICE_HOST").ok(); - - // Set the env var - unsafe { - std::env::set_var("KUBERNETES_SERVICE_HOST", "127.0.0.1"); - } - - let result = detect_driver(); - assert_eq!(result, Some(ComputeDriverKind::Kubernetes)); - - // Restore the original env var - unsafe { - match original { - Some(val) => std::env::set_var("KUBERNETES_SERVICE_HOST", val), - None => std::env::remove_var("KUBERNETES_SERVICE_HOST"), - } - } - } - #[test] fn supervisor_image_tag_prefers_explicit_build_tags() { use super::resolve_supervisor_image_tag; diff --git a/crates/openshell-core/src/operator_namespace_allowlist.rs b/crates/openshell-core/src/dynamic_string_allowlist.rs similarity index 84% rename from crates/openshell-core/src/operator_namespace_allowlist.rs rename to crates/openshell-core/src/dynamic_string_allowlist.rs index c8f0f7f3de..1a2968188f 100644 --- a/crates/openshell-core/src/operator_namespace_allowlist.rs +++ b/crates/openshell-core/src/dynamic_string_allowlist.rs @@ -4,16 +4,13 @@ use std::collections::BTreeSet; use std::sync::{Arc, RwLock}; -/// Thread-safe dynamic allowlist of Kubernetes operator-mode namespaces. -/// -/// This type lives in the public core API because both the Kubernetes driver -/// and gateway authentication boundary consume it. +/// Thread-safe dynamic allowlist of strings shared across component boundaries. #[derive(Debug, Clone)] -pub struct OperatorNamespaceAllowlist { +pub struct DynamicStringAllowlist { inner: Arc>>, } -impl OperatorNamespaceAllowlist { +impl DynamicStringAllowlist { fn read_guard(&self) -> std::sync::RwLockReadGuard<'_, BTreeSet> { self.inner .read() @@ -71,7 +68,7 @@ impl OperatorNamespaceAllowlist { } } -impl Default for OperatorNamespaceAllowlist { +impl Default for DynamicStringAllowlist { fn default() -> Self { Self::new() } diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index 96be19e1e2..67e4bcc606 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -16,6 +16,7 @@ pub mod container_paths; pub mod denial; pub mod driver_mounts; pub mod driver_utils; +pub mod dynamic_string_allowlist; pub mod endpoint_path; pub mod error; #[cfg(unix)] @@ -28,10 +29,10 @@ pub mod host_pattern; pub mod image; pub mod inference; pub mod jwt; +pub mod local_api_socket; pub mod metadata; pub mod middleware; pub mod net; -pub mod operator_namespace_allowlist; pub mod paths; pub mod policy; pub mod progress; @@ -47,16 +48,16 @@ pub mod time; pub mod transport_errors; pub use config::{ - ComputeDriverKind, Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, - GatewayInterceptorBindingPolicy, GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, - GatewayInterceptorPhaseConfig, GatewayJwtConfig, GatewayProviderProfileSourceConfig, - MtlsAuthConfig, OidcConfig, PolicyValidationFailureMode, TlsConfig, + Config, GatewayAuthConfig, GatewayInterceptorBindingOverride, GatewayInterceptorBindingPolicy, + GatewayInterceptorConfig, GatewayInterceptorFailurePolicy, GatewayInterceptorPhaseConfig, + GatewayJwtConfig, GatewayProviderProfileSourceConfig, MtlsAuthConfig, OidcConfig, + PolicyValidationFailureMode, TlsConfig, }; +pub use dynamic_string_allowlist::DynamicStringAllowlist; pub use error::{ComputeDriverError, Error, Result}; pub use metadata::{ GetResourceVersion, ObjectId, ObjectLabels, ObjectName, ObjectWorkspace, SetResourceVersion, }; -pub use operator_namespace_allowlist::OperatorNamespaceAllowlist; /// Build version string derived from git metadata. /// diff --git a/crates/openshell-core/src/local_api_socket.rs b/crates/openshell-core/src/local_api_socket.rs new file mode 100644 index 0000000000..6804ae513a --- /dev/null +++ b/crates/openshell-core/src/local_api_socket.rs @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Generic discovery and probing for local HTTP APIs over Unix sockets. + +use std::path::{Path, PathBuf}; + +/// Return the first candidate whose HTTP ping response is accepted. +#[must_use] +pub fn first_responsive_socket( + candidates: &[PathBuf], + accepts_response: impl Fn(&[u8]) -> bool, +) -> Option { + candidates + .iter() + .find(|path| socket_responds(path, &accepts_response)) + .cloned() +} + +/// Return whether a byte slice contains another, ignoring ASCII case. +#[must_use] +pub fn contains_ascii(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window.eq_ignore_ascii_case(needle)) +} + +/// Return whether an HTTP response starts with a successful status line. +#[must_use] +pub fn http_response_is_success(response: &[u8]) -> bool { + response.starts_with(b"HTTP/1.1 200") || response.starts_with(b"HTTP/1.0 200") +} + +#[cfg(unix)] +fn socket_responds(path: &Path, accepts_response: &impl Fn(&[u8]) -> bool) -> bool { + use std::io::{Read as _, Write as _}; + use std::os::unix::fs::FileTypeExt as _; + use std::time::Duration; + + const PROBE_TIMEOUT: Duration = Duration::from_secs(1); + const PING_REQUEST: &[u8] = + b"GET /_ping HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n"; + + if !path + .metadata() + .is_ok_and(|metadata| metadata.file_type().is_socket()) + { + return false; + } + let Ok(mut stream) = std::os::unix::net::UnixStream::connect(path) else { + return false; + }; + if stream.set_read_timeout(Some(PROBE_TIMEOUT)).is_err() + || stream.set_write_timeout(Some(PROBE_TIMEOUT)).is_err() + || stream.write_all(PING_REQUEST).is_err() + { + return false; + } + + let mut response = [0_u8; 512]; + let mut total = 0; + while total < response.len() { + let Ok(read) = stream.read(&mut response[total..]) else { + return false; + }; + if read == 0 { + break; + } + total += read; + if contains_ascii(&response[..total], b"\r\n\r\n") { + break; + } + } + total > 0 && accepts_response(&response[..total]) +} + +#[cfg(not(unix))] +fn socket_responds(path: &Path, accepts_response: &impl Fn(&[u8]) -> bool) -> bool { + let _ = (path, accepts_response); + false +} diff --git a/crates/openshell-core/src/telemetry.rs b/crates/openshell-core/src/telemetry.rs index b2c9b79152..ea2f6fe36b 100644 --- a/crates/openshell-core/src/telemetry.rs +++ b/crates/openshell-core/src/telemetry.rs @@ -159,47 +159,20 @@ impl SandboxTemplateSource { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TelemetryComputeDriver { - Docker, - Kubernetes, - Podman, - Vm, - Unknown, -} +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TelemetryComputeDriver(String); impl TelemetryComputeDriver { #[must_use] - pub const fn as_str(self) -> &'static str { - match self { - Self::Docker => "docker", - Self::Kubernetes => "kubernetes", - Self::Podman => "podman", - Self::Vm => "vm", - Self::Unknown => "unknown", - } + pub fn as_str(&self) -> &str { + &self.0 } #[must_use] pub fn from_raw(raw: &str) -> Self { - match raw.trim().to_ascii_lowercase().as_str() { - "docker" => Self::Docker, - "k8s" | "kubernetes" => Self::Kubernetes, - "podman" => Self::Podman, - "vm" => Self::Vm, - _ => Self::Unknown, - } - } - - #[must_use] - pub const fn from_driver_kind(driver_kind: Option) -> Self { - match driver_kind { - Some(crate::ComputeDriverKind::Docker) => Self::Docker, - Some(crate::ComputeDriverKind::Kubernetes) => Self::Kubernetes, - Some(crate::ComputeDriverKind::Podman) => Self::Podman, - Some(crate::ComputeDriverKind::Vm) => Self::Vm, - None => Self::Unknown, - } + let name = crate::config::normalize_compute_driver_name(raw) + .unwrap_or_else(|_| "unknown".to_string()); + Self(name) } } @@ -684,27 +657,21 @@ mod tests { } #[test] - fn compute_driver_values_are_sanitized() { - assert_eq!( - TelemetryComputeDriver::from_raw("docker").as_str(), - "docker" - ); - assert_eq!( - TelemetryComputeDriver::from_raw("k8s").as_str(), - "kubernetes" - ); + fn compute_driver_values_are_normalized_without_enumerating_backends() { + assert_eq!(TelemetryComputeDriver::from_raw("alpha").as_str(), "alpha"); assert_eq!( - TelemetryComputeDriver::from_raw("KUBERNETES").as_str(), - "kubernetes" + TelemetryComputeDriver::from_raw(" Alpha ").as_str(), + "alpha" ); - assert_eq!(TelemetryComputeDriver::from_raw("vm").as_str(), "vm"); assert_eq!( - TelemetryComputeDriver::from_raw("podman").as_str(), - "podman" + TelemetryComputeDriver::from_raw("CUSTOM_BACKEND").as_str(), + "custom_backend" ); + assert_eq!(TelemetryComputeDriver::from_raw("beta").as_str(), "beta"); + assert_eq!(TelemetryComputeDriver::from_raw("gamma").as_str(), "gamma"); assert_eq!( TelemetryComputeDriver::from_raw("private-driver").as_str(), - "unknown" + "private-driver" ); } diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 1b9fe1f23b..b1d0f5cb82 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -19,9 +19,7 @@ use bollard::query_parameters::{ }; use bytes::Bytes; use futures::{Stream, StreamExt}; -use openshell_core::config::{ - DEFAULT_DOCKER_NETWORK_NAME, DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS, -}; +use openshell_core::config::{DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS}; use openshell_core::driver_mounts; use openshell_core::driver_utils::{ LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, @@ -388,12 +386,41 @@ fn default_true() -> bool { type WatchStream = Pin> + Send + 'static>>; +/// Return the first responsive local Docker API socket. +#[must_use] +pub fn detect_socket() -> Option { + let mut candidates = Vec::new(); + if let Ok(host) = std::env::var("DOCKER_HOST") + && let Some(path) = host.trim().strip_prefix("unix://") + && !path.is_empty() + { + candidates.push(PathBuf::from(path)); + } + candidates.push(PathBuf::from("/var/run/docker.sock")); + if let Some(home) = std::env::var_os("HOME") { + candidates.push(PathBuf::from(home).join(".docker/run/docker.sock")); + } + if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { + candidates.push(PathBuf::from(runtime_dir).join("docker.sock")); + } + openshell_core::local_api_socket::first_responsive_socket(&candidates, |response| { + openshell_core::local_api_socket::http_response_is_success(response) + && openshell_core::local_api_socket::contains_ascii(response, b"Api-Version:") + && !openshell_core::local_api_socket::contains_ascii(response, b"Libpod-Api-Version:") + }) +} + +#[must_use] +pub fn is_available() -> bool { + detect_socket().is_some() +} + impl DockerComputeDriver { pub async fn new(config: &Config, docker_config: &DockerComputeConfig) -> CoreResult { let socket_path = docker_config .socket_path .clone() - .or_else(openshell_core::config::detect_docker_socket) + .or_else(detect_socket) .unwrap_or_else(|| PathBuf::from("/var/run/docker.sock")); let socket_path_str = socket_path.to_str().ok_or_else(|| { Error::config(format!( @@ -3659,3 +3686,4 @@ fn internal_status(operation: &str, err: BollardError) -> Status { #[cfg(test)] mod tests; +pub const DEFAULT_DOCKER_NETWORK_NAME: &str = "openshell-docker"; diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 217fec09ec..bd39d77cbb 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -pub use openshell_core::OperatorNamespaceAllowlist; +pub use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist; use openshell_core::config; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::BTreeMap; diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index d69f9749a1..378c995c72 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -13,4 +13,4 @@ pub use config::{ }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; -pub use openshell_core::OperatorNamespaceAllowlist; +pub use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist; diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index aabf28c5b2..126caba564 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -434,11 +434,11 @@ matter compared to cluster or rootful runtimes: ## Implementation References -- Gateway integration: `crates/openshell-server/src/compute/mod.rs` - (`new_podman` and `PodmanComputeDriver` wiring). -- Server configuration: `crates/openshell-server/src/lib.rs` - (`ComputeDriverKind::Podman` builds `PodmanComputeConfig` including - `sandbox_ssh_socket_path` from gateway `Config`). +- Gateway integration: `crates/openshell-gateway/src/lib.rs` registers the + driver factory and constructs `PodmanComputeConfig` from the generic server + build context. +- Server configuration: `crates/openshell-server/src/lib.rs` exposes the + backend-agnostic registry and factory context. - Gateway relay path: `openshell-core` `Config::sandbox_ssh_socket_path` in `crates/openshell-core/src/config.rs`. - SSRF mitigation: `crates/openshell-core/src/net.rs`, diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index 976c67b92d..ae6773825b 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -273,11 +273,42 @@ fn podman_gpu_selection_error(err: CdiGpuSelectionError) -> ComputeDriverError { ComputeDriverError::Precondition(err.to_string()) } +/// Return the first responsive local Podman API socket. +#[must_use] +pub fn detect_socket() -> Option { + let mut candidates = Vec::new(); + if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") + && !path.trim().is_empty() + { + candidates.push(PathBuf::from(path)); + } + if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { + candidates.push(PathBuf::from(runtime_dir).join("podman/podman.sock")); + } + #[cfg(target_os = "linux")] + candidates.push(PathBuf::from(format!( + "/run/user/{}/podman/podman.sock", + rustix::process::geteuid().as_raw() + ))); + if let Some(home) = std::env::var_os("HOME") { + candidates + .push(PathBuf::from(home).join(".local/share/containers/podman/machine/podman.sock")); + } + openshell_core::local_api_socket::first_responsive_socket(&candidates, |response| { + openshell_core::local_api_socket::http_response_is_success(response) + && openshell_core::local_api_socket::contains_ascii(response, b"Libpod-Api-Version:") + }) +} + +#[must_use] +pub fn is_available() -> bool { + detect_socket().is_some() +} + /// Resolve the socket to connect to: explicit configuration wins, otherwise /// fall back to `detect`. Returns an error if neither resolves. /// -/// Takes `detect` as a parameter (rather than calling -/// [`openshell_core::config::detect_podman_socket`] directly) so tests can +/// Takes `detect` as a parameter so tests can /// exercise the precedence deterministically, without touching real /// environment variables or the filesystem. fn resolve_socket_path( @@ -299,10 +330,7 @@ impl PodmanComputeDriver { const MAX_PING_RETRIES: u32 = 5; const PING_RETRY_DELAY: Duration = Duration::from_secs(2); - let socket_path = resolve_socket_path( - config.socket_path.clone(), - openshell_core::config::detect_podman_socket, - )?; + let socket_path = resolve_socket_path(config.socket_path.clone(), detect_socket)?; config.socket_path = Some(socket_path.clone()); if !socket_path.exists() { diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 28dc348800..81f78731f6 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -3599,7 +3599,7 @@ async fn connect_local_container_engine() -> Option { return Some(docker); } - let podman_socket = openshell_core::config::detect_podman_socket()?; + let podman_socket = detect_podman_socket()?; if let Ok(docker) = Docker::connect_with_unix(podman_socket.to_str()?, 120, bollard::API_DEFAULT_VERSION) && docker.ping().await.is_ok() @@ -3614,6 +3614,31 @@ async fn connect_local_container_engine() -> Option { None } +fn detect_podman_socket() -> Option { + let mut candidates = Vec::new(); + if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") + && !path.trim().is_empty() + { + candidates.push(PathBuf::from(path)); + } + if let Some(runtime_dir) = std::env::var_os("XDG_RUNTIME_DIR") { + candidates.push(PathBuf::from(runtime_dir).join("podman/podman.sock")); + } + #[cfg(target_os = "linux")] + candidates.push(PathBuf::from(format!( + "/run/user/{}/podman/podman.sock", + rustix::process::geteuid().as_raw() + ))); + if let Some(home) = std::env::var_os("HOME") { + candidates + .push(PathBuf::from(home).join(".local/share/containers/podman/machine/podman.sock")); + } + openshell_core::local_api_socket::first_responsive_socket(&candidates, |response| { + openshell_core::local_api_socket::http_response_is_success(response) + && openshell_core::local_api_socket::contains_ascii(response, b"Libpod-Api-Version:") + }) +} + fn is_openshell_local_build_image_ref(image_ref: &str) -> bool { image_ref.starts_with("openshell/sandbox-from:") } diff --git a/crates/openshell-gateway/Cargo.toml b/crates/openshell-gateway/Cargo.toml new file mode 100644 index 0000000000..524d6b0515 --- /dev/null +++ b/crates/openshell-gateway/Cargo.toml @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-gateway" +description = "OpenShell gateway binary composition" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +repository.workspace = true + +[[bin]] +name = "openshell-gateway" +path = "src/main.rs" + +[dependencies] +openshell-core = { path = "../openshell-core", default-features = false } +openshell-server = { path = "../openshell-server", default-features = false } +openshell-otel = { path = "../openshell-otel", optional = true } +async-trait = "0.1" +miette = { workspace = true } +tokio = { workspace = true } + +[target.'cfg(not(target_os = "windows"))'.dependencies] +openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } +openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes", optional = true } +openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } +hyper-util = { workspace = true, optional = true } +nix = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +rustix = { workspace = true, optional = true } +tonic = { workspace = true, optional = true } +tower = { workspace = true, optional = true } +tracing = { workspace = true, optional = true } + +[features] +default = ["telemetry", "in-tree-compute-drivers"] +in-tree-compute-drivers = [ + "dep:openshell-driver-docker", + "dep:openshell-driver-kubernetes", + "dep:openshell-driver-podman", + "dep:openshell-otel", + "dep:hyper-util", + "dep:nix", + "dep:serde", + "dep:rustix", + "dep:tonic", + "dep:tower", + "dep:tracing", +] +telemetry = ["openshell-core/telemetry", "openshell-server/telemetry"] +bundled-z3 = ["openshell-server/bundled-z3"] + +[lints] +workspace = true + +[dev-dependencies] +tempfile = "3" diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs new file mode 100644 index 0000000000..cc37d811d8 --- /dev/null +++ b/crates/openshell-gateway/src/lib.rs @@ -0,0 +1,260 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Standard gateway binary composition. +//! +//! The server remains backend-agnostic. This crate is the composition boundary +//! that links first-party compute drivers into the distributed gateway binary. + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +mod vm; + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +use openshell_server::ComputeDriverRegistration; +use openshell_server::ComputeDriverRegistry; + +/// Install every first-party compute driver linked into the standard gateway. +#[must_use] +pub fn install_default_compute_drivers() -> ComputeDriverRegistry { + #[allow(unused_mut)] + let mut registry = ComputeDriverRegistry::new(); + #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] + install_in_tree_compute_drivers(&mut registry); + registry +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn install_in_tree_compute_drivers(registry: &mut ComputeDriverRegistry) { + for registration in [ + ComputeDriverRegistration::new( + "kubernetes", + 100, + Some(|| std::env::var_os("KUBERNETES_SERVICE_HOST").is_some()), + KubernetesFactory, + ) + .map(|registration| { + registration + .without_mtls_user_auth() + .with_inherited_config_keys(&[ + "namespace", + "default_image", + "supervisor_image", + "client_tls_secret_name", + "service_account_name", + "host_gateway_ip", + "enable_user_namespaces", + "sa_token_ttl_secs", + ]) + }), + ComputeDriverRegistration::new( + "podman", + 200, + Some(openshell_driver_podman::driver::is_available), + PodmanFactory, + ) + .map(|registration| { + registration + .with_local_singleplayer() + .with_inherited_config_keys(&[ + "default_image", + "supervisor_image", + "host_gateway_ip", + "guest_tls_ca", + "guest_tls_cert", + "guest_tls_key", + ]) + }), + ComputeDriverRegistration::new( + "docker", + 300, + Some(openshell_driver_docker::is_available), + DockerFactory, + ) + .map(|registration| { + registration + .with_local_singleplayer() + .with_inherited_config_keys(&[ + "sandbox_namespace", + "default_image", + "supervisor_image", + "host_gateway_ip", + "guest_tls_ca", + "guest_tls_cert", + "guest_tls_key", + ]) + }), + ComputeDriverRegistration::new("vm", u16::MAX, None, VmFactory).map(|registration| { + registration + .with_local_singleplayer() + .with_inherited_config_keys(&[ + "default_image", + "guest_tls_ca", + "guest_tls_cert", + "guest_tls_key", + ]) + }), + ] { + registry + .install(registration.expect("first-party driver name is valid")) + .expect("first-party driver names are unique"); + } +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[derive(Clone, Copy)] +struct KubernetesFactory; + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[async_trait::async_trait] +impl openshell_server::ComputeDriverFactory for KubernetesFactory { + async fn build( + &self, + context: openshell_server::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + let mut config: openshell_driver_kubernetes::KubernetesComputeConfig = + context.driver_config()?; + if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { + config.workspace_default_storage_size = size; + } + if let Ok(storage_class) = std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") { + config.workspace_storage_class = storage_class; + } + if context + .gateway_config() + .gateway_jwt + .as_ref() + .is_some_and(|jwt| jwt.ttl_secs == 0) + { + tracing::warn!( + "Kubernetes gateway configured with non-expiring sandbox JWTs; set gateway_jwt.ttl_secs > 0 for shared deployments" + ); + } + let driver = openshell_driver_kubernetes::KubernetesComputeDriver::new( + config, + context.shutdown_receiver(), + ) + .await + .map_err(|error| openshell_core::Error::execution(error.to_string()))?; + let allowlist = driver.operator_allowlist().cloned(); + let driver = openshell_driver_kubernetes::ComputeDriverService::new(driver); + context + .finish_in_process_with_allowlist(std::sync::Arc::new(driver), allowlist) + .await + } +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[derive(Clone, Copy)] +struct DockerFactory; + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[async_trait::async_trait] +impl openshell_server::ComputeDriverFactory for DockerFactory { + async fn build( + &self, + context: openshell_server::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + let mut config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?; + apply_guest_tls( + &mut config.guest_tls_ca, + &mut config.guest_tls_cert, + &mut config.guest_tls_key, + context.guest_tls_paths(), + ); + let driver = + openshell_driver_docker::DockerComputeDriver::new(context.gateway_config(), &config) + .await + .map_err(|error| openshell_core::Error::execution(error.to_string()))?; + context.finish_in_process(std::sync::Arc::new(driver)).await + } +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[derive(Clone, Copy)] +struct PodmanFactory; + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[async_trait::async_trait] +impl openshell_server::ComputeDriverFactory for PodmanFactory { + async fn build( + &self, + context: openshell_server::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + let mut config: openshell_driver_podman::PodmanComputeConfig = context.driver_config()?; + config.gateway_port = context.gateway_port(); + if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") { + config.socket_path = Some(path.into()); + } + if let Ok(ip) = std::env::var("OPENSHELL_PODMAN_HOST_GATEWAY_IP") { + config.host_gateway_ip = ip; + } + if let Ok(mode) = std::env::var("OPENSHELL_PODMAN_USERNS") { + config.userns = Some(mode); + } + apply_guest_tls( + &mut config.guest_tls_ca, + &mut config.guest_tls_cert, + &mut config.guest_tls_key, + context.guest_tls_paths(), + ); + let driver = openshell_driver_podman::PodmanComputeDriver::new(config) + .await + .map_err(|error| openshell_core::Error::execution(error.to_string()))?; + let driver = openshell_driver_podman::ComputeDriverService::new(driver); + context.finish_in_process(std::sync::Arc::new(driver)).await + } +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[derive(Clone, Copy)] +struct VmFactory; + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +#[async_trait::async_trait] +impl openshell_server::ComputeDriverFactory for VmFactory { + async fn build( + &self, + context: openshell_server::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + let mut config: vm::VmComputeConfig = context.driver_config()?; + if config.state_dir.as_os_str().is_empty() { + config.state_dir = vm::VmComputeConfig::default_state_dir(); + } + if config.grpc_endpoint.trim().is_empty() + && (!context.gateway_tls_enabled() || context.guest_tls_paths().is_some()) + { + let scheme = if context.gateway_tls_enabled() { + "https" + } else { + "http" + }; + config.grpc_endpoint = format!("{scheme}://127.0.0.1:{}", context.gateway_port()); + } + apply_guest_tls( + &mut config.guest_tls_ca, + &mut config.guest_tls_cert, + &mut config.guest_tls_key, + context.guest_tls_paths(), + ); + let endpoint = vm::spawn(context.gateway_config(), &config, context.otlp_config()).await?; + context.finish_remote(endpoint).await + } +} + +#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] +fn apply_guest_tls( + ca: &mut Option, + cert: &mut Option, + key: &mut Option, + defaults: Option<(&std::path::Path, &std::path::Path, &std::path::Path)>, +) { + if ca.is_none() + && cert.is_none() + && key.is_none() + && let Some((default_ca, default_cert, default_key)) = defaults + { + *ca = Some(default_ca.to_owned()); + *cert = Some(default_cert.to_owned()); + *key = Some(default_key.to_owned()); + } +} diff --git a/crates/openshell-server/src/main.rs b/crates/openshell-gateway/src/main.rs similarity index 59% rename from crates/openshell-server/src/main.rs rename to crates/openshell-gateway/src/main.rs index c76761016d..85d0611867 100644 --- a/crates/openshell-server/src/main.rs +++ b/crates/openshell-gateway/src/main.rs @@ -1,14 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! `OpenShell` Gateway binary entrypoint. - -use miette::Result; - #[tokio::main] -async fn main() -> Result<()> { +async fn main() -> miette::Result<()> { openshell_server::cli::run_cli_with_compute_drivers( - openshell_server::install_default_compute_drivers(), + openshell_gateway::install_default_compute_drivers(), ) .await } diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-gateway/src/vm.rs similarity index 94% rename from crates/openshell-server/src/compute/vm.rs rename to crates/openshell-gateway/src/vm.rs index 80b445d201..2a95f9ddc5 100644 --- a/crates/openshell-server/src/compute/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -29,19 +29,19 @@ //! trait implementation registering the VM driver against the generic //! interface. -use super::AcquiredRemoteDriverEndpoint; -#[cfg(unix)] -use super::ManagedDriverProcess; -use crate::config_file::OtlpConfig; -#[cfg(unix)] -use crate::otel_tracing::TraceContextInterceptor; #[cfg(unix)] use hyper_util::rt::TokioIo; #[cfg(unix)] use openshell_core::proto::compute::v1::{ GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, }; -use openshell_core::{ComputeDriverKind, Config, Error, Result}; +use openshell_core::{Config, Error, Result}; +#[cfg(unix)] +use openshell_otel::TraceContextInterceptor; +use openshell_server::AcquiredRemoteDriverEndpoint; +#[cfg(unix)] +use openshell_server::ManagedDriverProcess; +use openshell_server::config_file::OtlpConfig; #[cfg(unix)] use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; #[cfg(unix)] @@ -513,10 +513,8 @@ pub async fn spawn( })?; let channel = wait_for_compute_driver(&socket_path, &mut child).await?; let process = Arc::new(ManagedDriverProcess::new(child, socket_path)); - Ok(AcquiredRemoteDriverEndpoint::managed_builtin( - ComputeDriverKind::Vm, - channel, - process, + Ok(AcquiredRemoteDriverEndpoint::managed( + "vm", channel, process, )) } @@ -612,9 +610,8 @@ mod tests { VmComputeConfig, append_otlp_args, compute_driver_guest_tls_paths, compute_driver_socket_path, current_euid, prepare_compute_driver_socket_path, prepare_vm_state_dir, resolve_compute_driver_bin, resolve_driver_search_dirs, - wait_for_compute_driver, }; - use crate::config_file::OtlpConfig; + use openshell_server::config_file::OtlpConfig; use std::os::unix::fs::PermissionsExt; use std::os::unix::net::UnixListener as StdUnixListener; use std::path::PathBuf; @@ -639,43 +636,6 @@ mod tests { assert_eq!(args, ["--otlp-endpoint", "http://collector.internal:4317"]); } - #[tokio::test] - async fn readiness_probe_propagates_the_active_trace() { - use crate::otel_tracing::test_exporter; - use crate::test_support::FakeComputeDriver; - - let dir = tempdir().unwrap(); - let socket_path = dir.path().join("compute-driver.sock"); - let driver = FakeComputeDriver::new(); - let _server = driver.serve_uds(&socket_path).unwrap(); - let mut child = tokio::process::Command::new("sh") - .arg("-c") - .arg("read _") - .stdin(std::process::Stdio::piped()) - .kill_on_drop(true) - .spawn() - .unwrap(); - - let traced = test_exporter::install_traced(); - wait_for_compute_driver(&socket_path, &mut child) - .await - .unwrap(); - - let readiness = traced.spans_named("driver.wait_for_ready"); - assert_eq!(readiness.len(), 1, "one readiness operation should finish"); - test_exporter::assert_is_root(&readiness[0]); - let trace_id = readiness[0].span_context.trace_id().to_string(); - assert_eq!( - driver.traceparents().len(), - 1, - "the readiness capability probe should carry trace context" - ); - assert!( - driver.traceparents()[0].contains(&trace_id), - "the readiness probe should be part of the active trace" - ); - } - #[test] fn resolve_driver_bin_uses_driver_dir_when_binary_present() { let dir = tempdir().unwrap(); diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 215f6abfa4..bf40cae773 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -10,10 +10,6 @@ rust-version.workspace = true license.workspace = true repository.workspace = true -[[bin]] -name = "openshell-gateway" -path = "src/main.rs" - [dependencies] openshell-bootstrap = { path = "../openshell-bootstrap" } openshell-core = { path = "../openshell-core", default-features = false } @@ -114,20 +110,8 @@ x509-parser = "0.16" arc-swap = "1" notify = "8" -[target.'cfg(not(target_os = "windows"))'.dependencies] -openshell-driver-docker = { path = "../openshell-driver-docker", optional = true } -openshell-driver-kubernetes = { path = "../openshell-driver-kubernetes", optional = true } -openshell-driver-podman = { path = "../openshell-driver-podman", optional = true } - [features] -default = ["telemetry", "in-tree-compute-drivers"] -## Link the first-party compute drivers into the standard gateway binary. -## Disable this feature for a protocol-only gateway that uses external drivers. -in-tree-compute-drivers = [ - "dep:openshell-driver-docker", - "dep:openshell-driver-kubernetes", - "dep:openshell-driver-podman", -] +default = ["telemetry"] ## Compile in anonymous telemetry emission (forwards to openshell-core/telemetry). ## On by default; build with `--no-default-features` for a telemetry-free gateway ## that contains no telemetry endpoint, HTTP client, or emission code. diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index 131dbaba47..03dc692205 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -26,7 +26,7 @@ use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; -use openshell_core::OperatorNamespaceAllowlist; +use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist; use std::sync::Arc; use tonic::Status; use tracing::{debug, info, warn}; diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index ae831d2016..06d2dd59ad 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -6,7 +6,6 @@ use clap::parser::ValueSource; use clap::{ArgAction, ArgMatches, Command, CommandFactory, FromArgMatches, Parser}; use miette::{IntoDiagnostic, Result}; -use openshell_core::ComputeDriverKind; use openshell_core::config::DEFAULT_SERVER_PORT; use std::net::{IpAddr, SocketAddr}; use std::path::PathBuf; @@ -17,10 +16,7 @@ use crate::certgen; use crate::compute::driver_config::GuestTlsPaths; use crate::config_file::{self, ConfigFile, GatewayFileSection}; use crate::defaults::{self, LocalTlsPaths}; -use crate::{ - ComputeDriverRegistry, ServerStartupConfig, install_default_compute_drivers, run_server, - tracing_bus::TracingLogBus, -}; +use crate::{ComputeDriverRegistry, ServerStartupConfig, run_server, tracing_bus::TracingLogBus}; /// `OpenShell` gateway process - gRPC and HTTP server with protocol multiplexing. /// @@ -222,7 +218,7 @@ pub fn command() -> Command { } pub async fn run_cli() -> Result<()> { - run_cli_with_compute_drivers(install_default_compute_drivers()).await + run_cli_with_compute_drivers(ComputeDriverRegistry::new()).await } /// Run the gateway CLI with the compute drivers linked by the binary. @@ -240,7 +236,16 @@ pub async fn run_cli_with_compute_drivers(compute_drivers: ComputeDriverRegistry } } +#[cfg(test)] fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result { + prepare_server_config_with_drivers(args, matches, &ComputeDriverRegistry::new()) +} + +fn prepare_server_config_with_drivers( + args: &mut RunArgs, + matches: &ArgMatches, + compute_drivers: &ComputeDriverRegistry, +) -> Result { // Load TOML when explicitly requested, or from the default XDG location // when that file exists. Missing default config is not an error: runtime // defaults and OPENSHELL_* env vars are enough for package-managed starts. @@ -263,7 +268,8 @@ fn prepare_server_config(args: &mut RunArgs, matches: &ArgMatches) -> Result Result Result<()> { - let prepared = prepare_server_config(&mut args, &matches)?; + let prepared = prepare_server_config_with_drivers(&mut args, &matches, &compute_drivers)?; let tracing_log_bus = TracingLogBus::new(); let otlp_config = prepared @@ -785,25 +789,34 @@ fn normalize_compute_driver_socket_args(args: &mut RunArgs, matches: &ArgMatches } } -fn effective_single_driver(args: &RunArgs) -> Option { +fn effective_single_driver(args: &RunArgs) -> Option<&str> { match args.drivers.as_slice() { - [] => openshell_core::config::detect_driver(), - [driver] => driver.parse().ok(), + [driver] => Some(driver), _ => None, } } -fn is_singleplayer_driver(args: &RunArgs) -> bool { - matches!( - effective_single_driver(args), - Some(ComputeDriverKind::Docker | ComputeDriverKind::Podman | ComputeDriverKind::Vm) - ) +fn effective_registration<'a>( + args: &RunArgs, + compute_drivers: &'a ComputeDriverRegistry, +) -> Option<&'a crate::ComputeDriverRegistration> { + match effective_single_driver(args) { + Some(name) => compute_drivers.get(name), + None if args.drivers.is_empty() => compute_drivers.detect(), + None => None, + } +} + +fn is_singleplayer_driver(args: &RunArgs, compute_drivers: &ComputeDriverRegistry) -> bool { + effective_registration(args, compute_drivers) + .is_some_and(crate::ComputeDriverRegistration::is_local_singleplayer) } fn resolve_mtls_auth_enabled( args: &RunArgs, matches: &ArgMatches, file: Option<&ConfigFile>, + compute_drivers: &ComputeDriverRegistry, ) -> bool { let file_configured = file .and_then(|f| f.openshell.gateway.mtls_auth.as_ref()) @@ -816,7 +829,7 @@ fn resolve_mtls_auth_enabled( return false; } - is_singleplayer_driver(args) + is_singleplayer_driver(args, compute_drivers) } #[cfg(test)] @@ -826,6 +839,33 @@ mod tests { use clap::Parser; use std::net::{IpAddr, Ipv4Addr}; + #[derive(Clone, Copy)] + struct TestFactory; + + #[async_trait::async_trait] + impl crate::ComputeDriverFactory for TestFactory { + async fn build( + &self, + _context: crate::ComputeDriverBuildContext<'_>, + ) -> openshell_core::Result { + unreachable!("CLI metadata tests do not build drivers") + } + } + + fn test_registry(name: &str, singleplayer: bool, mtls: bool) -> crate::ComputeDriverRegistry { + let mut registration = + crate::ComputeDriverRegistration::new(name, 100, None, TestFactory).unwrap(); + if singleplayer { + registration = registration.with_local_singleplayer(); + } + if !mtls { + registration = registration.without_mtls_user_auth(); + } + let mut registry = crate::ComputeDriverRegistry::new(); + registry.install(registration).unwrap(); + registry + } + struct EnvVarGuard { key: &'static str, original: Option, @@ -1283,7 +1323,7 @@ mod tests { "--db-url", "sqlite::memory:", "--drivers", - "docker", + "local", "--tls-cert", "/tmp/server.crt", "--tls-key", @@ -1292,11 +1332,16 @@ mod tests { "/tmp/ca.crt", ]); - assert!(super::resolve_mtls_auth_enabled(&args, &matches, None)); + assert!(super::resolve_mtls_auth_enabled( + &args, + &matches, + None, + &test_registry("local", true, true) + )); } #[test] - fn mtls_auth_does_not_auto_default_for_kubernetes_driver() { + fn mtls_auth_does_not_auto_default_for_shared_driver() { let _lock = ENV_LOCK .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); @@ -1307,7 +1352,7 @@ mod tests { "--db-url", "sqlite::memory:", "--drivers", - "kubernetes", + "shared", "--tls-cert", "/tmp/server.crt", "--tls-key", @@ -1316,7 +1361,12 @@ mod tests { "/tmp/ca.crt", ]); - assert!(!super::resolve_mtls_auth_enabled(&args, &matches, None)); + assert!(!super::resolve_mtls_auth_enabled( + &args, + &matches, + None, + &test_registry("shared", false, false) + )); } #[test] @@ -1331,7 +1381,7 @@ mod tests { "--db-url", "sqlite::memory:", "--drivers", - "docker", + "local", "--tls-cert", "/tmp/server.crt", "--tls-key", @@ -1351,7 +1401,8 @@ enabled = false assert!(!super::resolve_mtls_auth_enabled( &args, &matches, - Some(&file) + Some(&file), + &test_registry("local", true, true) )); } @@ -1578,38 +1629,30 @@ ssh_session_ttl_secs = 1234 } #[test] - fn singleplayer_driver_matches_only_one_local_driver() { - for driver in ["docker", "podman", "vm"] { - let (args, _) = parse_with_args(&[ - "openshell-gateway", - "--db-url", - "sqlite::memory:", - "--drivers", - driver, - ]); - assert!( - super::is_singleplayer_driver(&args), - "{driver} should be singleplayer" - ); - } - - let (k8s, _) = parse_with_args(&[ + fn singleplayer_behavior_comes_from_registration() { + let (local, _) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", "--drivers", - "kubernetes", + "local", ]); - assert!(!super::is_singleplayer_driver(&k8s)); + assert!(super::is_singleplayer_driver( + &local, + &test_registry("local", true, true) + )); let (multi, _) = parse_with_args(&[ "openshell-gateway", "--db-url", "sqlite::memory:", "--drivers", - "docker,podman", + "alpha,beta", ]); - assert!(!super::is_singleplayer_driver(&multi)); + assert!(!super::is_singleplayer_driver( + &multi, + &test_registry("alpha", true, true) + )); } #[test] @@ -1635,7 +1678,7 @@ ssh_session_ttl_secs = 1234 Some(std::path::Path::new("/run/openshell/kyma.sock")) ); assert_eq!(args.drivers, ["kyma"]); - assert!(super::effective_single_driver(&args).is_none()); + assert_eq!(super::effective_single_driver(&args), Some("kyma")); } #[test] @@ -1827,53 +1870,4 @@ mem_mib = "not-a-number" assert!(file.openshell.drivers.contains_key("docker")); assert!(file.openshell.drivers.contains_key("vm")); } - - #[test] - #[cfg(not(target_os = "windows"))] - fn driver_inherits_shared_image_from_gateway_section() { - // [openshell.gateway].default_image inherits into the K8s driver - // table when the driver-specific table does not set it. - let file = config_file_from_toml( - r#" -[openshell.gateway] -default_image = "ghcr.io/nvidia/openshell/sandbox:1.0" - -[openshell.drivers.kubernetes] -namespace = "agents" -"#, - ); - let merged = crate::config_file::driver_table( - super::ComputeDriverKind::Kubernetes.as_str(), - &file.openshell.gateway, - file.openshell.drivers.get("kubernetes"), - ); - let parsed = merged - .try_into::() - .expect("merged table deserializes"); - assert_eq!(parsed.default_image, "ghcr.io/nvidia/openshell/sandbox:1.0"); - assert_eq!(parsed.namespace, "agents"); - } - - #[test] - #[cfg(not(target_os = "windows"))] - fn driver_specific_value_overrides_gateway_inheritance() { - let file = config_file_from_toml( - r#" -[openshell.gateway] -default_image = "gateway-default:1.0" - -[openshell.drivers.kubernetes] -default_image = "k8s-specific:1.0" -"#, - ); - let merged = crate::config_file::driver_table( - super::ComputeDriverKind::Kubernetes.as_str(), - &file.openshell.gateway, - file.openshell.drivers.get("kubernetes"), - ); - let parsed = merged - .try_into::() - .expect("deserializes"); - assert_eq!(parsed.default_image, "k8s-specific:1.0"); - } } diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index fad71efd36..98d798eed3 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -7,9 +7,6 @@ //! driver-specific environment overrides, and applying gateway startup defaults. //! It does not acquire, connect to, or start compute drivers. -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub mod builtin; - use crate::config_file; use crate::defaults::LocalTlsPaths; use openshell_core::{Error, Result}; @@ -108,10 +105,17 @@ pub fn kubernetes_sa_bootstrap_config( "K8s ServiceAccount bootstrap requires [openshell.drivers.kubernetes] when sandbox JWT issuing is enabled in-cluster", )); } - let merged = config_file::driver_table( + let merged = config_file::driver_table_with_inherited_keys( "kubernetes", &file.openshell.gateway, file.openshell.drivers.get("kubernetes"), + &[ + "namespace", + "service_account_name", + "host_gateway_ip", + "enable_user_namespaces", + "sa_token_ttl_secs", + ], ); merged.try_into().map_err(|error| { Error::config(format!( @@ -123,16 +127,18 @@ pub fn kubernetes_sa_bootstrap_config( pub fn driver_config_from_context( context: DriverStartupContext<'_>, driver_name: &str, + inherited_config_keys: &[&str], ) -> Result where T: Default + serde::de::DeserializeOwned, { - driver_config_from_file(context.file, driver_name) + driver_config_from_file(context.file, driver_name, inherited_config_keys) } fn driver_config_from_file( file: Option<&config_file::ConfigFile>, driver_name: &str, + inherited_config_keys: &[&str], ) -> Result where T: Default + serde::de::DeserializeOwned, @@ -140,10 +146,11 @@ where let Some(file) = file else { return Ok(T::default()); }; - let merged = config_file::driver_table( + let merged = config_file::driver_table_with_inherited_keys( driver_name, &file.openshell.gateway, file.openshell.drivers.get(driver_name), + inherited_config_keys, ); merged.try_into().map_err(|e| { Error::config(format!( diff --git a/crates/openshell-server/src/compute/driver_config/builtin.rs b/crates/openshell-server/src/compute/driver_config/builtin.rs deleted file mode 100644 index dea867237d..0000000000 --- a/crates/openshell-server/src/compute/driver_config/builtin.rs +++ /dev/null @@ -1,231 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Configuration construction for built-in compute drivers. - -use super::{DriverStartupContext, GuestTlsPaths, driver_config_from_context}; -use crate::compute::VmComputeConfig; -#[cfg(test)] -use crate::config_file; -use openshell_core::{ComputeDriverKind, Result}; -use openshell_driver_docker::DockerComputeConfig; -use openshell_driver_kubernetes::KubernetesComputeConfig; -use openshell_driver_podman::PodmanComputeConfig; -use std::path::PathBuf; - -/// Build the selected Kubernetes config from TOML plus runtime defaults. -pub fn kubernetes_config_from_context( - context: DriverStartupContext<'_>, -) -> Result { - let mut cfg = driver_config_from_context(context, ComputeDriverKind::Kubernetes.as_str())?; - apply_kubernetes_runtime_defaults(&mut cfg); - Ok(cfg) -} - -/// Build the selected Podman config from TOML plus runtime defaults. -pub fn podman_config_from_context( - context: DriverStartupContext<'_>, -) -> Result { - let mut podman = driver_config_from_context(context, ComputeDriverKind::Podman.as_str())?; - apply_podman_runtime_defaults(&mut podman, context); - Ok(podman) -} - -/// Build the selected Docker config from TOML plus runtime defaults. -pub fn docker_config_from_context( - context: DriverStartupContext<'_>, -) -> Result { - let mut cfg = driver_config_from_context(context, ComputeDriverKind::Docker.as_str())?; - apply_docker_runtime_defaults(&mut cfg, context); - Ok(cfg) -} - -/// Build the selected VM config from TOML plus runtime defaults. -pub fn vm_config_from_context(context: DriverStartupContext<'_>) -> Result { - let mut cfg = driver_config_from_context(context, ComputeDriverKind::Vm.as_str())?; - apply_vm_runtime_defaults(&mut cfg, context); - Ok(cfg) -} - -fn apply_kubernetes_runtime_defaults(k8s: &mut KubernetesComputeConfig) { - if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { - k8s.workspace_default_storage_size = size; - } - if let Ok(storage_class) = std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") { - k8s.workspace_storage_class = storage_class; - } -} - -fn apply_podman_runtime_defaults( - podman: &mut PodmanComputeConfig, - context: DriverStartupContext<'_>, -) { - podman.gateway_port = context.gateway_port; - apply_podman_env_overrides(podman); - apply_guest_tls_defaults_to_split_fields( - &mut podman.guest_tls_ca, - &mut podman.guest_tls_cert, - &mut podman.guest_tls_key, - context.guest_tls, - ); -} - -fn apply_docker_runtime_defaults(cfg: &mut DockerComputeConfig, context: DriverStartupContext<'_>) { - apply_guest_tls_defaults_to_split_fields( - &mut cfg.guest_tls_ca, - &mut cfg.guest_tls_cert, - &mut cfg.guest_tls_key, - context.guest_tls, - ); -} - -fn apply_vm_runtime_defaults(cfg: &mut VmComputeConfig, context: DriverStartupContext<'_>) { - if cfg.state_dir.as_os_str().is_empty() { - cfg.state_dir = VmComputeConfig::default_state_dir(); - } - if cfg.grpc_endpoint.trim().is_empty() - && (!context.gateway_tls_enabled || context.guest_tls.is_some()) - { - let scheme = if context.gateway_tls_enabled { - "https" - } else { - "http" - }; - cfg.grpc_endpoint = format!("{scheme}://127.0.0.1:{}", context.gateway_port); - } - - apply_guest_tls_defaults_to_split_fields( - &mut cfg.guest_tls_ca, - &mut cfg.guest_tls_cert, - &mut cfg.guest_tls_key, - context.guest_tls, - ); -} - -fn apply_guest_tls_defaults_to_split_fields( - ca: &mut Option, - cert: &mut Option, - key: &mut Option, - defaults: Option<&GuestTlsPaths>, -) { - if ca.is_none() - && cert.is_none() - && key.is_none() - && let Some(paths) = defaults - { - *ca = Some(paths.ca.clone()); - *cert = Some(paths.cert.clone()); - *key = Some(paths.key.clone()); - } -} - -fn apply_podman_env_overrides(podman: &mut PodmanComputeConfig) { - if let Ok(p) = std::env::var("OPENSHELL_PODMAN_SOCKET") { - podman.socket_path = Some(PathBuf::from(p)); - } - if let Ok(ip) = std::env::var("OPENSHELL_PODMAN_HOST_GATEWAY_IP") { - podman.host_gateway_ip = ip; - } - if let Ok(mode) = std::env::var("OPENSHELL_PODMAN_USERNS") { - podman.userns = Some(mode); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::BTreeMap; - - fn test_context(file: Option<&config_file::ConfigFile>) -> DriverStartupContext<'_> { - static EMPTY_ENDPOINT_OVERRIDES: std::sync::LazyLock> = - std::sync::LazyLock::new(BTreeMap::new); - DriverStartupContext { - file, - guest_tls: None, - gateway_port: openshell_core::config::DEFAULT_SERVER_PORT, - gateway_tls_enabled: false, - endpoint_overrides: &EMPTY_ENDPOINT_OVERRIDES, - } - } - - #[test] - fn podman_config_reads_bind_mount_opt_in_from_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r" -[openshell.drivers.podman] -enable_bind_mounts = true -", - ) - .expect("valid config"); - - let cfg = podman_config_from_context(test_context(Some(&file))).expect("podman config"); - - assert!(cfg.enable_bind_mounts); - } - - #[test] - fn docker_config_reads_bind_mount_opt_in_from_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r" -[openshell.drivers.docker] -enable_bind_mounts = true -", - ) - .expect("valid config"); - - let cfg = docker_config_from_context(test_context(Some(&file))).expect("docker config"); - - assert!(cfg.enable_bind_mounts); - } - - #[test] - fn docker_config_reads_socket_path_from_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r#" -[openshell.drivers.docker] -socket_path = "/tmp/docker.sock" -"#, - ) - .expect("valid config"); - - let cfg = docker_config_from_context(test_context(Some(&file))).expect("docker config"); - - assert_eq!(cfg.socket_path, Some(PathBuf::from("/tmp/docker.sock"))); - } - - #[test] - fn docker_config_reports_selected_invalid_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r" -[openshell.drivers.docker] -unknown_docker_key = true -", - ) - .expect("valid config"); - - let err = docker_config_from_context(test_context(Some(&file))).unwrap_err(); - - assert!( - err.to_string() - .contains("invalid [openshell.drivers.docker] table") - ); - } - - #[test] - fn vm_config_reports_selected_invalid_driver_table() { - let file: config_file::ConfigFile = toml::from_str( - r#" -[openshell.drivers.vm] -mem_mib = "not-a-number" -"#, - ) - .expect("valid config"); - - let err = vm_config_from_context(test_context(Some(&file))).unwrap_err(); - - assert!( - err.to_string() - .contains("invalid [openshell.drivers.vm] table") - ); - } -} diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 4d5c9447b5..3b1d03e1d2 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -5,17 +5,6 @@ pub mod driver_config; pub mod lease; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub mod vm; - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub use openshell_driver_docker::DockerComputeConfig; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub use openshell_driver_kubernetes::KubernetesComputeConfig; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub use openshell_driver_podman::PodmanComputeConfig; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -pub use vm::VmComputeConfig; use crate::grpc::policy::SANDBOX_SETTINGS_OBJECT_TYPE; use crate::otel_tracing::TraceContextInterceptor; @@ -30,7 +19,6 @@ use crate::tracing_bus::TracingLogBus; use futures::{Stream, StreamExt}; #[cfg(unix)] use hyper_util::rt::TokioIo; -use openshell_core::ComputeDriverKind; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, DeleteSandboxRequest, DeleteWorkspaceRequest, DeleteWorkspaceResponse, DriverCondition, DriverPlatformEvent, DriverResourceRequirements, DriverSandbox, @@ -49,15 +37,6 @@ use openshell_core::proto::{ SandboxTemplate, ServiceEndpoint, SshSession, }; use openshell_core::{ObjectLabels, ObjectWorkspace}; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -use openshell_driver_docker::DockerComputeDriver; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -use openshell_driver_kubernetes::{ - ComputeDriverService as KubernetesDriverService, KubernetesComputeDriver, - OperatorNamespaceAllowlist, -}; -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -use openshell_driver_podman::{ComputeDriverService as PodmanDriverService, PodmanComputeDriver}; use prost::Message; use std::collections::HashMap; use std::fmt; @@ -296,8 +275,8 @@ pub struct ManagedDriverProcess { } impl ManagedDriverProcess { - #[cfg(all(unix, any(test, feature = "in-tree-compute-drivers")))] - pub(crate) fn new(child: tokio::process::Child, socket_path: PathBuf) -> Self { + #[cfg(unix)] + pub fn new(child: tokio::process::Child, socket_path: PathBuf) -> Self { Self { child: std::sync::Mutex::new(Some(child)), socket_path, @@ -394,14 +373,13 @@ pub struct AcquiredRemoteDriverEndpoint { } impl AcquiredRemoteDriverEndpoint { - #[cfg(any(test, feature = "in-tree-compute-drivers"))] - pub(crate) fn managed_builtin( - driver_kind: ComputeDriverKind, + pub fn managed( + name: impl Into, channel: Channel, driver_process: Arc, ) -> Self { Self { - name: driver_kind.as_str().to_string(), + name: name.into(), channel, driver_process: Some(driver_process), } @@ -602,11 +580,9 @@ impl ComputeRuntime { compute_error_from_status(status) })? .into_inner(); - let driver_kind = driver_name.parse::().ok(); info!( configured_driver = %driver_name, advertised_driver = %capabilities.driver_name, - in_tree = driver_kind.is_some(), "Compute driver connected" ); let driver_info = ComputeDriverInfoSnapshot { @@ -711,63 +687,6 @@ impl ComputeRuntime { self.lifecycle_gates.entry_count() } - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - pub async fn new_docker( - config: openshell_core::Config, - docker_config: DockerComputeConfig, - store: Arc, - sandbox_index: SandboxIndex, - sandbox_watch_bus: SandboxWatchBus, - tracing_log_bus: TracingLogBus, - supervisor_sessions: Arc, - ) -> Result { - let driver: SharedComputeDriver = Arc::new( - DockerComputeDriver::new(&config, &docker_config) - .await - .map_err(|err| ComputeError::Message(err.to_string()))?, - ); - Self::from_driver( - ComputeDriverKind::Docker.as_str().to_string(), - driver, - None, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - } - - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - pub async fn new_kubernetes( - config: KubernetesComputeConfig, - store: Arc, - sandbox_index: SandboxIndex, - sandbox_watch_bus: SandboxWatchBus, - tracing_log_bus: TracingLogBus, - supervisor_sessions: Arc, - shutdown_rx: watch::Receiver, - ) -> Result<(Self, Option), ComputeError> { - let driver = KubernetesComputeDriver::new(config, shutdown_rx) - .await - .map_err(|err| ComputeError::Message(err.to_string()))?; - let operator_allowlist_arc = driver.operator_allowlist().cloned(); - let driver: SharedComputeDriver = Arc::new(KubernetesDriverService::new(driver)); - let runtime = Self::from_driver( - ComputeDriverKind::Kubernetes.as_str().to_string(), - driver, - None, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await?; - Ok((runtime, operator_allowlist_arc)) - } - pub(crate) async fn new_remote_driver( endpoint: AcquiredRemoteDriverEndpoint, store: Arc, @@ -790,32 +709,6 @@ impl ComputeRuntime { .await } - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - pub async fn new_podman( - config: PodmanComputeConfig, - store: Arc, - sandbox_index: SandboxIndex, - sandbox_watch_bus: SandboxWatchBus, - tracing_log_bus: TracingLogBus, - supervisor_sessions: Arc, - ) -> Result { - let driver = PodmanComputeDriver::new(config) - .await - .map_err(|err| ComputeError::Message(err.to_string()))?; - let driver: SharedComputeDriver = Arc::new(PodmanDriverService::new(driver)); - Self::from_driver( - ComputeDriverKind::Podman.as_str().to_string(), - driver, - None, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions, - ) - .await - } - #[must_use] pub fn default_image(&self) -> &str { &self.default_image @@ -827,8 +720,8 @@ impl ComputeRuntime { } #[must_use] - pub fn driver_kind(&self) -> Option { - self.driver_info.name.parse().ok() + pub fn configured_driver_name(&self) -> &str { + &self.driver_info.name } #[must_use] diff --git a/crates/openshell-server/src/config_file.rs b/crates/openshell-server/src/config_file.rs index 00b7a2f64d..971147ccfd 100644 --- a/crates/openshell-server/src/config_file.rs +++ b/crates/openshell-server/src/config_file.rs @@ -26,7 +26,6 @@ use std::net::SocketAddr; use std::path::{Path, PathBuf}; use base64::Engine as _; -use openshell_core::config::ComputeDriverKind; use openshell_core::proto::SupervisorMiddlewareService; use openshell_core::{ GatewayAuthConfig, GatewayInterceptorConfig, GatewayJwtConfig, @@ -427,13 +426,22 @@ pub fn driver_table( driver_name: &str, gateway: &GatewayFileSection, raw: Option<&toml::Value>, +) -> toml::Value { + driver_table_with_inherited_keys(driver_name, gateway, raw, &[]) +} + +pub(crate) fn driver_table_with_inherited_keys( + _driver_name: &str, + gateway: &GatewayFileSection, + raw: Option<&toml::Value>, + inheritable_keys: &[&str], ) -> toml::Value { let mut merged = match raw { Some(toml::Value::Table(table)) => table.clone(), _ => toml::Table::new(), }; - for key in inheritable_keys(driver_name) { + for key in inheritable_keys { if merged.contains_key(*key) { continue; } @@ -445,48 +453,6 @@ pub fn driver_table( toml::Value::Table(merged) } -/// Inheritance allowlist (the Q4 "high-overlap set"). Each driver opts in -/// to a specific subset so a gateway-wide default does not accidentally land -/// in a driver table that does not understand the field. -fn inheritable_keys(driver_name: &str) -> &'static [&'static str] { - match driver_name.parse::().ok() { - Some(ComputeDriverKind::Kubernetes) => &[ - "namespace", - "default_image", - "supervisor_image", - "client_tls_secret_name", - "service_account_name", - "host_gateway_ip", - "enable_user_namespaces", - "sa_token_ttl_secs", - ], - Some(ComputeDriverKind::Docker) => &[ - "sandbox_namespace", - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ], - Some(ComputeDriverKind::Podman) => &[ - "default_image", - "supervisor_image", - "host_gateway_ip", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ], - Some(ComputeDriverKind::Vm) => &[ - "default_image", - "guest_tls_ca", - "guest_tls_cert", - "guest_tls_key", - ], - None => &[], - } -} - fn gateway_inherited_value(g: &GatewayFileSection, key: &str) -> Option { match key { "namespace" | "sandbox_namespace" => g.sandbox_namespace.as_deref().map(string_value), @@ -978,10 +944,11 @@ version = 2 let raw = toml::toml! { namespace = "agents" }; - let merged = driver_table( - ComputeDriverKind::Kubernetes.as_str(), + let merged = driver_table_with_inherited_keys( + "alpha", &gateway, Some(&toml::Value::Table(raw)), + &["default_image", "supervisor_image"], ); let table = merged.as_table().expect("table"); assert_eq!( @@ -999,14 +966,19 @@ version = 2 } #[test] - fn docker_driver_table_inherits_gateway_defaults() { + fn registered_driver_table_inherits_selected_gateway_defaults() { let gateway = GatewayFileSection { sandbox_namespace: Some("agents".to_string()), default_image: Some("ghcr.io/nvidia/openshell/sandbox:0.9".to_string()), host_gateway_ip: Some("10.0.0.1".to_string()), ..Default::default() }; - let merged = driver_table(ComputeDriverKind::Docker.as_str(), &gateway, None); + let merged = driver_table_with_inherited_keys( + "alpha", + &gateway, + None, + &["sandbox_namespace", "default_image", "host_gateway_ip"], + ); let table = merged.as_table().expect("table"); assert_eq!( table.get("sandbox_namespace").and_then(|v| v.as_str()), @@ -1023,13 +995,18 @@ version = 2 } #[test] - fn podman_driver_table_inherits_gateway_host_gateway_ip() { + fn registered_driver_table_can_select_network_defaults() { let gateway = GatewayFileSection { default_image: Some("ghcr.io/nvidia/openshell/sandbox:0.9".to_string()), host_gateway_ip: Some("192.168.127.254".to_string()), ..Default::default() }; - let merged = driver_table(ComputeDriverKind::Podman.as_str(), &gateway, None); + let merged = driver_table_with_inherited_keys( + "beta", + &gateway, + None, + &["default_image", "host_gateway_ip"], + ); let table = merged.as_table().expect("table"); assert_eq!( table.get("default_image").and_then(|v| v.as_str()), @@ -1050,10 +1027,11 @@ version = 2 let raw = toml::toml! { default_image = "driver-specific" }; - let merged = driver_table( - ComputeDriverKind::Podman.as_str(), + let merged = driver_table_with_inherited_keys( + "alpha", &gateway, Some(&toml::Value::Table(raw)), + &["default_image"], ); assert_eq!( merged @@ -1067,13 +1045,12 @@ version = 2 #[test] fn driver_table_does_not_leak_keys_outside_allowlist() { - // `client_tls_secret_name` is K8s-only; Docker must not receive it - // even when set at gateway scope. + // Fields not selected by the registration must remain gateway-only. let gateway = GatewayFileSection { client_tls_secret_name: Some("openshell-sandbox-tls".to_string()), ..Default::default() }; - let merged = driver_table(ComputeDriverKind::Docker.as_str(), &gateway, None); + let merged = driver_table_with_inherited_keys("alpha", &gateway, None, &["default_image"]); assert!( !merged .as_table() diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 6fa9959564..6d44d01ddc 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -171,7 +171,7 @@ fn emit_sandbox_create_telemetry( request: &CreateSandboxRequest, outcome: TelemetryOutcome, ) { - let compute_driver = telemetry_compute_driver(state.compute.driver_kind()); + let compute_driver = telemetry_compute_driver(state.compute.configured_driver_name()); let Some(spec) = request.spec.as_ref() else { openshell_core::telemetry::emit_sandbox_create( outcome, @@ -204,10 +204,8 @@ fn emit_sandbox_create_telemetry( ); } -fn telemetry_compute_driver( - driver_kind: Option, -) -> TelemetryComputeDriver { - TelemetryComputeDriver::from_driver_kind(driver_kind) +fn telemetry_compute_driver(driver_name: &str) -> TelemetryComputeDriver { + TelemetryComputeDriver::from_raw(driver_name) } async fn handle_create_sandbox_inner( @@ -2400,24 +2398,24 @@ mod tests { #[test] fn telemetry_compute_driver_uses_resolved_driver_kind() { assert_eq!( - telemetry_compute_driver(Some(openshell_core::ComputeDriverKind::Docker)), - TelemetryComputeDriver::Docker + telemetry_compute_driver("docker"), + TelemetryComputeDriver::from_raw("docker") ); assert_eq!( - telemetry_compute_driver(Some(openshell_core::ComputeDriverKind::Kubernetes)), - TelemetryComputeDriver::Kubernetes + telemetry_compute_driver("kubernetes"), + TelemetryComputeDriver::from_raw("kubernetes") ); assert_eq!( - telemetry_compute_driver(Some(openshell_core::ComputeDriverKind::Podman)), - TelemetryComputeDriver::Podman + telemetry_compute_driver("podman"), + TelemetryComputeDriver::from_raw("podman") ); assert_eq!( - telemetry_compute_driver(Some(openshell_core::ComputeDriverKind::Vm)), - TelemetryComputeDriver::Vm + telemetry_compute_driver("vm"), + TelemetryComputeDriver::from_raw("vm") ); assert_eq!( - telemetry_compute_driver(None), - TelemetryComputeDriver::Unknown + telemetry_compute_driver(""), + TelemetryComputeDriver::from_raw("") ); } @@ -3361,8 +3359,7 @@ mod tests { #[tokio::test] async fn create_and_get_preserve_partial_process_identity() { - let state = - test_server_state_with_driver(openshell_core::ComputeDriverKind::Docker.as_str()).await; + let state = test_server_state_with_driver("docker").await; let policy = openshell_core::proto::SandboxPolicy { version: 1, process: Some(openshell_core::proto::ProcessPolicy { @@ -3425,9 +3422,7 @@ mod tests { #[tokio::test] async fn create_and_get_preserve_partial_process_identity_for_kubernetes() { - let state = - test_server_state_with_driver(openshell_core::ComputeDriverKind::Kubernetes.as_str()) - .await; + let state = test_server_state_with_driver("kubernetes").await; let policy = openshell_core::proto::SandboxPolicy { version: 1, process: Some(openshell_core::proto::ProcessPolicy { diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index a2f26f6ae8..7e061460bb 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -48,8 +48,6 @@ mod tracing_setup; mod ws_tunnel; use metrics_exporter_prometheus::PrometheusBuilder; -#[cfg(target_os = "windows")] -use openshell_core::ComputeDriverKind; use openshell_core::net::set_tcp_nodelay_best_effort; use openshell_core::{Config, Error, ObjectLabels, Result}; use openshell_extension_core::{ @@ -1064,16 +1062,10 @@ async fn terminate_signal() { let _ = signal.recv().await; } -#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] -fn unsupported_builtin_compute_driver(driver: ComputeDriverKind) -> compute::ComputeError { - compute::ComputeError::Message(format!( - "{} compute driver is unsupported on Windows", - driver.as_str() - )) -} - -type OperatorAllowlistArc = Option; -pub use compute::{DriverWatchStream, SharedComputeDriver}; +type OperatorAllowlistArc = Option; +pub use compute::{ + AcquiredRemoteDriverEndpoint, DriverWatchStream, ManagedDriverProcess, SharedComputeDriver, +}; /// Opaque result returned by a compiled compute-driver factory. pub struct ComputeDriverBuildOutput { @@ -1097,6 +1089,9 @@ pub struct ComputeDriverRegistration { detection_priority: u16, detect: Option bool>, factory: Arc, + inherited_config_keys: &'static [&'static str], + local_singleplayer: bool, + supports_mtls_user_auth: bool, } impl std::fmt::Debug for ComputeDriverRegistration { @@ -1125,8 +1120,42 @@ impl ComputeDriverRegistration { detection_priority, detect, factory: Arc::new(factory), + inherited_config_keys: &[], + local_singleplayer: false, + supports_mtls_user_auth: true, }) } + + /// Select gateway-wide defaults understood by this driver's config type. + #[must_use] + pub fn with_inherited_config_keys(mut self, keys: &'static [&'static str]) -> Self { + self.inherited_config_keys = keys; + self + } + + /// Mark a backend whose local deployment should use single-player defaults. + #[must_use] + pub fn with_local_singleplayer(mut self) -> Self { + self.local_singleplayer = true; + self + } + + /// Mark a backend that requires user authentication other than mTLS. + #[must_use] + pub fn without_mtls_user_auth(mut self) -> Self { + self.supports_mtls_user_auth = false; + self + } + + #[must_use] + pub(crate) fn is_local_singleplayer(&self) -> bool { + self.local_singleplayer + } + + #[must_use] + pub(crate) fn supports_mtls_user_auth(&self) -> bool { + self.supports_mtls_user_auth + } } /// Registry of compute drivers compiled into this gateway binary. @@ -1163,11 +1192,11 @@ impl ComputeDriverRegistry { self.drivers.keys().map(String::as_str) } - fn get(&self, name: &str) -> Option<&ComputeDriverRegistration> { + pub(crate) fn get(&self, name: &str) -> Option<&ComputeDriverRegistration> { self.drivers.get(name) } - fn detect(&self) -> Option<&ComputeDriverRegistration> { + pub(crate) fn detect(&self) -> Option<&ComputeDriverRegistration> { self.drivers .values() .filter(|registration| registration.detect.is_some_and(|detect| detect())) @@ -1175,70 +1204,6 @@ impl ComputeDriverRegistry { } } -/// Install every first-party compute driver linked into the standard gateway. -#[must_use] -pub fn install_default_compute_drivers() -> ComputeDriverRegistry { - #[allow(unused_mut)] - let mut registry = ComputeDriverRegistry::new(); - #[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] - { - registry - .install( - ComputeDriverRegistration::new( - "kubernetes", - 100, - Some(|| std::env::var_os("KUBERNETES_SERVICE_HOST").is_some()), - KubernetesComputeDriverFactory, - ) - .expect("valid kubernetes registration"), - ) - .expect("unique kubernetes registration"); - registry - .install( - ComputeDriverRegistration::new( - "podman", - 200, - Some(openshell_core::config::is_podman_available), - PodmanComputeDriverFactory, - ) - .expect("valid podman registration"), - ) - .expect("unique podman registration"); - registry - .install( - ComputeDriverRegistration::new( - "docker", - 300, - Some(openshell_core::config::is_docker_available), - DockerComputeDriverFactory, - ) - .expect("valid docker registration"), - ) - .expect("unique docker registration"); - registry - .install( - ComputeDriverRegistration::new("vm", u16::MAX, None, VmComputeDriverFactory) - .expect("valid vm registration"), - ) - .expect("unique vm registration"); - } - #[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] - for name in ["kubernetes", "podman", "docker", "vm"] { - registry - .install( - ComputeDriverRegistration::new( - name, - u16::MAX, - None, - UnsupportedComputeDriverFactory, - ) - .expect("valid unsupported registration"), - ) - .expect("unique unsupported registration"); - } - registry -} - pub struct ComputeDriverBuildContext<'a> { driver_name: String, config: &'a Config, @@ -1249,6 +1214,7 @@ pub struct ComputeDriverBuildContext<'a> { tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, shutdown_rx: watch::Receiver, + inherited_config_keys: &'static [&'static str], } impl ComputeDriverBuildContext<'_> { @@ -1285,7 +1251,11 @@ impl ComputeDriverBuildContext<'_> { where T: Default + serde::de::DeserializeOwned, { - compute::driver_config::driver_config_from_context(self.driver_startup, &self.driver_name) + compute::driver_config::driver_config_from_context( + self.driver_startup, + &self.driver_name, + self.inherited_config_keys, + ) } #[must_use] @@ -1293,10 +1263,26 @@ impl ComputeDriverBuildContext<'_> { self.shutdown_rx.clone() } + #[must_use] + pub fn otlp_config(&self) -> Option<&config_file::OtlpConfig> { + self.driver_startup + .file + .and_then(|file| file.openshell.gateway.otlp.as_ref()) + } + /// Finish construction of an in-process driver through the common runtime path. pub async fn finish_in_process( self, driver: SharedComputeDriver, + ) -> Result { + self.finish_in_process_with_allowlist(driver, None).await + } + + /// Finish construction while publishing a dynamic authentication allowlist. + pub async fn finish_in_process_with_allowlist( + self, + driver: SharedComputeDriver, + operator_allowlist: Option, ) -> Result { let runtime = ComputeRuntime::from_driver( self.driver_name, @@ -1310,155 +1296,24 @@ impl ComputeDriverBuildContext<'_> { ) .await .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist: None, - }) - } -} - -#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct UnsupportedComputeDriverFactory; - -#[cfg(all(target_os = "windows", feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for UnsupportedComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result { - Err(Error::execution( - unsupported_builtin_compute_driver( - context - .driver_name - .parse() - .expect("default driver names are valid"), - ) - .to_string(), - )) - } -} - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct KubernetesComputeDriverFactory; - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for KubernetesComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result { - warn_if_kubernetes_sandbox_jwt_expiry_disabled(context.config); - let config = compute::driver_config::builtin::kubernetes_config_from_context( - context.driver_startup, - )?; - let (runtime, operator_allowlist) = ComputeRuntime::new_kubernetes( - config, - context.store, - context.sandbox_index, - context.sandbox_watch_bus, - context.tracing_log_bus, - context.supervisor_sessions, - context.shutdown_rx, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; Ok(ComputeDriverBuildOutput { runtime, operator_allowlist, }) } -} - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct DockerComputeDriverFactory; - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for DockerComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result { - let driver_config = - compute::driver_config::builtin::docker_config_from_context(context.driver_startup)?; - let runtime = ComputeRuntime::new_docker( - context.config.clone(), - driver_config, - context.store, - context.sandbox_index, - context.sandbox_watch_bus, - context.tracing_log_bus, - context.supervisor_sessions, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist: None, - }) - } -} - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct PodmanComputeDriverFactory; - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for PodmanComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result { - let driver_config = - compute::driver_config::builtin::podman_config_from_context(context.driver_startup)?; - let runtime = ComputeRuntime::new_podman( - driver_config, - context.store, - context.sandbox_index, - context.sandbox_watch_bus, - context.tracing_log_bus, - context.supervisor_sessions, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist: None, - }) - } -} -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[derive(Clone, Copy)] -struct VmComputeDriverFactory; - -#[cfg(all(not(target_os = "windows"), feature = "in-tree-compute-drivers"))] -#[async_trait::async_trait] -impl ComputeDriverFactory for VmComputeDriverFactory { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, + /// Finish construction of a gateway-managed remote driver process. + pub async fn finish_remote( + self, + endpoint: AcquiredRemoteDriverEndpoint, ) -> Result { - let driver_config = - compute::driver_config::builtin::vm_config_from_context(context.driver_startup)?; - let otlp_config = context - .driver_startup - .file - .and_then(|file| file.openshell.gateway.otlp.as_ref()); - let endpoint = compute::vm::spawn(context.config, &driver_config, otlp_config).await?; let runtime = ComputeRuntime::new_remote_driver( endpoint, - context.store, - context.sandbox_index, - context.sandbox_watch_bus, - context.tracing_log_bus, - context.supervisor_sessions, + self.store, + self.sandbox_index, + self.sandbox_watch_bus, + self.tracing_log_bus, + self.supervisor_sessions, ) .await .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; @@ -1498,6 +1353,7 @@ async fn build_compute_runtime( tracing_log_bus, supervisor_sessions, shutdown_rx, + inherited_config_keys: registration.inherited_config_keys, }) .await?; (output.runtime, output.operator_allowlist) @@ -1553,10 +1409,16 @@ fn configured_compute_driver( match config.compute_drivers.as_slice() { [] => registry.detect().map_or_else( || { - Err(Error::config( + let installed = registry.installed_driver_names().collect::>(); + let hint = if installed.is_empty() { + "this gateway binary has no compiled compute drivers".to_string() + } else { + format!("installed drivers: {}", installed.join(", ")) + }; + Err(Error::config(format!( "no compute driver configured and auto-detection found no suitable driver; \ - set --drivers or OPENSHELL_DRIVERS to kubernetes, podman, docker, or vm", - )) + set --drivers or OPENSHELL_DRIVERS ({hint})" + ))) }, |registration| Ok(ConfiguredComputeDriver::Registered(registration.clone())), ), @@ -1586,23 +1448,6 @@ fn resolve_configured_compute_driver( Ok(ConfiguredComputeDriver::Remote { name }) } -#[cfg(any(test, feature = "in-tree-compute-drivers"))] -fn kubernetes_sandbox_jwt_expiry_disabled(config: &Config) -> bool { - config - .gateway_jwt - .as_ref() - .is_some_and(|jwt| jwt.ttl_secs == 0) -} - -#[cfg(feature = "in-tree-compute-drivers")] -fn warn_if_kubernetes_sandbox_jwt_expiry_disabled(config: &Config) { - if kubernetes_sandbox_jwt_expiry_disabled(config) { - warn!( - "Kubernetes gateway configured with non-expiring sandbox JWTs (gateway_jwt.ttl_secs = 0); set ttl_secs > 0 for shared Kubernetes deployments" - ); - } -} - pub(crate) async fn ensure_default_workspace(store: &Store) -> Result<()> { use grpc::workspace::{DEFAULT_WORKSPACE_NAME, WORKSPACE_OBJECT_TYPE}; use openshell_core::proto::Workspace; @@ -1669,11 +1514,10 @@ mod tests { GatewayListenerScope, MultiplexService, ServerState, TlsAcceptor, allow_plaintext_service_http, bind_gateway_listeners, classify_initial_bytes, configured_compute_driver, is_benign_tls_handshake_failure, - kubernetes_sandbox_jwt_expiry_disabled, mint_gateway_extension_credential, - serve_gateway_listener, + mint_gateway_extension_credential, serve_gateway_listener, }; use openshell_core::{ - ComputeDriverKind, Config, + Config, proto::{HealthRequest, open_shell_client::OpenShellClient}, }; use std::io::{Error, ErrorKind}; @@ -1796,7 +1640,21 @@ mod tests { } fn test_compute_drivers() -> super::ComputeDriverRegistry { - super::install_default_compute_drivers() + let mut registry = super::ComputeDriverRegistry::new(); + for (name, priority) in [("alpha", 100), ("beta", 200), ("gamma", 300)] { + registry + .install( + super::ComputeDriverRegistration::new( + name, + priority, + None, + TestComputeDriverFactory, + ) + .unwrap(), + ) + .unwrap(); + } + registry } #[derive(Clone, Copy)] @@ -2199,8 +2057,7 @@ mod tests { #[test] fn configured_compute_driver_rejects_multiple_entries() { - let config = Config::new(None) - .with_compute_drivers([ComputeDriverKind::Kubernetes, ComputeDriverKind::Podman]); + let config = Config::new(None).with_compute_drivers(["alpha", "beta"]); let err = configured_compute_driver( &test_compute_drivers(), &config, @@ -2211,42 +2068,12 @@ mod tests { err.to_string() .contains("multiple compute drivers are not supported yet") ); - assert!(err.to_string().contains("kubernetes,podman")); - } - - #[test] - fn configured_compute_driver_accepts_podman() { - let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Podman]); - let driver = configured_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap(); - assert!(matches!( - driver, - ConfiguredComputeDriver::Registered(registration) if registration.name == "podman" - )); - } - - #[test] - fn configured_compute_driver_accepts_vm() { - let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Vm]); - let driver = configured_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ) - .unwrap(); - assert!(matches!( - driver, - ConfiguredComputeDriver::Registered(registration) if registration.name == "vm" - )); + assert!(err.to_string().contains("alpha,beta")); } #[test] - fn configured_compute_driver_accepts_docker() { - let config = Config::new(None).with_compute_drivers([ComputeDriverKind::Docker]); + fn configured_compute_driver_accepts_registered_name() { + let config = Config::new(None).with_compute_drivers(["beta"]); let driver = configured_compute_driver( &test_compute_drivers(), &config, @@ -2255,7 +2082,7 @@ mod tests { .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Registered(registration) if registration.name == "docker" + ConfiguredComputeDriver::Registered(registration) if registration.name == "beta" )); } @@ -2284,10 +2111,10 @@ mod tests { } #[test] - fn configured_compute_driver_uses_vm_endpoint_override() { + fn configured_compute_driver_uses_endpoint_override() { let config = Config::new(None) - .with_compute_drivers([ComputeDriverKind::Vm]) - .with_compute_driver_endpoint("vm", "/run/openshell/vm.sock"); + .with_compute_drivers(["alpha"]) + .with_compute_driver_endpoint("alpha", "/run/openshell/alpha.sock"); let driver = configured_compute_driver( &test_compute_drivers(), @@ -2297,15 +2124,15 @@ mod tests { .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Remote { name } if name == "vm" + ConfiguredComputeDriver::Remote { name } if name == "alpha" )); } #[test] fn configured_compute_driver_uses_builtin_endpoint_override() { let config = Config::new(None) - .with_compute_drivers([ComputeDriverKind::Docker]) - .with_compute_driver_endpoint("docker", "/run/openshell/docker.sock"); + .with_compute_drivers(["beta"]) + .with_compute_driver_endpoint("beta", "/run/openshell/beta.sock"); let driver = configured_compute_driver( &test_compute_drivers(), @@ -2315,50 +2142,10 @@ mod tests { .unwrap(); assert!(matches!( driver, - ConfiguredComputeDriver::Remote { name } if name == "docker" + ConfiguredComputeDriver::Remote { name } if name == "beta" )); } - #[test] - fn kubernetes_sandbox_jwt_expiry_disabled_warns_for_zero_ttl() { - fn config_with_jwt_ttl(ttl_secs: u64) -> Config { - let mut config = Config::new(None); - config.gateway_jwt = Some(openshell_core::GatewayJwtConfig { - signing_key_path: "/tmp/signing.pem".into(), - public_key_path: "/tmp/public.pem".into(), - kid_path: "/tmp/kid".into(), - gateway_id: "openshell".to_string(), - ttl_secs, - }); - config - } - - assert!(kubernetes_sandbox_jwt_expiry_disabled( - &config_with_jwt_ttl(0) - )); - assert!(!kubernetes_sandbox_jwt_expiry_disabled( - &config_with_jwt_ttl(3600) - )); - assert!(!kubernetes_sandbox_jwt_expiry_disabled(&Config::new(None))); - } - - #[cfg(target_os = "windows")] - #[test] - fn windows_builtin_compute_drivers_report_unsupported() { - for driver in [ - ComputeDriverKind::Docker, - ComputeDriverKind::Kubernetes, - ComputeDriverKind::Podman, - ComputeDriverKind::Vm, - ] { - let message = super::unsupported_builtin_compute_driver(driver).to_string(); - assert!( - message.contains("unsupported on Windows"), - "{driver} rejection should be explicit, got: {message}" - ); - } - } - #[tokio::test] async fn failed_gateway_listener_bind_does_not_attempt_persisted_sandbox_start() { let occupied_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); diff --git a/deploy/docker/Dockerfile.gateway-macos b/deploy/docker/Dockerfile.gateway-macos index 122f16eba8..c7d526a039 100644 --- a/deploy/docker/Dockerfile.gateway-macos +++ b/deploy/docker/Dockerfile.gateway-macos @@ -53,6 +53,7 @@ ENV BINDGEN_EXTRA_CLANG_ARGS_aarch64_apple_darwin=--target=arm64-apple-macosx\ - COPY Cargo.toml Cargo.lock ./ COPY crates/openshell-core/Cargo.toml crates/openshell-core/Cargo.toml +COPY crates/openshell-gateway/Cargo.toml crates/openshell-gateway/Cargo.toml COPY crates/openshell-driver-kubernetes/Cargo.toml crates/openshell-driver-kubernetes/Cargo.toml COPY crates/openshell-policy/Cargo.toml crates/openshell-policy/Cargo.toml COPY crates/openshell-prover/Cargo.toml crates/openshell-prover/Cargo.toml @@ -61,39 +62,42 @@ COPY crates/openshell-server/Cargo.toml crates/openshell-server/Cargo.toml COPY crates/openshell-core/build.rs crates/openshell-core/build.rs COPY proto/ proto/ -RUN sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-server", "crates/openshell-core", "crates/openshell-driver-kubernetes", "crates/openshell-policy", "crates/openshell-prover", "crates/openshell-router"]|' Cargo.toml +RUN sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-gateway", "crates/openshell-server", "crates/openshell-core", "crates/openshell-driver-kubernetes", "crates/openshell-policy", "crates/openshell-prover", "crates/openshell-router"]|' Cargo.toml RUN mkdir -p crates/openshell-core/src \ + crates/openshell-gateway/src \ crates/openshell-driver-kubernetes/src \ crates/openshell-policy/src \ crates/openshell-prover/src \ crates/openshell-router/src \ crates/openshell-server/src && \ touch crates/openshell-core/src/lib.rs && \ + touch crates/openshell-gateway/src/lib.rs && \ + printf 'fn main() {}\n' > crates/openshell-gateway/src/main.rs && \ touch crates/openshell-driver-kubernetes/src/lib.rs && \ printf 'fn main() {}\n' > crates/openshell-driver-kubernetes/src/main.rs && \ touch crates/openshell-policy/src/lib.rs && \ touch crates/openshell-prover/src/lib.rs && \ touch crates/openshell-router/src/lib.rs && \ - touch crates/openshell-server/src/lib.rs && \ - printf 'fn main() {}\n' > crates/openshell-server/src/main.rs + touch crates/openshell-server/src/lib.rs RUN --mount=type=cache,id=cargo-registry-gateway-macos,sharing=locked,target=/root/.cargo/registry \ --mount=type=cache,id=cargo-git-gateway-macos,sharing=locked,target=/root/.cargo/git \ --mount=type=cache,id=cargo-target-gateway-macos-${CARGO_TARGET_CACHE_SCOPE},sharing=locked,target=/build/target \ - cargo build --release --target aarch64-apple-darwin -p openshell-server --features bundled-z3 2>/dev/null || true + cargo build --release --target aarch64-apple-darwin -p openshell-gateway --features bundled-z3 2>/dev/null || true COPY crates/ crates/ COPY providers/ providers/ RUN touch crates/openshell-core/src/lib.rs \ + crates/openshell-gateway/src/lib.rs \ + crates/openshell-gateway/src/main.rs \ crates/openshell-driver-kubernetes/src/lib.rs \ crates/openshell-driver-kubernetes/src/main.rs \ crates/openshell-policy/src/lib.rs \ crates/openshell-prover/src/lib.rs \ crates/openshell-router/src/lib.rs \ crates/openshell-server/src/lib.rs \ - crates/openshell-server/src/main.rs \ crates/openshell-core/build.rs \ proto/*.proto @@ -105,7 +109,7 @@ RUN --mount=type=cache,id=cargo-registry-gateway-macos,sharing=locked,target=/ro if [ -n "${OPENSHELL_CARGO_VERSION:-}" ]; then \ sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${OPENSHELL_CARGO_VERSION}"'"/}' Cargo.toml; \ fi && \ - cargo build --release --target aarch64-apple-darwin -p openshell-server --features bundled-z3 && \ + cargo build --release --target aarch64-apple-darwin -p openshell-gateway --features bundled-z3 && \ cp target/aarch64-apple-darwin/release/openshell-gateway /openshell-gateway FROM scratch AS binary diff --git a/e2e/no-compute-driver-gateway.sh b/e2e/no-compute-driver-gateway.sh index 7ad3896ca1..64a2b8d841 100755 --- a/e2e/no-compute-driver-gateway.sh +++ b/e2e/no-compute-driver-gateway.sh @@ -8,11 +8,12 @@ ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "${ROOT}" echo "Building gateway without compiled compute drivers..." -cargo build -p openshell-server --bin openshell-gateway \ +cargo build -p openshell-gateway --bin openshell-gateway \ --no-default-features --features telemetry -dependency_tree="$(cargo tree -p openshell-server \ +dependency_tree="$(cargo tree -p openshell-gateway \ --no-default-features --features telemetry --edges normal)" +server_dependency_tree="$(cargo tree -p openshell-server --edges normal)" for driver in \ openshell-driver-docker \ openshell-driver-kubernetes \ @@ -22,7 +23,18 @@ for driver in \ echo "ERROR: driver-free gateway dependency graph contains ${driver}" >&2 exit 1 fi + if grep -q "${driver} v" <<<"${server_dependency_tree}"; then + echo "ERROR: openshell-server dependency graph contains ${driver}" >&2 + exit 1 + fi done +if rg -n \ + 'ComputeDriverKind|openshell_driver_(docker|podman|kubernetes)([^_[:alnum:]]|$)|ComputeRuntime::new_(docker|podman|kubernetes)|VmComputeConfig|compute::vm|driver_config::builtin|libkrun|gvproxy|qemu' \ + crates/openshell-core crates/openshell-server; then + echo "ERROR: backend-specific compute-driver knowledge leaked into core/server" >&2 + exit 1 +fi + "${ROOT}/target/debug/openshell-gateway" --version echo "Driver-free gateway build passed." diff --git a/e2e/run.sh b/e2e/run.sh index 0505730f05..875186394c 100755 --- a/e2e/run.sh +++ b/e2e/run.sh @@ -250,7 +250,7 @@ guest_gateway_bin= if [ "${mode}" = host ]; then echo "==> Building native host openshell-gateway" mise x -- cargo build "${cargo_jobs[@]}" \ - -p openshell-server \ + -p openshell-gateway \ --bin openshell-gateway \ --features bundled-z3 host_gateway_bin="${target_dir}/debug/openshell-gateway" @@ -268,7 +268,7 @@ else mise x -- cargo zigbuild "${cargo_jobs[@]}" \ --release \ --target "${linux_gateway_zig_target}" \ - -p openshell-server \ + -p openshell-gateway \ --bin openshell-gateway \ --features bundled-z3 ) diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index a1ff90f48d..ffc32ff0b0 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -100,10 +100,10 @@ if [ -z "${OPENSHELL_GATEWAY_BIN:-}" ]; then if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then echo "==> Building driver-free openshell-gateway" cargo build \ - -p openshell-server --bin openshell-gateway \ + -p openshell-gateway --bin openshell-gateway \ --no-default-features --features telemetry else - build_packages+=(-p openshell-server) + build_packages+=(-p openshell-gateway) fi else echo "==> Using prebuilt openshell-gateway at ${GATEWAY_BIN}" diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index 34412408a9..64cc84920e 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -219,11 +219,11 @@ e2e_build_gateway_binaries() { echo "Building openshell-gateway..." if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then cargo build "${jobs[@]}" \ - -p openshell-server --bin openshell-gateway \ + -p openshell-gateway --bin openshell-gateway \ --no-default-features --features telemetry else cargo build "${jobs[@]}" \ - -p openshell-server --bin openshell-gateway + -p openshell-gateway --bin openshell-gateway fi else echo "Using prebuilt openshell gateway at ${OPENSHELL_GATEWAY_BIN}" diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 8a2ceff63f..1064714794 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -660,7 +660,7 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then echo "ERROR: external Kubernetes driver image composition currently requires a Linux build host." >&2 exit 2 fi - cargo build -p openshell-server --bin openshell-gateway \ + cargo build -p openshell-gateway --bin openshell-gateway \ --no-default-features --features telemetry cargo build -p openshell-driver-kubernetes --bin openshell-driver-kubernetes case "$(uname -m)" in diff --git a/tasks/ci.toml b/tasks/ci.toml index 4c0b5f8ea7..cc4158448f 100644 --- a/tasks/ci.toml +++ b/tasks/ci.toml @@ -31,7 +31,7 @@ hide = true description = "Build release Rust binaries consumed by the hand-staged snap" run = [ "cargo build --release -p openshell-cli", - "cargo build --release -p openshell-server --features bundled-z3", + "cargo build --release -p openshell-gateway --features bundled-z3", "cargo build --release -p openshell-sandbox", ] diff --git a/tasks/gateway.toml b/tasks/gateway.toml index 83cf35d8fb..9c3359a241 100644 --- a/tasks/gateway.toml +++ b/tasks/gateway.toml @@ -5,7 +5,7 @@ ["build:gateway"] description = "Build the standalone openshell-gateway binary" -run = "cargo build -p openshell-server --bin openshell-gateway" +run = "cargo build -p openshell-gateway --bin openshell-gateway" hide = true ["gateway"] diff --git a/tasks/rust.toml b/tasks/rust.toml index 854c2ac939..8313aa5b3d 100644 --- a/tasks/rust.toml +++ b/tasks/rust.toml @@ -51,10 +51,10 @@ description = "Verify telemetry emission code is compiled out with --no-default- run = [ # Positive control: the default (telemetry-on) gateway must contain the # markers, so the absent checks below can never become silently vacuous. - "cargo build -p openshell-server --bin openshell-gateway", + "cargo build -p openshell-gateway --bin openshell-gateway", "tasks/scripts/verify-telemetry-compiled-out.sh present target/debug/openshell-gateway", # Guard: telemetry-free builds must contain no telemetry markers. - "cargo build -p openshell-server --bin openshell-gateway --no-default-features", + "cargo build -p openshell-gateway --bin openshell-gateway --no-default-features", "tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-gateway", "cargo build -p openshell-sandbox --bin openshell-sandbox --no-default-features --features bundled-ca-roots", "tasks/scripts/verify-telemetry-compiled-out.sh absent target/debug/openshell-sandbox", diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 895d5a62fc..9a5406a417 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -127,7 +127,7 @@ fi echo "Building openshell-gateway..." cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \ - -p openshell-server --bin openshell-gateway + -p openshell-gateway --bin openshell-gateway TLS_DIR="${STATE_DIR}/tls" echo "Generating local gateway credentials..." diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 22ba1b039f..8d7e8cc587 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -296,7 +296,7 @@ fi echo "==> Building openshell-gateway and openshell-driver-vm" cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \ - -p openshell-server -p openshell-driver-vm + -p openshell-gateway -p openshell-driver-vm if [ "$(uname -s)" = "Darwin" ]; then echo "==> Codesigning openshell-driver-vm (Hypervisor entitlement)" diff --git a/tasks/scripts/package-deb-install.sh b/tasks/scripts/package-deb-install.sh index b6e4730674..e20d409bbd 100755 --- a/tasks/scripts/package-deb-install.sh +++ b/tasks/scripts/package-deb-install.sh @@ -56,7 +56,7 @@ remove_existing_gateway_registration() { echo "==> Building release binaries" cargo build --release \ -p openshell-cli \ - -p openshell-server \ + -p openshell-gateway \ -p openshell-driver-vm echo "==> Building Debian package" diff --git a/tasks/scripts/stage-prebuilt-binaries.sh b/tasks/scripts/stage-prebuilt-binaries.sh index b7eb1dad74..093cf0f50e 100755 --- a/tasks/scripts/stage-prebuilt-binaries.sh +++ b/tasks/scripts/stage-prebuilt-binaries.sh @@ -124,7 +124,7 @@ components_for_target() { resolve_component() { case "$1" in gateway) - crate=openshell-server + crate=openshell-gateway binary=openshell-gateway target_libc=gnu ;; diff --git a/tasks/scripts/vm/smoke-orphan-cleanup.sh b/tasks/scripts/vm/smoke-orphan-cleanup.sh index 6da48919d1..7d0b05334d 100755 --- a/tasks/scripts/vm/smoke-orphan-cleanup.sh +++ b/tasks/scripts/vm/smoke-orphan-cleanup.sh @@ -37,7 +37,7 @@ trap cleanup_stray EXIT build_binaries() { echo "==> Ensuring binaries are built" if [ ! -x "$ROOT/target/debug/openshell-gateway" ] || [ ! -x "$ROOT/target/debug/openshell-driver-vm" ]; then - cargo build -p openshell-server -p openshell-driver-vm >&2 + cargo build -p openshell-gateway -p openshell-driver-vm >&2 fi if [ "$(uname -s)" = "Darwin" ]; then codesign \ From 9c107ce49591d69b967ae5257ac6661fc5744a46 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 18 Aug 2026 17:52:56 -0700 Subject: [PATCH 04/13] fix(build): update gateway package references Signed-off-by: Drew Newberry --- CONTRIBUTING.md | 2 +- README.md | 2 +- architecture/build.md | 2 +- bazel/releases/BUILD.bazel | 6 +- crates/openshell-core/src/settings.rs | 2 +- crates/openshell-driver-vm/README.md | 6 +- crates/openshell-driver-vm/runtime/README.md | 2 +- crates/openshell-gateway/BUILD.bazel | 64 +++++++++++++++++++ crates/openshell-server/BUILD.bazel | 15 +---- deploy/docker/Dockerfile.python-wheels | 3 +- deploy/docker/Dockerfile.python-wheels-macos | 3 +- examples/governance-interceptor/smoke.sh | 2 +- .../smoke.sh | 2 +- 13 files changed, 80 insertions(+), 31 deletions(-) create mode 100644 crates/openshell-gateway/BUILD.bazel diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cc52782560..e5e15d39da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -445,7 +445,7 @@ The following Bazel commands are available alongside the mise tasks above. Cargo | Build everything | `bazel build //...` | All crates and protos | | Run all tests | `bazel test //...` | Unit tests only, no E2E | | Build the CLI | `bazel build //crates/openshell-cli:openshell` | | -| Build the gateway | `bazel build //crates/openshell-server:openshell-gateway` | | +| Build the gateway | `bazel build //crates/openshell-gateway:openshell-gateway-bin` | | | Build the supervisor | `bazel build //crates/openshell-sandbox:openshell-sandbox-bin` | | | Clean | `bazel clean` | | diff --git a/README.md b/README.md index b8ce17a6d2..e1da69fc03 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ OpenShell collects anonymous telemetry to help improve the project for developer Disable telemetry at runtime by setting `OPENSHELL_TELEMETRY_ENABLED=false` on the gateway deployment. For Helm installs, set `server.telemetryEnabled=false`. OpenShell propagates this deployment setting into sandbox supervisor environments so sandbox-side telemetry collection is disabled as well. -You can also compile telemetry out entirely. Telemetry support is a default-on `telemetry` Cargo feature; building with `--no-default-features` produces binaries that contain no telemetry endpoint, no telemetry HTTP client, and no emission code. Build telemetry-free artifacts with, for example, `cargo build --release -p openshell-server --no-default-features` (gateway) and the equivalent for `openshell-sandbox` and `openshell-driver-vm`. With telemetry compiled out, the gateway emits nothing and reports telemetry disabled to the sandboxes it launches. +You can also compile telemetry out entirely. Telemetry support is a default-on `telemetry` Cargo feature; building with `--no-default-features` produces binaries that contain no telemetry endpoint, no telemetry HTTP client, and no emission code. Build a telemetry-free gateway with `cargo build --release -p openshell-gateway --no-default-features --features in-tree-compute-drivers`, and use the equivalent feature selection for `openshell-sandbox` and `openshell-driver-vm`. With telemetry compiled out, the gateway emits nothing and reports telemetry disabled to the sandboxes it launches. Telemetry events are limited to anonymous operational categories and counts, such as sandbox lifecycle outcomes, provider profile buckets, policy decision counts, and aggregate network activity denial categories. OpenShell telemetry does not collect sandbox names or IDs, hostnames, file paths, binary paths, prompts, credentials, provider names, model names, or user content. diff --git a/architecture/build.md b/architecture/build.md index 5c5751772a..f64815dd6f 100644 --- a/architecture/build.md +++ b/architecture/build.md @@ -26,7 +26,7 @@ Sandbox community images are built outside this repository. Anonymous telemetry emission is gated behind a default-on `telemetry` Cargo feature. It is defined in `openshell-core` (where the emission code, HTTP client, and endpoint live) and forwarded by the binary crates that emit or -collect telemetry: `openshell-server` (gateway), `openshell-sandbox` +collect telemetry: `openshell-gateway`, `openshell-sandbox` (supervisor), and `openshell-driver-vm`. Every crate depends on `openshell-core` with `default-features = false`, so the binary crate's feature is the single switch that enables `openshell-core/telemetry` for its build diff --git a/bazel/releases/BUILD.bazel b/bazel/releases/BUILD.bazel index bc859ed16c..6a84fcd027 100644 --- a/bazel/releases/BUILD.bazel +++ b/bazel/releases/BUILD.bazel @@ -59,7 +59,7 @@ platform_transition_binary( platform_transition_binary( name = "openshell_gateway_linux_x86_64", basename = "openshell-gateway", - binary = "//crates/openshell-server:openshell-gateway", + binary = "//crates/openshell-gateway:openshell-gateway-bin", tags = ["manual"], target_platform = ":linux_x86_64_gnu_2_28", ) @@ -67,7 +67,7 @@ platform_transition_binary( platform_transition_binary( name = "openshell_gateway_linux_aarch64", basename = "openshell-gateway", - binary = "//crates/openshell-server:openshell-gateway", + binary = "//crates/openshell-gateway:openshell-gateway-bin", tags = ["manual"], target_platform = ":linux_aarch64_gnu_2_28", ) @@ -91,7 +91,7 @@ platform_transition_binary( platform_transition_binary( name = "openshell_gateway_macos_aarch64", basename = "openshell-gateway", - binary = "//crates/openshell-server:openshell-gateway", + binary = "//crates/openshell-gateway:openshell-gateway-bin", tags = ["manual"], target_platform = "@rules_rs//rs/platforms:aarch64-apple-darwin", ) diff --git a/crates/openshell-core/src/settings.rs b/crates/openshell-core/src/settings.rs index 156e4c3845..db942a9686 100644 --- a/crates/openshell-core/src/settings.rs +++ b/crates/openshell-core/src/settings.rs @@ -65,7 +65,7 @@ impl RegisteredSetting { /// /// 1. Add a [`RegisteredSetting`] entry to this array with the key name and /// [`SettingValueKind`]. -/// 2. Recompile `openshell-server` (gateway) and `openshell-sandbox` +/// 2. Recompile `openshell-gateway` and `openshell-sandbox` /// (supervisor). No database migration is needed -- new keys are stored in /// the existing settings JSON blob. /// 3. Add sandbox-side consumption in `openshell-sandbox` to read and act on diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index bf902013e1..b38baca4df 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -9,7 +9,7 @@ Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) fo ```mermaid flowchart LR subgraph host["Host process"] - gateway["openshell-server
(compute::vm::spawn)"] + gateway["openshell-gateway
(vm::spawn)"] driver["openshell-driver-vm
├── libkrun (VM)
├── gvproxy (net)
└── openshell-sandbox.zst"] gateway <-->|"gRPC over UDS
compute-driver.sock"| driver end @@ -98,7 +98,7 @@ mise run vm:supervisor # if openshell-sandbox.zst is not already presen # 2. Build both binaries with the staged artifacts embedded OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ - cargo build -p openshell-server -p openshell-driver-vm + cargo build -p openshell-gateway -p openshell-driver-vm # 3. macOS only: codesign the driver for Hypervisor.framework codesign \ @@ -283,5 +283,5 @@ the user explicitly overrides it. ## TODOs -- The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` notes in `crates/openshell-server/src/lib.rs` and `crates/openshell-server/src/compute/vm.rs`. +- The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` note in `crates/openshell-gateway/src/vm.rs`. - macOS local builds are codesigned by `tasks/scripts/gateway-vm.sh`; the generated Homebrew formula signs the release tarball driver for local installs. diff --git a/crates/openshell-driver-vm/runtime/README.md b/crates/openshell-driver-vm/runtime/README.md index 11aab67f43..b686874ba2 100644 --- a/crates/openshell-driver-vm/runtime/README.md +++ b/crates/openshell-driver-vm/runtime/README.md @@ -41,7 +41,7 @@ mise run vm:supervisor # Build the gateway and VM driver with embedded runtime artifacts OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \ - cargo build -p openshell-server -p openshell-driver-vm + cargo build -p openshell-gateway -p openshell-driver-vm ``` Use `FROM_SOURCE=1 mise run vm:setup` to build the runtime from source instead diff --git a/crates/openshell-gateway/BUILD.bazel b/crates/openshell-gateway/BUILD.bazel new file mode 100644 index 0000000000..90b464ef15 --- /dev/null +++ b/crates/openshell-gateway/BUILD.bazel @@ -0,0 +1,64 @@ +load("@crates//:defs.bzl", "aliases", "all_crate_deps") +load("@rules_rs//rs:rust_binary.bzl", "rust_binary") +load("@rules_rs//rs:rust_library.bzl", "rust_library") +load("@rules_rs//rs:rust_test.bzl", "rust_test") +load("@rules_rust//rust:defs.bzl", "rustfmt_test") +load("@workspace_version//:version.bzl", "WORKSPACE_VERSION") + +rust_library( + name = "openshell-gateway", + srcs = glob( + ["src/**/*.rs"], + exclude = ["src/main.rs"], + ), + aliases = aliases(), + crate_features = [ + "in-tree-compute-drivers", + "telemetry", + ], + version = WORKSPACE_VERSION, + visibility = ["//visibility:public"], + deps = all_crate_deps(normal = True), +) + +rust_binary( + name = "openshell-gateway-bin", + srcs = ["src/main.rs"], + aliases = aliases(), + binary_name = "openshell-gateway", + version = WORKSPACE_VERSION, + visibility = ["//visibility:public"], + deps = all_crate_deps(normal = True) + [":openshell-gateway"], +) + +rust_test( + name = "openshell-gateway_lib_test", + crate = ":openshell-gateway", + crate_features = [ + "in-tree-compute-drivers", + "telemetry", + ], + deps = all_crate_deps(normal_dev = True), +) + +rust_test( + name = "openshell-gateway_bin_test", + srcs = ["src/main.rs"], + aliases = aliases(), + version = WORKSPACE_VERSION, + deps = all_crate_deps( + normal = True, + normal_dev = True, + ) + [":openshell-gateway"], +) + +rustfmt_test( + name = "rustfmt_test", + targets = [ + ":openshell-gateway", + ":openshell-gateway-bin", + ":openshell-gateway_bin_test", + ":openshell-gateway_lib_test", + ], + visibility = ["//crates:__pkg__"], +) diff --git a/crates/openshell-server/BUILD.bazel b/crates/openshell-server/BUILD.bazel index 3c3293c720..374e5c883e 100644 --- a/crates/openshell-server/BUILD.bazel +++ b/crates/openshell-server/BUILD.bazel @@ -1,15 +1,11 @@ load("@crates//:defs.bzl", "aliases", "all_crate_deps") -load("@rules_rs//rs:rust_binary.bzl", "rust_binary") load("@rules_rs//rs:rust_library.bzl", "rust_library") load("@rules_rs//rs:rust_test.bzl", "rust_test") load("@rules_rust//rust:defs.bzl", "rustfmt_test") rust_library( name = "openshell-server", - srcs = glob( - ["src/**/*.rs"], - exclude = ["src/main.rs"], - ), + srcs = glob(["src/**/*.rs"]), aliases = aliases(), compile_data = glob(["migrations/**/*"]), crate_features = ["telemetry"], @@ -20,14 +16,6 @@ rust_library( deps = all_crate_deps(normal = True), ) -rust_binary( - name = "openshell-gateway", - srcs = ["src/main.rs"], - aliases = aliases(), - visibility = ["//visibility:public"], - deps = all_crate_deps(normal = True) + [":openshell-server"], -) - rust_test( name = "openshell-server_test", crate = ":openshell-server", @@ -110,7 +98,6 @@ rustfmt_test( ":health_endpoint_integration_test", ":multiplex_integration_test", ":multiplex_tls_integration_test", - ":openshell-gateway", ":openshell-server", ":openshell-server_test", ":supervisor_relay_integration_test", diff --git a/deploy/docker/Dockerfile.python-wheels b/deploy/docker/Dockerfile.python-wheels index fe58e3fcc7..3cbf81e4e4 100644 --- a/deploy/docker/Dockerfile.python-wheels +++ b/deploy/docker/Dockerfile.python-wheels @@ -58,7 +58,6 @@ COPY proto/ proto/ RUN mkdir -p crates/openshell-cli/src crates/openshell-core/src crates/openshell-ocsf/src crates/openshell-policy/src crates/openshell-providers/src crates/openshell-prover/src crates/openshell-router/src crates/openshell-sandbox/src crates/openshell-server/src crates/openshell-bootstrap/src crates/openshell-tui/src && \ echo "fn main() {}" > crates/openshell-cli/src/main.rs && \ echo "fn main() {}" > crates/openshell-sandbox/src/main.rs && \ - echo "fn main() {}" > crates/openshell-server/src/main.rs && \ touch crates/openshell-core/src/lib.rs && \ touch crates/openshell-ocsf/src/lib.rs && \ touch crates/openshell-providers/src/lib.rs && \ @@ -90,7 +89,7 @@ RUN touch crates/openshell-cli/src/main.rs \ crates/openshell-providers/src/lib.rs \ crates/openshell-router/src/lib.rs \ crates/openshell-sandbox/src/main.rs \ - crates/openshell-server/src/main.rs \ + crates/openshell-server/src/lib.rs \ crates/openshell-core/build.rs \ proto/*.proto diff --git a/deploy/docker/Dockerfile.python-wheels-macos b/deploy/docker/Dockerfile.python-wheels-macos index 1825dd6276..acb10007cd 100644 --- a/deploy/docker/Dockerfile.python-wheels-macos +++ b/deploy/docker/Dockerfile.python-wheels-macos @@ -74,7 +74,6 @@ COPY proto/ proto/ RUN mkdir -p crates/openshell-cli/src crates/openshell-core/src crates/openshell-ocsf/src crates/openshell-policy/src crates/openshell-providers/src crates/openshell-prover/src crates/openshell-router/src crates/openshell-sandbox/src crates/openshell-server/src crates/openshell-bootstrap/src crates/openshell-tui/src && \ echo "fn main() {}" > crates/openshell-cli/src/main.rs && \ echo "fn main() {}" > crates/openshell-sandbox/src/main.rs && \ - echo "fn main() {}" > crates/openshell-server/src/main.rs && \ touch crates/openshell-core/src/lib.rs && \ touch crates/openshell-ocsf/src/lib.rs && \ touch crates/openshell-providers/src/lib.rs && \ @@ -106,7 +105,7 @@ RUN touch crates/openshell-cli/src/main.rs \ crates/openshell-providers/src/lib.rs \ crates/openshell-router/src/lib.rs \ crates/openshell-sandbox/src/main.rs \ - crates/openshell-server/src/main.rs \ + crates/openshell-server/src/lib.rs \ crates/openshell-core/build.rs \ proto/*.proto diff --git a/examples/governance-interceptor/smoke.sh b/examples/governance-interceptor/smoke.sh index 88610cf1ee..6c59fb687c 100755 --- a/examples/governance-interceptor/smoke.sh +++ b/examples/governance-interceptor/smoke.sh @@ -650,7 +650,7 @@ wait_until_stopped() { cd "$ROOT" -run_setup_step "building gateway" cargo build --quiet -p openshell-server --bin openshell-gateway +run_setup_step "building gateway" cargo build --quiet -p openshell-gateway --bin openshell-gateway run_setup_step "building governance interceptor" cargo build --quiet --manifest-path "$EXAMPLE_DIR/Cargo.toml" run_setup_step "building CLI" cargo build --quiet -p openshell-cli --bin openshell diff --git a/examples/supervisor-middleware-content-guard/smoke.sh b/examples/supervisor-middleware-content-guard/smoke.sh index 509ffd403d..b30475c8ca 100755 --- a/examples/supervisor-middleware-content-guard/smoke.sh +++ b/examples/supervisor-middleware-content-guard/smoke.sh @@ -444,7 +444,7 @@ EXAMPLE_TARGET_DIR="$(cargo_target_dir "$EXAMPLE_DIR/Cargo.toml")" GATEWAY_BIN="$ROOT_TARGET_DIR/debug/openshell-gateway" CLI_BIN="$ROOT_TARGET_DIR/debug/openshell" MIDDLEWARE_BIN="$EXAMPLE_TARGET_DIR/debug/supervisor-middleware-content-guard" -run_setup_step "building gateway" cargo build --quiet -p openshell-server --bin openshell-gateway +run_setup_step "building gateway" cargo build --quiet -p openshell-gateway --bin openshell-gateway run_setup_step "building content guard" cargo build --quiet --manifest-path "$EXAMPLE_DIR/Cargo.toml" run_setup_step "building CLI" cargo build --quiet -p openshell-cli --bin openshell generate_gateway_jwt_bundle From 5e3695f9cf33f564185bfe45b4db2bbf39739019 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 18 Aug 2026 20:32:44 -0700 Subject: [PATCH 05/13] refactor(compute): simplify compiled driver boundary Signed-off-by: Drew Newberry --- architecture/compute-runtimes.md | 20 +- crates/openshell-core/src/driver_utils.rs | 13 +- crates/openshell-core/src/error.rs | 4 +- crates/openshell-core/src/telemetry.rs | 2 +- crates/openshell-driver-docker/src/lib.rs | 14 +- crates/openshell-driver-docker/src/main.rs | 5 +- crates/openshell-driver-podman/src/watcher.rs | 11 +- crates/openshell-gateway/src/lib.rs | 51 ++-- crates/openshell-gateway/src/vm.rs | 8 +- crates/openshell-server/src/auth/k8s_sa.rs | 219 ++++++++++++++- crates/openshell-server/src/cli.rs | 23 +- .../src/compute/driver_config.rs | 16 ++ crates/openshell-server/src/compute/mod.rs | 3 +- .../openshell-server/src/gateway_listener.rs | 96 ++++--- crates/openshell-server/src/lib.rs | 258 +++++++++--------- e2e/no-compute-driver-gateway.sh | 1 + 16 files changed, 479 insertions(+), 265 deletions(-) diff --git a/architecture/compute-runtimes.md b/architecture/compute-runtimes.md index f0e0554b45..ac99d05cd3 100644 --- a/architecture/compute-runtimes.md +++ b/architecture/compute-runtimes.md @@ -105,8 +105,10 @@ registry. Adding or removing a compiled driver therefore changes registration rather than the server's selection flow. Alternate gateway binaries can install their own `ComputeDriverFactory` registrations and hand the completed registry to `run_cli_with_compute_drivers`; factories receive merged driver config and -finish through the same in-process runtime adapter. A configured UDS endpoint -still takes precedence over a compiled registration with the same name. +return either an in-process driver or a gateway-managed remote endpoint. The +server constructs the common runtime adapter and snapshots `GetCapabilities` +for either result. A configured UDS endpoint still takes precedence over a +compiled registration with the same name. The `openshell-gateway` composition crate groups first-party registrations behind the `in-tree-compute-drivers` feature. `openshell-server` has no compute @@ -402,13 +404,13 @@ image-pull Secrets in every operator-managed namespace. **Operator** uses pre-provisioned namespaces discovered through two optional sources: a K8s label selector (`operator_namespace_label`) and a drop-in -allowlist file (`operator_namespace_file`). At least one must be configured. -The `OperatorNamespaceAllowlist` (`Arc>>`) is populated -at runtime by background watchers and read by the namespace resolver. Sandbox -creation fails closed if the workspace is not in the current allowlist. Platform -teams manage namespace lifecycle externally. RBAC uses the same ClusterRole as -managed mode but without namespace `create`/`delete` or ServiceAccount -permissions. +allowlist file (`operator_namespace_file`). Exactly one must be configured. +The compute driver and the gateway's ServiceAccount authenticator independently +watch that public config source; no in-process driver state crosses into the +server. Sandbox creation and token bootstrap fail closed if the workspace is +not in the current allowlist. Platform teams manage namespace lifecycle +externally. RBAC uses the same ClusterRole as managed mode but without namespace +`create`/`delete` or ServiceAccount permissions. ### Watching and Querying diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 523d9ccc42..c37408e4ca 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -352,9 +352,9 @@ pub fn read_upstream_proxy_credential_file(path: &str) -> Result /// /// The resulting path is `$XDG_STATE_HOME/openshell/[/]//sandbox.jwt`. /// -/// `driver_subdir` is driver-specific, e.g. `"docker-sandbox-tokens"` or -/// `"podman-sandbox-tokens"`. When `namespace` is `Some`, it is appended as -/// an additional path component (with `/` and `\` replaced by `-`). +/// `driver_subdir` is driver-specific. When `namespace` is `Some`, it is +/// appended as an additional path component (with `/` and `\` replaced by +/// `-`). /// /// # Errors /// Returns an error if the XDG state directory cannot be resolved. @@ -405,7 +405,7 @@ pub fn sandbox_log_level(sandbox: &DriverSandbox, default_level: &str) -> String } // --------------------------------------------------------------------------- -// Supervisor image helpers (shared by Docker and Podman drivers) +// Supervisor image helpers shared by container-backed drivers // --------------------------------------------------------------------------- /// Return the tag portion of a supervisor image reference, or `None` if the @@ -440,7 +440,7 @@ pub fn supervisor_image_should_refresh(image: &str) -> bool { } // --------------------------------------------------------------------------- -// Supervisor binary extraction helpers (shared by Docker and Podman drivers) +// Supervisor binary extraction helpers shared by container-backed drivers // --------------------------------------------------------------------------- #[cfg(feature = "driver-extraction")] @@ -517,8 +517,7 @@ pub fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> Result<(), /// Return the host-side cache path for an extracted supervisor binary. /// /// The path is `$XDG_DATA_HOME/openshell///openshell-sandbox`. -/// `driver_subdir` distinguishes caches across drivers (e.g. `"docker-supervisor"`, -/// `"podman-supervisor"`). +/// `driver_subdir` distinguishes caches across drivers. pub fn supervisor_cache_path(driver_subdir: &str, digest: &str) -> Result { let base = crate::paths::xdg_data_dir() .map_err(|err| format!("failed to resolve XDG data dir: {err}"))?; diff --git a/crates/openshell-core/src/error.rs b/crates/openshell-core/src/error.rs index 8c23e30198..145106012d 100644 --- a/crates/openshell-core/src/error.rs +++ b/crates/openshell-core/src/error.rs @@ -106,8 +106,8 @@ impl Error { /// Error type shared by all compute driver implementations. /// -/// Both the Podman and Kubernetes drivers map their backend-specific -/// errors into these variants before crossing crate boundaries. +/// Drivers map backend-specific errors into these variants before crossing +/// crate boundaries. #[derive(Debug, Error)] pub enum ComputeDriverError { /// The requested sandbox already exists. diff --git a/crates/openshell-core/src/telemetry.rs b/crates/openshell-core/src/telemetry.rs index ea2f6fe36b..f092f5f8aa 100644 --- a/crates/openshell-core/src/telemetry.rs +++ b/crates/openshell-core/src/telemetry.rs @@ -760,7 +760,7 @@ mod disabled_tests { 1, false, SandboxTemplateSource::Default, - TelemetryComputeDriver::Docker, + TelemetryComputeDriver::from_raw("test-driver"), ); emit_policy_decision( PolicyDecisionOperation::Approve, diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index b1d0f5cb82..d0e82dca41 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -52,7 +52,7 @@ use openshell_core::proto::compute::v1::{ use openshell_core::proto_struct::{ deserialize_optional_non_empty_string_list, struct_to_json_value, }; -use openshell_core::{Config, Error, Result as CoreResult}; +use openshell_core::{Error, Result as CoreResult}; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; @@ -416,7 +416,11 @@ pub fn is_available() -> bool { } impl DockerComputeDriver { - pub async fn new(config: &Config, docker_config: &DockerComputeConfig) -> CoreResult { + pub async fn new( + gateway_bind_address: SocketAddr, + gateway_log_level: &str, + docker_config: &DockerComputeConfig, + ) -> CoreResult { let socket_path = docker_config .socket_path .clone() @@ -446,7 +450,7 @@ impl DockerComputeDriver { let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); let allow_all_default_gpu = docker_info_reports_wsl2(&info); validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; - let gateway_port = config.bind_address.port(); + let gateway_port = gateway_bind_address.port(); if gateway_port == 0 { return Err(Error::config( "docker compute driver requires a fixed non-zero gateway bind port", @@ -458,7 +462,7 @@ impl DockerComputeDriver { let gateway_route = docker_gateway_route(&info, bridge_gateway_ip, gateway_port, host_gateway_ip); let gateway_callback_bind_address = - docker_gateway_callback_bind_address(&gateway_route, config.bind_address); + docker_gateway_callback_bind_address(&gateway_route, gateway_bind_address); let mut docker_config = docker_config.clone(); if docker_config.grpc_endpoint.trim().is_empty() { let scheme = if docker_guest_tls_configured(&docker_config) { @@ -490,7 +494,7 @@ impl DockerComputeDriver { gateway_callback_bind_address, ssh_socket_path: docker_config.ssh_socket_path.clone(), stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, - log_level: config.log_level.clone(), + log_level: gateway_log_level.to_string(), supervisor_bin, guest_tls, daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), diff --git a/crates/openshell-driver-docker/src/main.rs b/crates/openshell-driver-docker/src/main.rs index 3c539ba11b..fc171152c1 100644 --- a/crates/openshell-driver-docker/src/main.rs +++ b/crates/openshell-driver-docker/src/main.rs @@ -6,8 +6,8 @@ use std::path::PathBuf; use clap::Parser; use miette::{IntoDiagnostic, Result}; +use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_core::{Config, VERSION}; use openshell_driver_docker::{DockerComputeConfig, DockerComputeDriver}; use tracing::info; use tracing_subscriber::EnvFilter; @@ -46,8 +46,7 @@ async fn main() -> Result<()> { let config_source = std::fs::read_to_string(&args.config).into_diagnostic()?; let docker_config: DockerComputeConfig = toml::from_str(&config_source).into_diagnostic()?; - let gateway_config = Config::new(None).with_bind_address(args.gateway_bind); - let driver = DockerComputeDriver::new(&gateway_config, &docker_config) + let driver = DockerComputeDriver::new(args.gateway_bind, &args.log_level, &docker_config) .await .into_diagnostic()?; diff --git a/crates/openshell-driver-podman/src/watcher.rs b/crates/openshell-driver-podman/src/watcher.rs index 3e98d16271..b0f43d0fb5 100644 --- a/crates/openshell-driver-podman/src/watcher.rs +++ b/crates/openshell-driver-podman/src/watcher.rs @@ -121,15 +121,12 @@ fn deleted_event(sandbox_id: String) -> WatchSandboxesEvent { /// drops (daemon restart, socket error, or clean shutdown), the stream /// terminates with a final error item and stops producing events. /// -/// Callers are responsible for reconnecting by calling [`start_watch`] again. -/// The server's `ComputeRuntime::watch_loop` in `openshell-server` provides -/// this behaviour with a 2-second backoff: when the stream terminates with an -/// error, `watch_loop` sleeps and then calls `watch_sandboxes()` again, which -/// ultimately calls `start_watch()` again and re-syncs state. +/// Callers are responsible for reconnecting by calling [`start_watch`] again +/// and re-synchronizing state. /// /// **Do not add reconnection logic inside this function.** A local reconnect -/// would race with `watch_loop`'s retry and produce duplicate initial-sync -/// events that corrupt the server's sandbox index. +/// would race with the consumer's retry and produce duplicate initial-sync +/// events. pub async fn start_watch( client: PodmanClient, lifecycle_event_fences: LifecycleEventFences, diff --git a/crates/openshell-gateway/src/lib.rs b/crates/openshell-gateway/src/lib.rs index cc37d811d8..56e16a4c71 100644 --- a/crates/openshell-gateway/src/lib.rs +++ b/crates/openshell-gateway/src/lib.rs @@ -110,7 +110,7 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, - ) -> openshell_core::Result { + ) -> openshell_core::Result { let mut config: openshell_driver_kubernetes::KubernetesComputeConfig = context.driver_config()?; if let Ok(size) = std::env::var("OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE") { @@ -119,27 +119,16 @@ impl openshell_server::ComputeDriverFactory for KubernetesFactory { if let Ok(storage_class) = std::env::var("OPENSHELL_K8S_WORKSPACE_STORAGE_CLASS") { config.workspace_storage_class = storage_class; } - if context - .gateway_config() - .gateway_jwt - .as_ref() - .is_some_and(|jwt| jwt.ttl_secs == 0) - { - tracing::warn!( - "Kubernetes gateway configured with non-expiring sandbox JWTs; set gateway_jwt.ttl_secs > 0 for shared deployments" - ); - } let driver = openshell_driver_kubernetes::KubernetesComputeDriver::new( config, context.shutdown_receiver(), ) .await .map_err(|error| openshell_core::Error::execution(error.to_string()))?; - let allowlist = driver.operator_allowlist().cloned(); let driver = openshell_driver_kubernetes::ComputeDriverService::new(driver); - context - .finish_in_process_with_allowlist(std::sync::Arc::new(driver), allowlist) - .await + Ok(openshell_server::ComputeDriverInstance::InProcess( + std::sync::Arc::new(driver), + )) } } @@ -153,7 +142,7 @@ impl openshell_server::ComputeDriverFactory for DockerFactory { async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, - ) -> openshell_core::Result { + ) -> openshell_core::Result { let mut config: openshell_driver_docker::DockerComputeConfig = context.driver_config()?; apply_guest_tls( &mut config.guest_tls_ca, @@ -161,11 +150,16 @@ impl openshell_server::ComputeDriverFactory for DockerFactory { &mut config.guest_tls_key, context.guest_tls_paths(), ); - let driver = - openshell_driver_docker::DockerComputeDriver::new(context.gateway_config(), &config) - .await - .map_err(|error| openshell_core::Error::execution(error.to_string()))?; - context.finish_in_process(std::sync::Arc::new(driver)).await + let driver = openshell_driver_docker::DockerComputeDriver::new( + context.gateway_bind_address(), + context.gateway_log_level(), + &config, + ) + .await + .map_err(|error| openshell_core::Error::execution(error.to_string()))?; + Ok(openshell_server::ComputeDriverInstance::InProcess( + std::sync::Arc::new(driver), + )) } } @@ -179,7 +173,7 @@ impl openshell_server::ComputeDriverFactory for PodmanFactory { async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, - ) -> openshell_core::Result { + ) -> openshell_core::Result { let mut config: openshell_driver_podman::PodmanComputeConfig = context.driver_config()?; config.gateway_port = context.gateway_port(); if let Ok(path) = std::env::var("OPENSHELL_PODMAN_SOCKET") { @@ -201,7 +195,9 @@ impl openshell_server::ComputeDriverFactory for PodmanFactory { .await .map_err(|error| openshell_core::Error::execution(error.to_string()))?; let driver = openshell_driver_podman::ComputeDriverService::new(driver); - context.finish_in_process(std::sync::Arc::new(driver)).await + Ok(openshell_server::ComputeDriverInstance::InProcess( + std::sync::Arc::new(driver), + )) } } @@ -215,7 +211,7 @@ impl openshell_server::ComputeDriverFactory for VmFactory { async fn build( &self, context: openshell_server::ComputeDriverBuildContext<'_>, - ) -> openshell_core::Result { + ) -> openshell_core::Result { let mut config: vm::VmComputeConfig = context.driver_config()?; if config.state_dir.as_os_str().is_empty() { config.state_dir = vm::VmComputeConfig::default_state_dir(); @@ -236,8 +232,11 @@ impl openshell_server::ComputeDriverFactory for VmFactory { &mut config.guest_tls_key, context.guest_tls_paths(), ); - let endpoint = vm::spawn(context.gateway_config(), &config, context.otlp_config()).await?; - context.finish_remote(endpoint).await + let endpoint = + vm::spawn(context.gateway_log_level(), &config, context.otlp_config()).await?; + Ok(openshell_server::ComputeDriverInstance::ManagedRemote( + endpoint, + )) } } diff --git a/crates/openshell-gateway/src/vm.rs b/crates/openshell-gateway/src/vm.rs index 2a95f9ddc5..7530ce74cc 100644 --- a/crates/openshell-gateway/src/vm.rs +++ b/crates/openshell-gateway/src/vm.rs @@ -35,7 +35,7 @@ use hyper_util::rt::TokioIo; use openshell_core::proto::compute::v1::{ GetCapabilitiesRequest, compute_driver_client::ComputeDriverClient, }; -use openshell_core::{Config, Error, Result}; +use openshell_core::{Error, Result}; #[cfg(unix)] use openshell_otel::TraceContextInterceptor; use openshell_server::AcquiredRemoteDriverEndpoint; @@ -453,7 +453,7 @@ pub fn compute_driver_guest_tls_paths( /// kills the subprocess and removes the socket on drop. #[cfg(unix)] pub async fn spawn( - config: &Config, + gateway_log_level: &str, vm_config: &VmComputeConfig, otlp_config: Option<&OtlpConfig>, ) -> Result { @@ -477,7 +477,7 @@ pub async fn spawn( command .arg("--expected-peer-pid") .arg(std::process::id().to_string()); - command.arg("--log-level").arg(&config.log_level); + command.arg("--log-level").arg(gateway_log_level); append_otlp_args(&mut command, otlp_config); command .arg("--openshell-endpoint") @@ -527,7 +527,7 @@ fn append_otlp_args(command: &mut Command, otlp_config: Option<&OtlpConfig>) { #[cfg(not(unix))] pub async fn spawn( - _config: &Config, + _gateway_log_level: &str, _vm_config: &VmComputeConfig, _otlp_config: Option<&OtlpConfig>, ) -> Result { diff --git a/crates/openshell-server/src/auth/k8s_sa.rs b/crates/openshell-server/src/auth/k8s_sa.rs index 03dc692205..a3fc62aa25 100644 --- a/crates/openshell-server/src/auth/k8s_sa.rs +++ b/crates/openshell-server/src/auth/k8s_sa.rs @@ -18,16 +18,21 @@ use super::authenticator::Authenticator; use super::principal::{Principal, SandboxIdentitySource, SandboxPrincipal}; use async_trait::async_trait; +use futures::{StreamExt, TryStreamExt}; use k8s_openapi::api::{ authentication::v1::{TokenReview, TokenReviewSpec, TokenReviewStatus, UserInfo}, - core::v1::Pod, + core::v1::{Namespace, Pod}, }; use k8s_openapi::apimachinery::pkg::apis::meta::v1::ObjectMeta; use kube::Error as KubeError; use kube::api::{Api, ApiResource, PostParams}; use kube::core::{DynamicObject, gvk::GroupVersionKind}; +use kube::runtime::watcher::{self, Event}; use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist; +use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::{Duration, SystemTime}; +use tokio::sync::{mpsc, watch}; use tonic::Status; use tracing::{debug, info, warn}; @@ -159,6 +164,218 @@ impl NamespaceValidator { } } +pub fn operator_namespace_allowlist( + config: &crate::compute::driver_config::KubernetesSaBootstrapConfig, + client: kube::Client, + shutdown_rx: watch::Receiver, +) -> openshell_core::Result { + let allowlist = OperatorNamespaceAllowlist::new(); + match ( + config.operator_namespace_label.as_deref(), + config.operator_namespace_file.as_deref(), + ) { + (Some(label), None) if !label.trim().is_empty() => { + spawn_namespace_label_watcher( + client, + label.to_string(), + allowlist.clone(), + shutdown_rx, + ); + } + (None, Some(path)) if !path.trim().is_empty() => { + spawn_namespace_file_watcher(path.into(), allowlist.clone(), shutdown_rx); + } + (None, None) => { + return Err(openshell_core::Error::config( + "operator workspace mode requires operator_namespace_label or operator_namespace_file", + )); + } + (Some(_), Some(_)) => { + return Err(openshell_core::Error::config( + "operator workspace mode accepts only one of operator_namespace_label or operator_namespace_file", + )); + } + _ => { + return Err(openshell_core::Error::config( + "operator namespace source must not be empty", + )); + } + } + Ok(allowlist) +} + +fn spawn_namespace_label_watcher( + client: kube::Client, + label_selector: String, + allowlist: OperatorNamespaceAllowlist, + mut shutdown_rx: watch::Receiver, +) { + let namespace_api: Api = Api::all(client); + let watcher_config = watcher::Config::default().labels(&label_selector); + let jitter_seed = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |duration| { + duration.as_secs() ^ u64::from(duration.subsec_nanos()) + }); + + tokio::spawn(async move { + let mut retry_attempt = 0; + loop { + let mut stream = + watcher::watcher(namespace_api.clone(), watcher_config.clone()).boxed(); + loop { + let event = tokio::select! { + result = stream.try_next() => result, + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + continue; + } + }; + match event { + Ok(Some(Event::Applied(namespace))) => { + retry_attempt = 0; + if let Some(name) = namespace.metadata.name + && allowlist.insert(name.clone()) + { + info!(namespace = name, "operator namespace added to allowlist"); + } + } + Ok(Some(Event::Deleted(namespace))) => { + retry_attempt = 0; + if let Some(name) = namespace.metadata.name.as_deref() + && allowlist.remove(name) + { + info!( + namespace = name, + "operator namespace removed from allowlist" + ); + } + } + Ok(Some(Event::Restarted(namespaces))) => { + retry_attempt = 0; + allowlist.replace( + namespaces + .into_iter() + .filter_map(|namespace| namespace.metadata.name) + .collect(), + ); + } + Ok(None) => { + warn!("operator namespace watcher stream ended unexpectedly"); + break; + } + Err(error) => { + warn!(%error, "operator namespace watcher stream error"); + break; + } + } + } + + let retry_delay = namespace_watcher_retry_delay(retry_attempt, jitter_seed); + warn!(?retry_delay, "operator namespace watcher reconnecting"); + tokio::select! { + () = tokio::time::sleep(retry_delay) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + } + } + retry_attempt = retry_attempt.saturating_add(1); + } + }); + + info!(%label_selector, "operator namespace label watcher spawned"); +} + +fn namespace_watcher_retry_delay(attempt: u32, jitter_seed: u64) -> Duration { + let base_secs = 2_u64.saturating_mul(1_u64 << attempt.min(4)).min(24); + let max_jitter_secs = base_secs / 4; + let mixed_seed = + jitter_seed.wrapping_add(u64::from(attempt).wrapping_mul(0x9e37_79b9_7f4a_7c15)); + Duration::from_secs(base_secs + mixed_seed % (max_jitter_secs + 1)) +} + +fn load_namespace_file(path: &Path) -> Result, String> { + let contents = std::fs::read_to_string(path) + .map_err(|error| format!("failed to read {}: {error}", path.display()))?; + let names: Vec = serde_json::from_str(&contents) + .map_err(|error| format!("failed to parse {}: {error}", path.display()))?; + Ok(names.into_iter().collect()) +} + +fn spawn_namespace_file_watcher( + path: PathBuf, + allowlist: OperatorNamespaceAllowlist, + mut shutdown_rx: watch::Receiver, +) { + match load_namespace_file(&path) { + Ok(names) => allowlist.replace(names), + Err(error) => { + warn!(%error, "failed to load initial operator namespace file, allowlist empty"); + } + } + + let watch_dir = path + .parent() + .unwrap_or_else(|| Path::new(".")) + .to_path_buf(); + tokio::spawn(async move { + let (tx, mut rx) = mpsc::unbounded_channel(); + let mut file_watcher = match notify::recommended_watcher( + move |result: Result| { + if let Ok(event) = result + && matches!( + event.kind, + notify::EventKind::Modify(_) | notify::EventKind::Create(_) + ) + { + let _ = tx.send(()); + } + }, + ) { + Ok(watcher) => watcher, + Err(error) => { + warn!(%error, "failed to start operator namespace file watcher"); + return; + } + }; + if let Err(error) = notify::Watcher::watch( + &mut file_watcher, + &watch_dir, + notify::RecursiveMode::NonRecursive, + ) { + warn!(%error, dir = %watch_dir.display(), "failed to watch operator namespace file directory"); + return; + } + + loop { + let got_event = tokio::select! { + event = rx.recv() => event.is_some(), + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return; + } + continue; + } + }; + if !got_event { + return; + } + tokio::time::sleep(Duration::from_secs(1)).await; + while rx.try_recv().is_ok() {} + match load_namespace_file(&path) { + Ok(names) => allowlist.replace(names), + Err(error) => { + warn!(%error, "failed to reload operator namespace file, keeping existing allowlist"); + } + } + } + }); +} + #[derive(Debug)] struct TokenReviewIdentity { namespace: String, diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 06d2dd59ad..a4138ed07a 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -95,12 +95,10 @@ struct RunArgs { /// Compute drivers configured for this gateway. /// - /// Accepts a comma-delimited list such as `kubernetes` or - /// `kubernetes,podman`. The configuration format is future-proofed for - /// multiple drivers, but the gateway currently requires exactly one. - /// When unset, the gateway auto-detects the driver based on the runtime - /// environment (Kubernetes → Podman → Docker). VM is never - /// auto-detected and requires explicit configuration. + /// Accepts a comma-delimited list of registered driver names. The + /// configuration format is future-proofed for multiple drivers, but the + /// gateway currently requires exactly one. When unset, the gateway runs + /// detection probes supplied by the drivers compiled into the binary. #[arg( long, alias = "driver", @@ -114,9 +112,8 @@ struct RunArgs { /// implementing `compute_driver.proto`. /// /// When set, the socket is associated with the single driver name supplied - /// by `--drivers` or `OPENSHELL_DRIVERS`. The endpoint overrides built-in - /// construction when the selected name is Docker, Podman, Kubernetes, or - /// VM. + /// by `--drivers` or `OPENSHELL_DRIVERS`. The endpoint overrides a compiled + /// registration with the same name. #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] compute_driver_socket: Option, @@ -134,9 +131,9 @@ struct RunArgs { /// Enable mTLS client certificate authentication for local single-user gateways. /// - /// When unset, this defaults on for Docker, Podman, and VM gateways that - /// have client certificate verification configured and no OIDC issuer. - /// Kubernetes deployments must use OIDC or fronting-proxy auth instead. + /// When unset, this defaults on for drivers registered as local + /// single-player backends when client certificate verification is + /// configured and no OIDC issuer is present. #[arg( long = "enable-mtls-auth", env = "OPENSHELL_ENABLE_MTLS_AUTH", @@ -847,7 +844,7 @@ mod tests { async fn build( &self, _context: crate::ComputeDriverBuildContext<'_>, - ) -> openshell_core::Result { + ) -> openshell_core::Result { unreachable!("CLI metadata tests do not build drivers") } } diff --git a/crates/openshell-server/src/compute/driver_config.rs b/crates/openshell-server/src/compute/driver_config.rs index 98d798eed3..51911d7e82 100644 --- a/crates/openshell-server/src/compute/driver_config.rs +++ b/crates/openshell-server/src/compute/driver_config.rs @@ -79,6 +79,8 @@ pub struct KubernetesSaBootstrapConfig { pub service_account_name: String, pub workspace_mode: String, pub gateway_id: String, + pub operator_namespace_label: Option, + pub operator_namespace_file: Option, } impl Default for KubernetesSaBootstrapConfig { @@ -88,6 +90,8 @@ impl Default for KubernetesSaBootstrapConfig { service_account_name: "default".to_string(), workspace_mode: "shared".to_string(), gateway_id: "openshell".to_string(), + operator_namespace_label: None, + operator_namespace_file: None, } } } @@ -115,6 +119,8 @@ pub fn kubernetes_sa_bootstrap_config( "host_gateway_ip", "enable_user_namespaces", "sa_token_ttl_secs", + "operator_namespace_label", + "operator_namespace_file", ], ); merged.try_into().map_err(|error| { @@ -253,6 +259,8 @@ socket_path = "/run/openshell/kubernetes.sock" workspace_mode = "managed" gateway_id = "gateway-a" service_account_name = "sandbox-sa" +operator_namespace_label = "openshell.ai/workspace=true" +operator_namespace_file = "/etc/openshell/namespaces.json" "#, ) .expect("valid config"); @@ -262,6 +270,14 @@ service_account_name = "sandbox-sa" assert_eq!(cfg.workspace_mode, "managed"); assert_eq!(cfg.gateway_id, "gateway-a"); assert_eq!(cfg.service_account_name, "sandbox-sa"); + assert_eq!( + cfg.operator_namespace_label.as_deref(), + Some("openshell.ai/workspace=true") + ); + assert_eq!( + cfg.operator_namespace_file.as_deref(), + Some("/etc/openshell/namespaces.json") + ); } #[test] diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 3b1d03e1d2..734dc4a14d 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -5260,8 +5260,7 @@ mod tests { )); } - /// Driver calls are a remote boundary even in-process: they reach the - /// Docker daemon, the Kubernetes API, or a Podman socket. + /// Driver calls are a remote boundary even when the driver is in-process. #[tokio::test] async fn driver_calls_export_spans_with_parents() { use tracing::Instrument as _; diff --git a/crates/openshell-server/src/gateway_listener.rs b/crates/openshell-server/src/gateway_listener.rs index 640757bfb3..b638fc33e4 100644 --- a/crates/openshell-server/src/gateway_listener.rs +++ b/crates/openshell-server/src/gateway_listener.rs @@ -386,10 +386,10 @@ mod tests { #[test] fn gateway_listener_specs_reuse_primary_when_wildcard_covers_driver_address() { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); - let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); + let callback: SocketAddr = "172.18.0.1:8080".parse().unwrap(); let requirements = [ - docker_listener_requirement(docker), - docker_listener_requirement(docker), + exact_listener_requirement(callback), + exact_listener_requirement(callback), ]; assert_eq!( @@ -401,15 +401,15 @@ mod tests { #[test] fn gateway_listener_scope_for_reused_primary_remains_primary() { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); - let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); + let callback: SocketAddr = "172.18.0.1:8080".parse().unwrap(); let loopback: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let [spec] = gateway_listener_specs(primary, &[docker_listener_requirement(docker)]) + let [spec] = gateway_listener_specs(primary, &[exact_listener_requirement(callback)]) .unwrap() .try_into() .unwrap(); assert_eq!( - spec.scope_for_local_addr(docker), + spec.scope_for_local_addr(callback), GatewayListenerScope::Primary, ); assert_eq!( @@ -421,10 +421,10 @@ mod tests { #[test] fn gateway_listener_specs_preserve_driver_callback_scope() { let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let docker: SocketAddr = "172.18.0.1:8080".parse().unwrap(); + let callback: SocketAddr = "172.18.0.1:8080".parse().unwrap(); let requirements = [ - docker_listener_requirement(docker), - docker_listener_requirement(docker), + exact_listener_requirement(callback), + exact_listener_requirement(callback), ]; assert_eq!( @@ -437,11 +437,11 @@ mod tests { provenance: None, }, GatewayListenerSpec { - address: docker, + address: callback, scope: GatewayListenerScope::ComputeDriverCallback, covered_addresses: Vec::new(), provenance: Some(GatewayListenerProvenance { - driver_name: "docker".to_string(), + driver_name: "alpha".to_string(), reason: "managed bridge".to_string(), }), }, @@ -473,7 +473,7 @@ mod tests { "172.18.0.1:0", "172.18.0.1:9090", ] { - let requirement = docker_listener_requirement(address.parse().unwrap()); + let requirement = exact_listener_requirement(address.parse().unwrap()); assert!( gateway_listener_specs(primary, &[requirement]).is_err(), "{address} should be rejected" @@ -482,41 +482,41 @@ mod tests { } #[test] - fn gateway_listener_specs_use_exact_podman_network_gateway() { + fn gateway_listener_specs_use_exact_network_gateway() { let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); - let podman_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); + let network_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs(primary, &[podman_listener_requirement(podman_gateway)]) + gateway_listener_specs(primary, &[network_listener_requirement(network_gateway)]) .unwrap(), vec![ primary_listener_spec(primary), - callback_listener_spec(podman_gateway, "podman", "Podman managed bridge",), + callback_listener_spec(network_gateway, "beta", "managed bridge",), ] ); } #[test] - fn gateway_listener_specs_reuse_primary_when_it_covers_podman_exact() { + fn gateway_listener_specs_reuse_primary_when_it_covers_exact_requirement() { let primary: SocketAddr = "0.0.0.0:8080".parse().unwrap(); - let podman_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); + let network_gateway: SocketAddr = "10.89.1.1:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs(primary, &[podman_listener_requirement(podman_gateway)],) + gateway_listener_specs(primary, &[network_listener_requirement(network_gateway)],) .unwrap(), vec![primary_listener_spec(primary)] ); } #[test] - fn gateway_listener_specs_resolve_podman_default_route_source() { + fn gateway_listener_specs_resolve_default_route_source() { let primary: SocketAddr = "127.0.0.1:8080".parse().unwrap(); let default_route_ip = "192.168.20.20".parse().unwrap(); assert_eq!( gateway_listener_specs_with_default_route_ip( primary, - &[podman_default_route_listener_requirement()], + &[default_route_listener_requirement()], Some(default_route_ip), ) .unwrap(), @@ -524,8 +524,8 @@ mod tests { primary_listener_spec(primary), callback_listener_spec( "192.168.20.20:8080".parse().unwrap(), - "podman", - "rootless pasta upstream interface", + "beta", + "default route interface", ), ] ); @@ -537,7 +537,7 @@ mod tests { let err = gateway_listener_specs_with_default_route_ip( primary, - &[podman_default_route_listener_requirement()], + &[default_route_listener_requirement()], Some("203.0.113.20".parse().unwrap()), ) .unwrap_err(); @@ -552,7 +552,7 @@ mod tests { assert_eq!( gateway_listener_specs_with_default_route_ip( primary, - &[podman_default_route_listener_requirement()], + &[default_route_listener_requirement()], Some(default_route_ip), ) .unwrap(), @@ -561,28 +561,24 @@ mod tests { } #[test] - fn gateway_listener_specs_resolve_podman_loopback_separately() { + fn gateway_listener_specs_resolve_loopback_separately() { let primary: SocketAddr = "192.168.20.20:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(), vec![ primary_listener_spec(primary), - callback_listener_spec( - "127.0.0.1:8080".parse().unwrap(), - "podman", - "Podman machine host forwarder", - ), + callback_listener_spec("127.0.0.1:8080".parse().unwrap(), "beta", "host forwarder",), ] ); } #[test] - fn gateway_listener_specs_reuse_wildcard_primary_for_podman_loopback() { + fn gateway_listener_specs_reuse_wildcard_primary_for_loopback() { let primary = "0.0.0.0:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(), vec![primary_listener_spec(primary)] ); } @@ -592,7 +588,7 @@ mod tests { let primary = "127.0.0.1:8080".parse().unwrap(); assert_eq!( - gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(), + gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(), vec![primary_listener_spec(primary)] ); } @@ -602,7 +598,7 @@ mod tests { for primary in ["[::1]:8080", "[::]:8080"] { let primary = primary.parse().unwrap(); let specs = - gateway_listener_specs(primary, &[podman_loopback_listener_requirement()]).unwrap(); + gateway_listener_specs(primary, &[loopback_listener_requirement()]).unwrap(); assert_eq!(specs.len(), 2); assert_eq!(specs[1].address, SocketAddr::from(([127, 0, 0, 1], 8080))); @@ -613,7 +609,7 @@ mod tests { fn gateway_listener_specs_validate_selector_independently_of_driver_name() { let primary: SocketAddr = "192.168.20.20:8080".parse().unwrap(); let requirement = GatewayListenerRequirement::LoopbackInterface { - driver_name: "docker".to_string(), + driver_name: "alpha".to_string(), reason: "wrong selector".to_string(), }; @@ -634,7 +630,7 @@ mod tests { let result: openshell_core::Result<()> = async { let _listeners = bind_gateway_listeners( primary_address, - &[docker_listener_requirement(occupied_address)], + &[exact_listener_requirement(occupied_address)], ) .await?; continuation_reached.store(true, Ordering::SeqCst); @@ -663,7 +659,7 @@ mod tests { drop(probe); let primary = format!("[::]:{port}").parse().unwrap(); - let listeners = bind_gateway_listeners(primary, &[podman_loopback_listener_requirement()]) + let listeners = bind_gateway_listeners(primary, &[loopback_listener_requirement()]) .await .expect("IPv6 wildcard and IPv4 callback listeners should both bind"); @@ -675,33 +671,33 @@ mod tests { ); } - fn docker_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { + fn exact_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { GatewayListenerRequirement::Exact { address, - driver_name: "docker".to_string(), + driver_name: "alpha".to_string(), reason: "managed bridge".to_string(), } } - fn podman_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { + fn network_listener_requirement(address: SocketAddr) -> GatewayListenerRequirement { GatewayListenerRequirement::Exact { address, - driver_name: "podman".to_string(), - reason: "Podman managed bridge".to_string(), + driver_name: "beta".to_string(), + reason: "managed bridge".to_string(), } } - fn podman_default_route_listener_requirement() -> GatewayListenerRequirement { + fn default_route_listener_requirement() -> GatewayListenerRequirement { GatewayListenerRequirement::DefaultRouteInterface { - driver_name: "podman".to_string(), - reason: "rootless pasta upstream interface".to_string(), + driver_name: "beta".to_string(), + reason: "default route interface".to_string(), } } - fn podman_loopback_listener_requirement() -> GatewayListenerRequirement { + fn loopback_listener_requirement() -> GatewayListenerRequirement { GatewayListenerRequirement::LoopbackInterface { - driver_name: "podman".to_string(), - reason: "Podman machine host forwarder".to_string(), + driver_name: "beta".to_string(), + reason: "host forwarder".to_string(), } } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 7e061460bb..ad7203b833 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -283,8 +283,8 @@ pub struct ServerState { /// Registry of active supervisor sessions and pending relay channels. /// - /// Stored as `Arc` so compute drivers (e.g. the Docker driver) - /// can be constructed before `ServerState` and still + /// Stored as `Arc` so compiled compute drivers can be constructed before + /// `ServerState` and still /// query session state to surface supervisor readiness. pub supervisor_sessions: Arc, @@ -582,7 +582,7 @@ pub(crate) async fn run_server( gateway_tls_enabled: config.tls.is_some(), endpoint_overrides: &config.compute_driver_endpoints, }; - let (compute, operator_allowlist) = build_compute_runtime( + let compute = build_compute_runtime( &compute_drivers, &config, driver_startup, @@ -665,24 +665,29 @@ pub(crate) async fn run_server( compute::driver_config::kubernetes_sa_bootstrap_config(config_file.as_ref())?; let sandbox_namespace = kubernetes_config.namespace.clone(); let sandbox_service_account = kubernetes_config.service_account_name.clone(); - let namespace_validator = match kubernetes_config.workspace_mode.as_str() { - "shared" => auth::k8s_sa::NamespaceValidator::Exact(kubernetes_config.namespace), - "managed" => auth::k8s_sa::NamespaceValidator::Prefix(format!( - "openshell-{}-", - kubernetes_config.gateway_id - )), - "operator" => { - let allowlist = operator_allowlist.clone().unwrap_or_default(); - auth::k8s_sa::NamespaceValidator::Allowlist(allowlist) - } - mode => { - return Err(Error::config(format!( - "invalid Kubernetes workspace_mode '{mode}' for ServiceAccount bootstrap" - ))); - } - }; match kube::Client::try_default().await { Ok(client) => { + let namespace_validator = match kubernetes_config.workspace_mode.as_str() { + "shared" => { + auth::k8s_sa::NamespaceValidator::Exact(kubernetes_config.namespace) + } + "managed" => auth::k8s_sa::NamespaceValidator::Prefix(format!( + "openshell-{}-", + kubernetes_config.gateway_id + )), + "operator" => auth::k8s_sa::NamespaceValidator::Allowlist( + auth::k8s_sa::operator_namespace_allowlist( + &kubernetes_config, + client.clone(), + shutdown_rx.clone(), + )?, + ), + mode => { + return Err(Error::config(format!( + "invalid Kubernetes workspace_mode '{mode}' for ServiceAccount bootstrap" + ))); + } + }; let resolver = Arc::new(auth::k8s_sa::LiveK8sResolver::new( client, namespace_validator, @@ -1062,24 +1067,22 @@ async fn terminate_signal() { let _ = signal.recv().await; } -type OperatorAllowlistArc = Option; pub use compute::{ AcquiredRemoteDriverEndpoint, DriverWatchStream, ManagedDriverProcess, SharedComputeDriver, }; -/// Opaque result returned by a compiled compute-driver factory. -pub struct ComputeDriverBuildOutput { - runtime: ComputeRuntime, - operator_allowlist: OperatorAllowlistArc, +/// Driver instance returned by a compiled compute-driver factory. +pub enum ComputeDriverInstance { + /// A driver hosted in the gateway process. + InProcess(SharedComputeDriver), + /// A driver process launched and owned by the gateway. + ManagedRemote(AcquiredRemoteDriverEndpoint), } /// Factory for a compute driver linked into a gateway binary. #[async_trait::async_trait] pub trait ComputeDriverFactory: Send + Sync { - async fn build( - &self, - context: ComputeDriverBuildContext<'_>, - ) -> Result; + async fn build(&self, context: ComputeDriverBuildContext<'_>) -> Result; } /// One named compiled-driver registration. @@ -1206,13 +1209,9 @@ impl ComputeDriverRegistry { pub struct ComputeDriverBuildContext<'a> { driver_name: String, - config: &'a Config, + gateway_bind_address: SocketAddr, + gateway_log_level: &'a str, driver_startup: compute::driver_config::DriverStartupContext<'a>, - store: Arc, - sandbox_index: SandboxIndex, - sandbox_watch_bus: SandboxWatchBus, - tracing_log_bus: TracingLogBus, - supervisor_sessions: Arc, shutdown_rx: watch::Receiver, inherited_config_keys: &'static [&'static str], } @@ -1224,8 +1223,13 @@ impl ComputeDriverBuildContext<'_> { } #[must_use] - pub fn gateway_config(&self) -> &Config { - self.config + pub fn gateway_bind_address(&self) -> SocketAddr { + self.gateway_bind_address + } + + #[must_use] + pub fn gateway_log_level(&self) -> &str { + self.gateway_log_level } #[must_use] @@ -1269,59 +1273,6 @@ impl ComputeDriverBuildContext<'_> { .file .and_then(|file| file.openshell.gateway.otlp.as_ref()) } - - /// Finish construction of an in-process driver through the common runtime path. - pub async fn finish_in_process( - self, - driver: SharedComputeDriver, - ) -> Result { - self.finish_in_process_with_allowlist(driver, None).await - } - - /// Finish construction while publishing a dynamic authentication allowlist. - pub async fn finish_in_process_with_allowlist( - self, - driver: SharedComputeDriver, - operator_allowlist: Option, - ) -> Result { - let runtime = ComputeRuntime::from_driver( - self.driver_name, - driver, - None, - self.store, - self.sandbox_index, - self.sandbox_watch_bus, - self.tracing_log_bus, - self.supervisor_sessions, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist, - }) - } - - /// Finish construction of a gateway-managed remote driver process. - pub async fn finish_remote( - self, - endpoint: AcquiredRemoteDriverEndpoint, - ) -> Result { - let runtime = ComputeRuntime::new_remote_driver( - endpoint, - self.store, - self.sandbox_index, - self.sandbox_watch_bus, - self.tracing_log_bus, - self.supervisor_sessions, - ) - .await - .map_err(|error| Error::execution(format!("failed to create compute runtime: {error}")))?; - Ok(ComputeDriverBuildOutput { - runtime, - operator_allowlist: None, - }) - } } #[allow(clippy::too_many_arguments)] @@ -1335,28 +1286,64 @@ async fn build_compute_runtime( tracing_log_bus: TracingLogBus, supervisor_sessions: Arc, shutdown_rx: watch::Receiver, -) -> Result<(ComputeRuntime, OperatorAllowlistArc)> { +) -> Result { let driver = configured_compute_driver(registry, config, driver_startup)?; info!(driver = %driver.name(), "Using compute driver"); + if config + .gateway_jwt + .as_ref() + .is_some_and(|jwt| jwt.ttl_secs == 0) + && !driver.is_local_singleplayer(registry) + { + warn!( + "Gateway configured with non-expiring sandbox JWTs; set gateway_jwt.ttl_secs > 0 for shared deployments" + ); + } - let (runtime, operator_allowlist) = match driver { + let runtime = match driver { ConfiguredComputeDriver::Registered(registration) => { - let output = registration + let instance = registration .factory .build(ComputeDriverBuildContext { - driver_name: registration.name, - config, + driver_name: registration.name.clone(), + gateway_bind_address: config.bind_address, + gateway_log_level: &config.log_level, driver_startup, + shutdown_rx, + inherited_config_keys: registration.inherited_config_keys, + }) + .await?; + match instance { + ComputeDriverInstance::InProcess(driver) => ComputeRuntime::from_driver( + registration.name, + driver, + None, store, sandbox_index, sandbox_watch_bus, tracing_log_bus, supervisor_sessions, - shutdown_rx, - inherited_config_keys: registration.inherited_config_keys, - }) - .await?; - (output.runtime, output.operator_allowlist) + ) + .await + .map_err(|error| { + Error::execution(format!("failed to create compute runtime: {error}")) + })?, + ComputeDriverInstance::ManagedRemote(mut endpoint) => { + endpoint.name = registration.name; + ComputeRuntime::new_remote_driver( + endpoint, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions, + ) + .await + .map_err(|error| { + Error::execution(format!("failed to create compute runtime: {error}")) + })? + } + } } ConfiguredComputeDriver::Remote { name } => { let remote_config = @@ -1369,7 +1356,7 @@ async fn build_compute_runtime( let endpoint = compute::connect_remote_compute_driver(name, &remote_config.socket_path) .await .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - let rt = ComputeRuntime::new_remote_driver( + ComputeRuntime::new_remote_driver( endpoint, store, sandbox_index, @@ -1378,12 +1365,11 @@ async fn build_compute_runtime( supervisor_sessions, ) .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))?; - (rt, None) + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))? } }; - Ok((runtime, operator_allowlist)) + Ok(runtime) } #[derive(Debug, Clone)] @@ -1399,6 +1385,15 @@ impl ConfiguredComputeDriver { Self::Remote { name } => name, } } + + fn is_local_singleplayer(&self, registry: &ComputeDriverRegistry) -> bool { + match self { + Self::Registered(registration) => registration.is_local_singleplayer(), + Self::Remote { name } => registry + .get(name) + .is_some_and(ComputeDriverRegistration::is_local_singleplayer), + } + } } fn configured_compute_driver( @@ -1665,7 +1660,7 @@ mod tests { async fn build( &self, _context: super::ComputeDriverBuildContext<'_>, - ) -> openshell_core::Result { + ) -> openshell_core::Result { unreachable!("selection tests do not construct the driver") } } @@ -1979,38 +1974,31 @@ mod tests { #[test] fn configured_compute_driver_triggers_auto_detection_when_empty() { - let config = Config::new(None).with_compute_drivers(std::iter::empty::()); - // Empty drivers triggers auto-detection, which may return Some or None - // depending on the environment. This test verifies the auto-detection path - // is taken rather than immediately returning an error. - let result = configured_compute_driver( - &test_compute_drivers(), - &config, - test_driver_startup(&config, None), - ); - // Either we get a detected driver or an error about none being detected. - match result { - Ok(ConfiguredComputeDriver::Registered(registration)) => { - assert!( - matches!( - registration.name.as_str(), - "kubernetes" | "docker" | "podman" - ), - "auto-detected unexpected driver: {}", - registration.name - ); - } - Ok(ConfiguredComputeDriver::Remote { name }) => { - panic!("auto-detection returned remote driver: {name}"); - } - Err(e) => { - assert!( - e.to_string() - .contains("auto-detection found no suitable driver"), - "unexpected error: {e}" - ); - } + fn available() -> bool { + true } + + let mut registry = super::ComputeDriverRegistry::new(); + registry + .install( + super::ComputeDriverRegistration::new( + "detected", + 100, + Some(available), + TestComputeDriverFactory, + ) + .unwrap(), + ) + .unwrap(); + let config = Config::new(None).with_compute_drivers(std::iter::empty::()); + let result = + configured_compute_driver(®istry, &config, test_driver_startup(&config, None)) + .unwrap(); + + let ConfiguredComputeDriver::Registered(registration) = result else { + panic!("auto-detection must select a registered driver"); + }; + assert_eq!(registration.name, "detected"); } #[test] diff --git a/e2e/no-compute-driver-gateway.sh b/e2e/no-compute-driver-gateway.sh index 64a2b8d841..baa4a7d193 100755 --- a/e2e/no-compute-driver-gateway.sh +++ b/e2e/no-compute-driver-gateway.sh @@ -10,6 +10,7 @@ cd "${ROOT}" echo "Building gateway without compiled compute drivers..." cargo build -p openshell-gateway --bin openshell-gateway \ --no-default-features --features telemetry +cargo check -p openshell-core --no-default-features --all-targets dependency_tree="$(cargo tree -p openshell-gateway \ --no-default-features --features telemetry --edges normal)" From 6714b07c214692d840e83d54645ed66a4dcb2ed5 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 18 Aug 2026 23:27:44 -0700 Subject: [PATCH 06/13] test(e2e): fix external driver setup Signed-off-by: Drew Newberry --- e2e/rust/e2e-vm.sh | 2 ++ e2e/with-kube-gateway.sh | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index ffc32ff0b0..3a38f8f17f 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -53,6 +53,7 @@ DRIVER_BIN="${OPENSHELL_VM_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-vm} CLI_BIN="${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}" E2E_TEST_OVERRIDE="${OPENSHELL_E2E_VM_TEST:-}" E2E_FEATURES="${OPENSHELL_E2E_VM_FEATURES:-e2e-vm}" +SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" # The VM driver places `compute-driver.sock` under `[openshell.drivers.vm].state_dir`. # AF_UNIX SUN_LEN is 104 bytes on macOS (108 on Linux), so paths anchored @@ -296,6 +297,7 @@ if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then --bind-socket "${DRIVER_SOCKET}" \ --allow-same-uid-peer \ --openshell-endpoint "https://host.openshell.internal:${HOST_PORT}" \ + --default-image "${SANDBOX_IMAGE}" \ --state-dir "${RUN_STATE_DIR}" \ --guest-tls-ca "${PKI_DIR}/ca.crt" \ --guest-tls-cert "${PKI_DIR}/client/tls.crt" \ diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 1064714794..e9e20f9c54 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -682,8 +682,14 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ bash "${ROOT}/tasks/scripts/docker-build-image.sh" gateway fi - CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ - bash "${ROOT}/tasks/scripts/docker-build-image.sh" supervisor + supervisor_image="${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" + if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" != "1" ] \ + || ! docker image inspect "${supervisor_image}" >/dev/null 2>&1; then + CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ + bash "${ROOT}/tasks/scripts/docker-build-image.sh" supervisor + else + echo "Reusing existing supervisor image ${supervisor_image}" + fi fi if [ -n "${import_cluster_name}" ]; then From 5c8cbd85acc11355367e686e3413421703548c48 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 18 Aug 2026 23:29:39 -0700 Subject: [PATCH 07/13] docs(server): remove stale backend-specific comments Signed-off-by: Drew Newberry --- crates/openshell-server/src/compute/lease.rs | 8 ++++---- crates/openshell-server/src/compute/mod.rs | 10 +++++----- crates/openshell-server/src/grpc/sandbox.rs | 7 ++----- crates/openshell-server/src/sandbox_index.rs | 2 +- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/crates/openshell-server/src/compute/lease.rs b/crates/openshell-server/src/compute/lease.rs index bf58fae48b..3310946dab 100644 --- a/crates/openshell-server/src/compute/lease.rs +++ b/crates/openshell-server/src/compute/lease.rs @@ -242,10 +242,10 @@ impl ReconcilerLease { /// Derive a stable replica identity for lease ownership. /// -/// Kubernetes sets `HOSTNAME` to the pod name, Docker sets it to the -/// container ID, and systemd units inherit the machine hostname. -/// `OPENSHELL_REPLICA_ID` allows explicit override. The UUID fallback -/// handles edge cases where neither env var is set. +/// Managed workloads commonly receive a stable runtime identity through +/// `HOSTNAME`, while systemd units inherit the machine hostname. +/// `OPENSHELL_REPLICA_ID` allows an explicit override. The UUID fallback +/// handles environments where neither variable is set. pub fn replica_id() -> String { std::env::var("OPENSHELL_REPLICA_ID") .or_else(|_| std::env::var("HOSTNAME")) diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 734dc4a14d..c46f204c0f 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -1196,10 +1196,10 @@ impl ComputeRuntime { let suspension_progressing = expected_stopped && driver_snapshot_confirms_stopping(&snapshot); if suspension_progressing { - // The Kubernetes controller has accepted the stop and - // is waiting for its pod to terminate. Preserve the - // durable transition so a later watch event can complete - // it instead of claiming the sandbox is running again. + // The backend has accepted the stop but has not finished + // terminating the sandbox. Preserve the durable transition + // so a later watch event can complete it instead of claiming + // the sandbox is running again. debug!(sandbox_id, "Sandbox stop is still progressing"); } else if backend_phase == SandboxPhase::Error || observed_stopped == expected_stopped @@ -7482,7 +7482,7 @@ mod tests { #[tokio::test] async fn backend_not_ready_with_supervisor_becomes_ready() { - // VM path: supervisor connects before backend reports Ready. + // The supervisor may connect before the backend reports Ready. let runtime = test_runtime(Arc::new(TestDriver::default())).await; let sandbox = sandbox_record("sb-1", "sandbox-a", SandboxPhase::Provisioning); runtime.store.put_message(&sandbox).await.unwrap(); diff --git a/crates/openshell-server/src/grpc/sandbox.rs b/crates/openshell-server/src/grpc/sandbox.rs index 6d44d01ddc..ec789ca4b4 100644 --- a/crates/openshell-server/src/grpc/sandbox.rs +++ b/crates/openshell-server/src/grpc/sandbox.rs @@ -319,11 +319,8 @@ async fn handle_create_sandbox_inner( status })?; - // Mint the gateway JWT for singleplayer drivers. K8s sandboxes skip - // this mint and bootstrap via `IssueSandboxToken` at supervisor - // startup; identifying "is this K8s?" lives in the compute layer, so - // we mint unconditionally here when the issuer is configured and let - // the K8s driver simply ignore the field. + // Mint a gateway JWT whenever the issuer is configured. Compute runtimes + // that bootstrap through another authentication mechanism may ignore it. let sandbox_token = state.sandbox_jwt_issuer.as_ref().map(|issuer| { issuer.mint(&id).map(|minted| { tracing::info!( diff --git a/crates/openshell-server/src/sandbox_index.rs b/crates/openshell-server/src/sandbox_index.rs index 589f88fd88..c119ca6889 100644 --- a/crates/openshell-server/src/sandbox_index.rs +++ b/crates/openshell-server/src/sandbox_index.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! In-memory indexes for correlating Kubernetes objects back to sandbox ids. +//! In-memory indexes for correlating compute resources back to sandbox ids. use std::collections::HashMap; use std::sync::{Arc, RwLock}; From d4edf84095009b4e3d1298709579fc4072a3df7b Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Tue, 18 Aug 2026 23:58:27 -0700 Subject: [PATCH 08/13] test(e2e): avoid reloading reused supervisor image Signed-off-by: Drew Newberry --- e2e/with-kube-gateway.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index e9e20f9c54..113765615a 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -554,6 +554,7 @@ if [ -z "${OPENSHELL_E2E_KUBE_BUILD_IMAGES+x}" ]; then fi fi +reuse_supervisor_image=0 if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then REGISTRY_VALUE="${OPENSHELL_REGISTRY:-openshell}" IMAGE_TAG_VALUE="${IMAGE_TAG:-e2e-${CLUSTER_NAME:-local}}" @@ -688,6 +689,7 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \ bash "${ROOT}/tasks/scripts/docker-build-image.sh" supervisor else + reuse_supervisor_image=1 echo "Reusing existing supervisor image ${supervisor_image}" fi fi @@ -706,9 +708,13 @@ elif [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ] \ && [[ "${KUBE_CONTEXT}" == kind-* ]] \ && command -v kind >/dev/null 2>&1; then kind_cluster_name="${KUBE_CONTEXT#kind-}" - for image in \ - "${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \ - "${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}"; do + kind_images=("${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}") + # The CI workflow loads its published supervisor archive before invoking this + # wrapper. Only load a supervisor image here when this script rebuilt it. + if [ "${reuse_supervisor_image}" != "1" ]; then + kind_images+=("${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}") + fi + for image in "${kind_images[@]}"; do echo "Loading ${image} into kind cluster ${kind_cluster_name}..." kind load docker-image "${image}" --name "${kind_cluster_name}" done From d15b036eb736f5731884d8e6d5444cf1a8a8a284 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 19 Aug 2026 00:26:24 -0700 Subject: [PATCH 09/13] test(e2e): bundle z3 in driver-free gateway Signed-off-by: Drew Newberry --- e2e/with-kube-gateway.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index 113765615a..d1970f1613 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -661,8 +661,9 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then echo "ERROR: external Kubernetes driver image composition currently requires a Linux build host." >&2 exit 2 fi + # Keep the driver-free gateway portable inside the distroless runtime. cargo build -p openshell-gateway --bin openshell-gateway \ - --no-default-features --features telemetry + --no-default-features --features telemetry,bundled-z3 cargo build -p openshell-driver-kubernetes --bin openshell-driver-kubernetes case "$(uname -m)" in x86_64) external_arch=amd64 ;; From 9f15c12fa82f6457061dfb4b75b83e41c7c27c9a Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 19 Aug 2026 00:52:29 -0700 Subject: [PATCH 10/13] ci(e2e): install bundled z3 build tooling Signed-off-by: Drew Newberry --- .github/workflows/e2e-kubernetes-test.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-kubernetes-test.yml b/.github/workflows/e2e-kubernetes-test.yml index bf13ab7080..3e36570144 100644 --- a/.github/workflows/e2e-kubernetes-test.yml +++ b/.github/workflows/e2e-kubernetes-test.yml @@ -96,14 +96,17 @@ jobs: run: mise install --locked # The openshell-policy crate transitively pulls in z3-sys, whose - # build script needs the z3 C/C++ headers and clang/bindgen to - # compile. The bare runner doesn't ship them; the CI container + # build script needs the z3 C/C++ headers, clang/bindgen, and CMake to + # compile both system-linked and bundled-Z3 builds. The bare runner + # doesn't ship them; the CI container # image used by other Rust e2e jobs does, but we can't run this job # there (the runner's container handler injects its own --network # bridge, which conflicts with the --network host we need so kind's # API server is reachable from the test process). - name: Install z3 build deps - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libz3-dev clang + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends libz3-dev clang cmake - name: Log in to GHCR run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin From a90ce99d893e0d7e713cc3c916b80dbe8b81217f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 19 Aug 2026 01:33:26 -0700 Subject: [PATCH 11/13] test(e2e): stabilize Podman gateway cleanup Signed-off-by: Drew Newberry --- e2e/rust/tests/podman_corporate_proxy.rs | 32 +++++++++++++++++++----- e2e/support/gateway-common.sh | 23 +++++++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/e2e/rust/tests/podman_corporate_proxy.rs b/e2e/rust/tests/podman_corporate_proxy.rs index 2f18fc91c4..70236647bd 100644 --- a/e2e/rust/tests/podman_corporate_proxy.rs +++ b/e2e/rust/tests/podman_corporate_proxy.rs @@ -414,6 +414,28 @@ async fn wait_for_secret_removal(secret: &str, timeout: Duration) -> Result<(), } } +async fn wait_for_proxy_connect( + proxy: &SupportContainer, + allowed_ip: &str, + timeout: Duration, +) -> Result { + let expected = format!("CONNECT {allowed_ip}:{UPSTREAM_PORT} auth=ok"); + let start = std::time::Instant::now(); + loop { + let logs = proxy.logs()?; + if logs.contains(&expected) { + return Ok(logs); + } + if start.elapsed() > timeout { + return Err(format!( + "proxy did not record '{expected}' within {}s. Logs:\n{logs}", + timeout.as_secs() + )); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} + /// Assert the workload's results and the proxy's own record of what it saw. /// /// `output` is the sandbox's stdout; `proxy_logs` is the fake proxy's log, @@ -560,12 +582,10 @@ async fn podman_corporate_proxy_routes_approved_tls_egress() { .await .expect("create sandbox behind the corporate proxy"); - assert_proxied_egress( - &sandbox.create_output, - &proxy.logs().expect("read fake proxy logs"), - &allowed_ip, - &denied_ip, - ); + let proxy_logs = wait_for_proxy_connect(&proxy, &allowed_ip, Duration::from_secs(10)) + .await + .expect("wait for fake proxy CONNECT log"); + assert_proxied_egress(&sandbox.create_output, &proxy_logs, &allowed_ip, &denied_ip); // ── Secret lifecycle ────────────────────────────────────────────── let secrets_live = proxy_auth_secret_names().expect("list proxy-auth secrets while sandbox up"); diff --git a/e2e/support/gateway-common.sh b/e2e/support/gateway-common.sh index 64cc84920e..e2520826db 100644 --- a/e2e/support/gateway-common.sh +++ b/e2e/support/gateway-common.sh @@ -332,6 +332,29 @@ e2e_stop_gateway() { if [ -n "${gateway_pid}" ] && kill -0 "${gateway_pid}" 2>/dev/null; then echo "Stopping openshell-gateway (pid ${gateway_pid})..." kill "${gateway_pid}" 2>/dev/null || true + + # A Rust E2E test may have restarted the gateway and updated the PID file. + # That replacement process is not a child of this shell, so `wait` returns + # immediately even though gateway shutdown (including its sandbox stop + # sweep) is still in progress. Poll until either the process exits or a + # child process becomes a zombie that the final `wait` can reap. + local attempts=0 + local process_state="" + while kill -0 "${gateway_pid}" 2>/dev/null && [ "${attempts}" -lt 120 ]; do + process_state="$(ps -p "${gateway_pid}" -o stat= 2>/dev/null || true)" + case "${process_state}" in + *Z*) break ;; + esac + sleep 0.5 + attempts=$((attempts + 1)) + done + if kill -0 "${gateway_pid}" 2>/dev/null; then + process_state="$(ps -p "${gateway_pid}" -o stat= 2>/dev/null || true)" + case "${process_state}" in + *Z*) ;; + *) kill -KILL "${gateway_pid}" 2>/dev/null || true ;; + esac + fi wait "${gateway_pid}" 2>/dev/null || true fi } From 263f66f491781cef31a6307f6fbcd5b1bc994910 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 19 Aug 2026 01:37:39 -0700 Subject: [PATCH 12/13] test(e2e): isolate external Kubernetes driver setup Signed-off-by: Drew Newberry --- .../openshell/templates/_gateway-workload.tpl | 47 ------------------- .../openshell/templates/gateway-config.yaml | 5 -- .../Dockerfile.external-kubernetes-gateway | 14 ++++++ .../kustomization.yaml | 29 ++++++++++++ .../plugin.yaml | 11 +++++ .../post-renderer.sh | 18 +++++++ .../workload-patch.yaml | 37 +++++++++++++++ e2e/with-kube-gateway.sh | 9 +++- 8 files changed, 117 insertions(+), 53 deletions(-) create mode 100644 e2e/helm-plugins/openshell-external-compute-driver/kustomization.yaml create mode 100644 e2e/helm-plugins/openshell-external-compute-driver/plugin.yaml create mode 100755 e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh create mode 100644 e2e/helm-plugins/openshell-external-compute-driver/workload-patch.yaml diff --git a/deploy/helm/openshell/templates/_gateway-workload.tpl b/deploy/helm/openshell/templates/_gateway-workload.tpl index 4acbe2ee45..5ff608ae59 100644 --- a/deploy/helm/openshell/templates/_gateway-workload.tpl +++ b/deploy/helm/openshell/templates/_gateway-workload.tpl @@ -5,9 +5,6 @@ Gateway pod template shared by the StatefulSet and Deployment workload shapes. */}} {{- define "openshell.gatewayPodTemplate" -}} -{{- $testing := index .Values "testing" | default (dict) -}} -{{- $externalKubernetesComputeDriver := index $testing "externalKubernetesComputeDriver" | default (dict) -}} -{{- $externalKubernetesComputeDriverSocket := $externalKubernetesComputeDriver.socketPath | default "/var/run/openshell-compute/driver.sock" -}} metadata: annotations: # Roll the gateway workload when the rendered gateway TOML changes - the @@ -81,10 +78,6 @@ spec: - name: OPENSHELL_TELEMETRY_ENABLED value: {{ .Values.server.telemetryEnabled | quote }} volumeMounts: - {{- if $externalKubernetesComputeDriver.enabled }} - - name: compute-driver-socket - mountPath: {{ dir $externalKubernetesComputeDriverSocket | quote }} - {{- end }} {{- if eq (include "openshell.workloadKind" .) "statefulset" }} - name: openshell-data mountPath: /var/openshell @@ -152,47 +145,7 @@ spec: failureThreshold: {{ .Values.probes.readiness.failureThreshold }} resources: {{- toYaml .Values.resources | nindent 8 }} - {{- if $externalKubernetesComputeDriver.enabled }} - - name: kubernetes-compute-driver - securityContext: - {{- toYaml .Values.securityContext | nindent 8 }} - image: {{ include "openshell.image" . | quote }} - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: ["/usr/local/bin/openshell-driver-kubernetes"] - args: - - --bind-socket - - {{ $externalKubernetesComputeDriverSocket | quote }} - - --workspace-mode - - {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} - - --gateway-id - - {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} - - --sandbox-namespace - - {{ include "openshell.sandboxNamespace" . | quote }} - - --sandbox-service-account - - {{ include "openshell.sandboxServiceAccountName" . | quote }} - - --grpc-endpoint - - {{ include "openshell.grpcEndpoint" . | quote }} - - --sandbox-image - - {{ .Values.server.sandboxImage | quote }} - - --supervisor-image - - {{ include "openshell.supervisorImage" . | quote }} - - --supervisor-sideload-method - - {{ include "openshell.supervisorSideloadMethod" . | quote }} - - --topology - - {{ .Values.supervisor.topology | default "combined" | quote }} - - --client-tls-secret-name - - {{ .Values.server.tls.clientTlsSecretName | quote }} - volumeMounts: - - name: compute-driver-socket - mountPath: {{ dir $externalKubernetesComputeDriverSocket | quote }} - resources: - {{- toYaml .Values.resources | nindent 8 }} - {{- end }} volumes: - {{- if $externalKubernetesComputeDriver.enabled }} - - name: compute-driver-socket - emptyDir: {} - {{- end }} - name: gateway-config configMap: name: {{ include "openshell.fullname" . }}-config diff --git a/deploy/helm/openshell/templates/gateway-config.yaml b/deploy/helm/openshell/templates/gateway-config.yaml index 00e38215c0..9d24dbd917 100644 --- a/deploy/helm/openshell/templates/gateway-config.yaml +++ b/deploy/helm/openshell/templates/gateway-config.yaml @@ -13,8 +13,6 @@ One value is intentionally NOT rendered here: --db-url arg for SQLite */}} {{- $credentialDrivers := list -}} -{{- $testing := index .Values "testing" | default (dict) -}} -{{- $externalKubernetesComputeDriver := index $testing "externalKubernetesComputeDriver" | default (dict) -}} {{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled -}} {{- $credentialDrivers = append $credentialDrivers "kubernetes-secrets" -}} {{- end -}} @@ -137,9 +135,6 @@ data: {{- end }} [openshell.drivers.kubernetes] - {{- if $externalKubernetesComputeDriver.enabled }} - socket_path = {{ $externalKubernetesComputeDriver.socketPath | default "/var/run/openshell-compute/driver.sock" | quote }} - {{- end }} workspace_mode = {{ .Values.server.drivers.kubernetes.workspaceMode | default "shared" | quote }} gateway_id = {{ .Values.server.sandboxJwt.gatewayId | default (include "openshell.fullname" .) | quote }} grpc_endpoint = {{ include "openshell.grpcEndpoint" . | quote }} diff --git a/e2e/docker/Dockerfile.external-kubernetes-gateway b/e2e/docker/Dockerfile.external-kubernetes-gateway index 15e43b9e6b..5d650bae89 100644 --- a/e2e/docker/Dockerfile.external-kubernetes-gateway +++ b/e2e/docker/Dockerfile.external-kubernetes-gateway @@ -7,9 +7,23 @@ ARG GATEWAY_BASE_IMAGE=gcr.io/distroless/cc-debian13:nonroot@sha256:d97bc0a941b8 FROM ${GATEWAY_BASE_IMAGE} ARG TARGETARCH +ARG SUPERVISOR_IMAGE=ghcr.io/nvidia/openshell/supervisor:latest COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-gateway /usr/local/bin/openshell-gateway COPY deploy/docker/.build/prebuilt-binaries/${TARGETARCH}/openshell-driver-kubernetes /usr/local/bin/openshell-driver-kubernetes +ENV OPENSHELL_DRIVERS=kubernetes \ + OPENSHELL_COMPUTE_DRIVER_SOCKET=/var/run/openshell-compute/driver/driver.sock \ + OPENSHELL_GATEWAY_ID=openshell \ + OPENSHELL_SANDBOX_NAMESPACE=openshell \ + OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT=openshell-sandbox \ + OPENSHELL_SANDBOX_IMAGE=ghcr.io/nvidia/openshell-community/sandboxes/base:latest \ + OPENSHELL_SANDBOX_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_GRPC_ENDPOINT=http://openshell.openshell.svc.cluster.local:8080 \ + OPENSHELL_SUPERVISOR_IMAGE=${SUPERVISOR_IMAGE} \ + OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=IfNotPresent \ + OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container \ + OPENSHELL_K8S_TOPOLOGY=combined + USER 1000:1000 EXPOSE 8080 ENTRYPOINT ["/usr/local/bin/openshell-gateway"] diff --git a/e2e/helm-plugins/openshell-external-compute-driver/kustomization.yaml b/e2e/helm-plugins/openshell-external-compute-driver/kustomization.yaml new file mode 100644 index 0000000000..3c01ab6963 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/kustomization.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rendered.yaml +patches: + - path: workload-patch.yaml + target: + group: apps + version: v1 + kind: StatefulSet + name: openshell +replacements: + - source: + group: apps + version: v1 + kind: StatefulSet + name: openshell + fieldPath: spec.template.spec.containers.[name=openshell-gateway].image + targets: + - select: + group: apps + version: v1 + kind: StatefulSet + name: openshell + fieldPaths: + - spec.template.spec.containers.[name=kubernetes-compute-driver].image diff --git a/e2e/helm-plugins/openshell-external-compute-driver/plugin.yaml b/e2e/helm-plugins/openshell-external-compute-driver/plugin.yaml new file mode 100644 index 0000000000..2a58ba4f06 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/plugin.yaml @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: v1 +type: postrenderer/v1 +name: openshell-external-compute-driver +version: 0.1.0 +runtime: subprocess +runtimeConfig: + platformCommand: + - command: ${HELM_PLUGIN_DIR}/post-renderer.sh diff --git a/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh new file mode 100755 index 0000000000..4d9d2eeff2 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/post-renderer.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Helm post-renderer for the external Kubernetes compute-driver smoke test. +# It keeps the test-only sidecar and Unix socket plumbing out of the chart. + +set -euo pipefail + +plugin_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +work_dir="$(mktemp -d "${TMPDIR:-/tmp}/openshell-external-compute-driver.XXXXXX")" +trap 'rm -rf "${work_dir}"' EXIT + +cp "${plugin_dir}/kustomization.yaml" "${work_dir}/kustomization.yaml" +cp "${plugin_dir}/workload-patch.yaml" "${work_dir}/workload-patch.yaml" +tee "${work_dir}/rendered.yaml" >/dev/null + +kubectl kustomize "${work_dir}" diff --git a/e2e/helm-plugins/openshell-external-compute-driver/workload-patch.yaml b/e2e/helm-plugins/openshell-external-compute-driver/workload-patch.yaml new file mode 100644 index 0000000000..2b041e8647 --- /dev/null +++ b/e2e/helm-plugins/openshell-external-compute-driver/workload-patch.yaml @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: openshell +spec: + template: + spec: + containers: + - name: openshell-gateway + volumeMounts: + - name: compute-driver-socket + mountPath: /var/run/openshell-compute + - name: kubernetes-compute-driver + image: replaced-by-kustomize + imagePullPolicy: IfNotPresent + command: + - /usr/local/bin/openshell-driver-kubernetes + args: + - --bind-socket + - /var/run/openshell-compute/driver/driver.sock + securityContext: + runAsNonRoot: true + runAsUser: 1000 + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + - name: compute-driver-socket + mountPath: /var/run/openshell-compute + resources: {} + volumes: + - name: compute-driver-socket + emptyDir: {} diff --git a/e2e/with-kube-gateway.sh b/e2e/with-kube-gateway.sh index d1970f1613..94b2e78be4 100755 --- a/e2e/with-kube-gateway.sh +++ b/e2e/with-kube-gateway.sh @@ -380,6 +380,7 @@ run_scenario() { --set "image.tag=${IMAGE_TAG_VALUE}" \ --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ + "${helm_post_renderer_args[@]}" \ "$@" \ --wait --timeout 5m HELM_INSTALLED=1 @@ -677,6 +678,7 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then "${external_stage}/openshell-driver-kubernetes" docker build \ --build-arg "TARGETARCH=${external_arch}" \ + --build-arg "SUPERVISOR_IMAGE=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" \ --tag "${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \ --file "${ROOT}/e2e/docker/Dockerfile.external-kubernetes-gateway" \ "${ROOT}" @@ -737,13 +739,17 @@ if [ "${OPENSHELL_E2E_CREDENTIAL_DRIVERS:-0}" = "1" ] \ fi helm_extra_args=() +helm_post_renderer_args=() helm_extra_args+=(--set "server.telemetryEnabled=${OPENSHELL_TELEMETRY_ENABLED}") if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" != "1" ]; then echo "ERROR: external Kubernetes driver e2e requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=1." >&2 exit 2 fi - helm_extra_args+=(--set "testing.externalKubernetesComputeDriver.enabled=true") + export HELM_PLUGINS="${ROOT}/e2e/helm-plugins" + helm_post_renderer_args+=( + --post-renderer openshell-external-compute-driver + ) fi if [ -n "${HOST_GATEWAY_IP}" ]; then helm_extra_args+=(--set "server.hostGatewayIP=${HOST_GATEWAY_IP}") @@ -866,6 +872,7 @@ else --set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \ --set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \ "${helm_extra_args[@]}" \ + "${helm_post_renderer_args[@]}" \ --wait --timeout 5m HELM_INSTALLED=1 From 7070b2d58088d196a9f293c63083aae98fcdc49f Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 19 Aug 2026 02:14:40 -0700 Subject: [PATCH 13/13] test(e2e): scope external Podman smoke Signed-off-by: Drew Newberry --- .github/workflows/e2e-test.yml | 5 +++- e2e/rust/tests/podman_corporate_proxy.rs | 32 +++++------------------- 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 9fdc5d698b..d633302c0d 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -265,7 +265,10 @@ jobs: run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin - name: Run rootless Podman E2E - run: env -u OPENSHELL_GATEWAY_BIN OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER=1 mise run --no-deps --skip-deps e2e:podman:rootless + run: mise run --no-deps --skip-deps e2e:podman:rootless + + - name: Run external Podman driver E2E + run: env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:podman:external-driver - name: Print AppArmor denials if: always() diff --git a/e2e/rust/tests/podman_corporate_proxy.rs b/e2e/rust/tests/podman_corporate_proxy.rs index 70236647bd..2f18fc91c4 100644 --- a/e2e/rust/tests/podman_corporate_proxy.rs +++ b/e2e/rust/tests/podman_corporate_proxy.rs @@ -414,28 +414,6 @@ async fn wait_for_secret_removal(secret: &str, timeout: Duration) -> Result<(), } } -async fn wait_for_proxy_connect( - proxy: &SupportContainer, - allowed_ip: &str, - timeout: Duration, -) -> Result { - let expected = format!("CONNECT {allowed_ip}:{UPSTREAM_PORT} auth=ok"); - let start = std::time::Instant::now(); - loop { - let logs = proxy.logs()?; - if logs.contains(&expected) { - return Ok(logs); - } - if start.elapsed() > timeout { - return Err(format!( - "proxy did not record '{expected}' within {}s. Logs:\n{logs}", - timeout.as_secs() - )); - } - tokio::time::sleep(Duration::from_millis(100)).await; - } -} - /// Assert the workload's results and the proxy's own record of what it saw. /// /// `output` is the sandbox's stdout; `proxy_logs` is the fake proxy's log, @@ -582,10 +560,12 @@ async fn podman_corporate_proxy_routes_approved_tls_egress() { .await .expect("create sandbox behind the corporate proxy"); - let proxy_logs = wait_for_proxy_connect(&proxy, &allowed_ip, Duration::from_secs(10)) - .await - .expect("wait for fake proxy CONNECT log"); - assert_proxied_egress(&sandbox.create_output, &proxy_logs, &allowed_ip, &denied_ip); + assert_proxied_egress( + &sandbox.create_output, + &proxy.logs().expect("read fake proxy logs"), + &allowed_ip, + &denied_ip, + ); // ── Secret lifecycle ────────────────────────────────────────────── let secrets_live = proxy_auth_secret_names().expect("list proxy-auth secrets while sandbox up");