diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index f84d792910..8f3a8b5c33 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -117,7 +117,6 @@ LSM decisions remain authoritative. | `restart_policy = no` | Keeps canonical main-process exit terminal. | | `PidsLimit` | Applies the configured sandbox PID budget. Omit `sandbox_pids_limit` to use OpenShell's default. Explicit zero is invalid. | | Private named volumes | Separate volumes carry sandbox bootstrap keys, the runtime-initiated Unix socket, and supervisor-only credentials. The supervisor mounts bootstrap keys read-only and only the socket volume writable. | -| In-memory `/run/openshell-supervisor-ca` tmpfs | Holds only the public supervisor CA certificate and trust bundle without making all of `/run` writable. | | CDI GPU request | Assigns the exact validated CDI devices requested by driver config or count-based selection. | New containers dial the supervisor at `/.openshell/channel/control/supervisor/control.sock`. diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d65aeeed36..95d10dce9f 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -6328,13 +6328,9 @@ fn build_container_create_body_for_image( // Podman, Kubernetes, and VM resolvers, send names to policy DNS // exactly as written so short names never expand to host domains. dns_search: Some(vec![".".to_string()]), - tmpfs: Some(HashMap::from([( - openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR.to_string(), - format!( - "rw,noexec,nosuid,nodev,size=1m,uid={},gid={},mode=0755", - workload_identity.uid, workload_identity.gid - ), - )])), + // No writable runtime storage: provider and CA trust files are + // served from the sandbox runtime's memory. + tmpfs: None, sysctls: Some(HashMap::from([( "net.ipv4.ip_unprivileged_port_start".to_string(), "0".to_string(), diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index c022adc923..8cdd73ded8 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -3772,17 +3772,10 @@ fn docker_supervisor_network_mode_is_an_operator_config_enum() { } #[test] -fn build_container_create_body_limits_writable_runtime_storage_to_supervisor_ca() { +fn build_container_create_body_mounts_no_writable_runtime_storage() { let create_body = build_container_create_body(&test_sandbox(), &runtime_config()).unwrap(); let host_config = create_body.host_config.expect("host_config is populated"); - let tmpfs = host_config.tmpfs.expect("sandbox tmpfs is populated"); - - assert_eq!(tmpfs.len(), 1); - assert_eq!( - tmpfs.get(openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR), - Some(&"rw,noexec,nosuid,nodev,size=1m,uid=1000,gid=1000,mode=0755".to_string()) - ); - assert!(!tmpfs.contains_key("/run")); + assert!(host_config.tmpfs.is_none_or(|tmpfs| tmpfs.is_empty())); } #[test] diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 9943d7a0b6..0a66de79fe 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1675,21 +1675,6 @@ pub fn build_isolation_specs( workload .mounts .retain(|mount| !trusted_mount(&mount.destination)); - workload.mounts.push(Mount { - kind: "tmpfs".into(), - source: "tmpfs".into(), - destination: openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT.into(), - options: vec![ - "rw".into(), - "noexec".into(), - "nosuid".into(), - "nodev".into(), - // Libpod's OCI `mounts` API rejects tmpfs uid/gid options. The - // unprivileged runtime creates the owned material subdirectory. - "mode=0777".into(), - "size=1m".into(), - ], - }); workload.volumes.push(NamedVolume { name: channel.clone(), dest: crate::isolation::CHANNEL_ROOT.into(), @@ -2234,26 +2219,13 @@ mod tests { .iter() .all(|mount| !trusted_mount(&mount.destination)) ); - let supervisor_ca_mount = specs - .workload - .mounts - .iter() - .find(|mount| { - mount.destination == openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT - }) - .expect("workload supervisor CA mount"); - assert_eq!(supervisor_ca_mount.kind, "tmpfs"); - assert_eq!(supervisor_ca_mount.source, "tmpfs"); - for option in ["rw", "noexec", "nosuid", "nodev", "mode=0777", "size=1m"] { - assert!(supervisor_ca_mount.options.contains(&option.to_string())); - } - assert!( - specs - .workload - .mounts - .iter() - .all(|mount| mount.destination != "/run") - ); + // The CA trust files are virtual: no workload mount carries them. + assert!(specs.workload.mounts.iter().all(|mount| { + !mount + .destination + .starts_with(openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_ROOT) + && mount.destination != "/run" + })); assert_eq!(specs.supervisor.secrets.len(), 1); assert_eq!(specs.supervisor.secrets[0].source, "jwt"); assert_eq!(specs.supervisor.secrets[0].uid, 1000); diff --git a/crates/openshell-sandbox-backend/src/lib.rs b/crates/openshell-sandbox-backend/src/lib.rs index 3df3f90ef7..b56b9a5dbe 100644 --- a/crates/openshell-sandbox-backend/src/lib.rs +++ b/crates/openshell-sandbox-backend/src/lib.rs @@ -42,14 +42,13 @@ pub const GPU_RESOURCE_CLAIM: &str = "openshell.gpu"; pub const ALLOW_EXTRA_SUPPLEMENTARY_GROUPS_RESOURCE_CLAIM: &str = "openshell.identity.allow_extra_supplementary_groups"; -/// Memory-backed parent used for supervisor CA material. +/// Workload path reserved for the supervisor CA material. Nothing is mounted +/// or written there: the sandbox runtime serves the files virtually, and +/// drivers keep user mounts away from it. pub const SUPERVISOR_CA_RUNTIME_ROOT: &str = "/run/openshell-supervisor-ca"; -/// Workload-visible directory for the supervisor's public HTTPS interception -/// certificate and combined trust bundle. -/// -/// Keeping the material below the mount root lets runtimes that cannot assign -/// tmpfs ownership create this child as the unprivileged sandbox identity. +/// Workload-visible directory of the supervisor's public HTTPS interception +/// certificate and combined trust bundle, as named by the TLS environment. pub const SUPERVISOR_CA_RUNTIME_DIR: &str = "/run/openshell-supervisor-ca/material"; /// Generated gRPC transport envelope for the `OpenShell` Sandbox Protocol. diff --git a/crates/openshell-sandbox/README.md b/crates/openshell-sandbox/README.md index 7e5dc6db3a..368d523fd1 100644 --- a/crates/openshell-sandbox/README.md +++ b/crates/openshell-sandbox/README.md @@ -42,39 +42,77 @@ level or option that does not fit a non-negative C `int`, still notifies and keeps the broker's `EPERM` and `EINVAL` results. The active qualification probe exercises both sides. -## Provider files - -Provider files appear to the workload beneath `/run/openshell/providers`. -Every workload `open`, `openat`, and `openat2` notifies the broker, which -serves a provider path by injecting a read-only descriptor for a sealed -in-memory file and continues every other open. - -Each published file is an immutable revision backed by one in-memory file. The -runtime creates that file when the revision is published, not when the -workload opens it: it writes the content, seals the file against writes, -growing, shrinking, and further sealing, and keeps only a read-only descriptor -for it. Each workload open reopens that descriptor through the runtime's own -`/proc/self/fd`, so every open gets its own read-only open file description -and file offset, positioned at the start. - -An update prepares every new revision before it replaces the published set in -one step, so a failed update leaves the previous set in place. A file whose -bytes did not change keeps its revision. Opens after an update see the new -revisions, descriptors opened earlier keep the bytes they opened, and a -detached file is `ENOENT` for later opens. - -All opens of one revision refer to one file, which the workload owns: it can -change that file's mode or reopen a descriptor for writing. Neither reaches -the content, which the seals protect, and the runtime restores the mode if it -denies an open. The runtime holds one descriptor per published file, at most -64. - -The broker classifies each open from an exact copy of the first six bytes of -its path. Six bytes identify a `/proc/` path and the start of a provider path; -only those candidates have their whole path read and decided. An open whose -first six bytes cannot be read, or match neither, continues, and unrelated -`openat2` calls do not require a copy of their `open_how`. Every continued -open remains subject to the kernel and Landlock. +## Virtual files: provider files and supervisor CA trust files + +Provider files appear to the workload beneath `/run/openshell/providers`, and +the supervisor's public CA certificate and combined trust bundle at +`/run/openshell-supervisor-ca/material/ca.crt` and `ca-bundle.crt`, where the +TLS environment variables point. None of them exists on a filesystem, and no +driver mounts or Landlock grant is involved. Every workload `open`, `openat`, +and `openat2` notifies the broker, which serves a published path by injecting +a read-only descriptor for a sealed in-memory file and continues every other +open. Every path-based `stat`, `lstat`, `fstatat`, `statx`, and `access` call +notifies too; for a published path the broker writes the metadata the +injected descriptor reports into the caller's buffer, and continues every +other path. + +Supported semantics are exactly: an absolute canonical path, read-only opens +(a write-capable open is `EACCES`, `O_DIRECTORY` is `ENOTDIR`, `O_PATH` and +`O_DIRECT` are `EINVAL`, `openat2` with a nonzero `mode` or `resolve` is +`EINVAL`), and the metadata calls above with the kernel's own flag sets. The +directories do not exist: `chdir`, `opendir`, a relative lookup, and an alias +through `..` are the kernel's `ENOENT`. A path beneath the provider prefix +that is not published is `ENOENT`; the CA paths continue to the kernel until +a pair is published. + +Each published file is an immutable revision backed by one sealed in-memory +file (`linux::sealed_file`). The runtime writes the content, seals the file +against writes, growing, shrinking, and further sealing, and keeps only a +read-only descriptor. Each workload open reopens that descriptor through the +runtime's own `/proc/self/fd`, so every open gets its own file offset. An +update prepares every new revision before it replaces the published set in +one step; a file whose bytes did not change keeps its revision; descriptors +opened earlier keep the bytes they opened. The CA pair is published before +the spawn, replaced only as a whole, never touched by a provider update, and +cleared by a start without CA material. It is refused if either file is +empty, exceeds the 1 MiB control frame bound, or the bundle does not contain +the certificate. + +The broker classifies each call from an exact copy of the first six bytes of +its path; only `/proc/` paths and candidates for the prefixes above have +their whole path read. + +### Metadata writes and signal delivery + +A `stat` result is delivered by writing into the workload. The broker opens +`/proc//mem`, which binds to one address space, and only then validates +the notification: a notification stays valid only while its task lives, so a +valid one proves the pinned space belongs to the notifying task and not to a +task that reused the number (`process_vm_writev` resolves the number again on +every call and cannot give that proof). A write that fails for any reason +fails the call with `EFAULT`; a task killed meanwhile is no longer answered. +Writes through `/proc//mem` use the kernel's `FOLL_FORCE`, which differs +from a native `stat` in one case: a result buffer in the caller's own +read-only private copy-on-write memory is written instead of failing with +`EFAULT`. The kernel refuses forced writes to shared and non-`VM_MAYWRITE` +mappings, so this reaches only memory the caller could already make writable +with `mprotect`. + +For that write to be safe the workload listener is installed with +`SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV` (Linux 5.19). A task can have only +one listener, so the flag covers every notified syscall: before the broker +receives a notification, signals interrupt the syscall as before; once +received, only a fatal signal ends the wait and a handled signal is delivered +after the broker responds. A `connect` awaiting the supervisor's network +admission decision therefore ignores handled `SIGALRM` and `SIGINT` until the +broker answers or its 30-second decision timeout fails the call; DNS relay +waits are bounded at 10 seconds. A blocking loopback `connect` is completed by +the broker with a native `connect` on a duplicate of the socket and waits for +the kernel's connect timeout, which can exceed 30 seconds. Fatal signals still +terminate the task. The runtime qualification probe writes into a dumpable child +through `/proc//mem` and fails closed where the posture refuses it. + +## Runtime boundary transport A boundary configured with `RuntimeInitiatedTlsTcp` binds no listener. It dials only the driver-provided supervisor addresses, which must be concrete, diff --git a/crates/openshell-sandbox/src/boundary_exec.rs b/crates/openshell-sandbox/src/boundary_exec.rs index d95ce62e4c..08f7579055 100644 --- a/crates/openshell-sandbox/src/boundary_exec.rs +++ b/crates/openshell-sandbox/src/boundary_exec.rs @@ -220,9 +220,7 @@ impl LocalBoundaryExec { workdir: Option<&str>, ) -> Result, BackendError> { crate::sandbox::linux::log_sandbox_readiness(&self.policy, workdir); - let runtime_read_only = - crate::process::ca_runtime_read_only_paths(self.ca_file_paths.as_deref()); - crate::process::prepare_child_sandbox(&self.policy, workdir, &runtime_read_only) + crate::process::prepare_child_sandbox(&self.policy, workdir) .map_err(|error| BackendError::Process(error.to_string())) } @@ -817,7 +815,10 @@ mod tests { std::thread::spawn(move || { while let Ok(notification) = listener.receive() { let syscall = i64::from(notification.syscall); - if syscall == libc::SYS_openat || syscall == libc::SYS_openat2 { + if syscall == libc::SYS_openat + || syscall == libc::SYS_openat2 + || crate::linux::seccomp_notify::is_metadata_syscall(syscall) + { let _ = listener.respond_continue(notification.id); } else { #[cfg(target_arch = "x86_64")] diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 9975d54055..cf97ffbc6a 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -3981,9 +3981,35 @@ mod linux { format!("invalid provider files: {error}"), ); } - let ca_file_paths = match install_ca_material(ca_cert, ca_bundle) { - Ok(paths) => paths, - Err(error) => return guest_error(BoundaryErrorKind::Process, error), + // The CA trust files exist only in the broker: publish the pair + // before anything can open it. A refused publication keeps the + // previously published pair, if any, and fails the start. A + // start without CA material forgets a pair cached by an earlier + // failed start before the spawn. + let ca_file_paths = match (ca_cert, ca_bundle) { + (Some(ca_cert), Some(ca_bundle)) => { + if let Err(error) = self + .network_broker + .provider_files() + .publish_ca_trust(ca_cert.as_bytes(), ca_bundle.as_bytes()) + { + return guest_error( + BoundaryErrorKind::Process, + format!("publish supervisor CA trust files: {error}"), + ); + } + Some(crate::ca_trust_files::canonical_paths()) + } + (None, None) => { + self.network_broker.provider_files().clear_ca_trust(); + None + } + _ => { + return guest_error( + BoundaryErrorKind::Process, + "supervisor CA certificate and bundle must be supplied together", + ); + } }; let mut policy = policy.into(); let gpu_requested = self @@ -4254,126 +4280,6 @@ mod linux { } } - fn install_ca_material( - ca_cert: Option, - ca_bundle: Option, - ) -> Result, String> { - let (ca_cert, ca_bundle) = match (ca_cert, ca_bundle) { - (Some(ca_cert), Some(ca_bundle)) => (ca_cert, ca_bundle), - (None, None) => return Ok(None), - _ => { - return Err( - "supervisor CA certificate and bundle must be supplied together".to_string(), - ); - } - }; - install_ca_material_at( - Path::new(openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR), - ca_cert.as_bytes(), - ca_bundle.as_bytes(), - ) - } - - fn install_ca_material_at( - directory: &Path, - ca_cert: &[u8], - ca_bundle: &[u8], - ) -> Result, String> { - use std::io::Write as _; - use std::os::unix::fs::{OpenOptionsExt as _, PermissionsExt as _}; - - let parent = directory - .parent() - .ok_or_else(|| "supervisor CA directory has no parent".to_string())?; - for path in [parent, directory] { - match std::fs::symlink_metadata(path) { - Ok(metadata) if metadata.file_type().is_symlink() => { - return Err(format!( - "supervisor CA directory component is a symlink: {}", - path.display() - )); - } - Ok(metadata) if !metadata.is_dir() => { - return Err(format!( - "supervisor CA directory component is not a directory: {}", - path.display() - )); - } - Ok(_) => {} - Err(error) if error.kind() == io::ErrorKind::NotFound => { - std::fs::create_dir(path).map_err(|error| { - format!("create supervisor CA directory {}: {error}", path.display()) - })?; - } - Err(error) => { - return Err(format!( - "inspect supervisor CA directory {}: {error}", - path.display() - )); - } - } - let current_mode = std::fs::metadata(path) - .map_err(|error| { - format!( - "inspect supervisor CA directory permissions {}: {error}", - path.display() - ) - })? - .permissions() - .mode(); - if path == directory { - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o755)).map_err( - |error| { - format!( - "set supervisor CA directory permissions {}: {error}", - path.display() - ) - }, - )?; - } else if current_mode & 0o111 != 0o111 { - return Err(format!( - "supervisor CA parent is not traversable by workload identities: {}", - path.display() - )); - } - } - let ca_path = directory.join("ca.crt"); - let bundle_path = directory.join("ca-bundle.crt"); - for (path, contents, label) in [ - (&ca_path, ca_cert, "supervisor CA"), - (&bundle_path, ca_bundle, "supervisor CA bundle"), - ] { - let temporary = path.with_extension("tmp"); - if let Ok(metadata) = std::fs::symlink_metadata(&temporary) { - if !metadata.is_file() || metadata.file_type().is_symlink() { - return Err(format!( - "refusing unsafe temporary {label} path: {}", - temporary.display() - )); - } - std::fs::remove_file(&temporary) - .map_err(|error| format!("remove stale temporary {label}: {error}"))?; - } - let mut file = std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o444) - .custom_flags(libc::O_NOFOLLOW) - .open(&temporary) - .map_err(|error| format!("create temporary {label}: {error}"))?; - if let Err(error) = file - .write_all(contents) - .and_then(|()| file.sync_all()) - .and_then(|()| file.set_permissions(std::fs::Permissions::from_mode(0o444))) - .and_then(|()| std::fs::rename(&temporary, path)) - { - let _ = std::fs::remove_file(&temporary); - return Err(format!("install {label}: {error}")); - } - } - Ok(Some((ca_path, bundle_path))) - } - type ProcessExit = Result; type SharedProcessExit = Arc<(Mutex>, Condvar)>; @@ -5706,65 +5612,6 @@ mod linux { assert!(!debug.contains(&verification_key.public_key_pem)); } - #[test] - fn installed_supervisor_ca_is_readable_by_a_non_root_workload_identity() { - use std::os::unix::fs::PermissionsExt as _; - - let root = tempfile::tempdir().expect("temporary CA root"); - std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); - let directory = root.path().join("openshell-supervisor-ca"); - let (ca_path, bundle_path) = install_ca_material_at( - &directory, - b"public test certificate", - b"public test bundle", - ) - .expect("install supervisor CA") - .expect("CA paths"); - - assert_eq!( - std::fs::metadata(directory.parent().unwrap()) - .unwrap() - .permissions() - .mode() - & 0o111, - 0o111, - "non-root workload identities must be able to traverse the full path" - ); - assert_eq!( - std::fs::metadata(&directory).unwrap().permissions().mode() & 0o777, - 0o755, - "non-root workload identities must be able to traverse the CA directory" - ); - for (path, expected) in [ - (&ca_path, b"public test certificate".as_slice()), - (&bundle_path, b"public test bundle".as_slice()), - ] { - assert_eq!( - std::fs::metadata(path).unwrap().permissions().mode() & 0o777, - 0o444, - "public CA material must be readable by the workload" - ); - assert_eq!(std::fs::read(path).unwrap(), expected); - } - } - - #[test] - fn supervisor_ca_install_rejects_a_symlinked_directory() { - let root = tempfile::tempdir().expect("temporary CA root"); - let target = root.path().join("target"); - std::fs::create_dir(&target).unwrap(); - let parent = root.path(); - std::os::unix::fs::symlink(&target, parent.join("openshell-supervisor-ca")).unwrap(); - - let error = install_ca_material_at( - &parent.join("openshell-supervisor-ca"), - b"certificate", - b"bundle", - ) - .expect_err("symlinked CA directory must fail closed"); - assert!(error.contains("symlink"), "unexpected error: {error}"); - } - #[test] fn supplementary_group_measurement_excludes_the_primary_group() { assert_eq!( diff --git a/crates/openshell-sandbox/src/ca_trust_files.rs b/crates/openshell-sandbox/src/ca_trust_files.rs new file mode 100644 index 0000000000..61f99f3023 --- /dev/null +++ b/crates/openshell-sandbox/src/ca_trust_files.rs @@ -0,0 +1,310 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The supervisor's public CA certificate and combined trust bundle, served +//! as virtual files. +//! +//! Nothing is written to any filesystem. The broker holds both files as one +//! sealed in-memory pair and answers every open, `stat`, `statx`, and +//! `access` of exactly the two canonical paths from it, with the semantics +//! described in [`crate::provider_files`]; the TLS environment variables +//! point at those paths. Both files are public material the supervisor +//! already sends the workload, and no private key is ever here. +//! +//! The two files are one generation. A publication prepares both before it +//! replaces the pair, so a failed update leaves the previous complete pair in +//! place, and a publication of the bytes already published keeps the files +//! every earlier open used. A start without CA material clears the pair. + +use std::io; +use std::sync::{Arc, RwLock}; + +use rustix::fs::Mode; + +use openshell_sandbox_backend::boundary_protocol::MAX_CONTROL_FRAME_BYTES; + +use crate::linux::sealed_file::{OwnDescriptors, SealedFile}; + +/// Canonical workload path of the supervisor CA certificate. +pub const CA_CERT_PATH: &str = "/run/openshell-supervisor-ca/material/ca.crt"; +/// Canonical workload path of the combined trust bundle. +pub const CA_BUNDLE_PATH: &str = "/run/openshell-supervisor-ca/material/ca-bundle.crt"; + +/// The canonical `(certificate, bundle)` paths the TLS environment names. +#[must_use] +pub fn canonical_paths() -> (std::path::PathBuf, std::path::PathBuf) { + (CA_CERT_PATH.into(), CA_BUNDLE_PATH.into()) +} + +/// The most either file may hold. +/// +/// A system trust bundle is commonly several hundred KiB, far beyond the +/// per-file bound of provider files, so the CA files have their own. The +/// bound equals the control frame that carries both files to the runtime: no +/// start request that the protocol admits is refused here, and no file +/// larger than a frame can be published. +pub const MAX_TRUST_FILE_BYTES: usize = MAX_CONTROL_FRAME_BYTES; + +/// The mode the files report: public, read-only. +const FILE_MODE: Mode = Mode::from_bits_truncate(0o444); + +/// Refuse a pair that is not a complete, consistent supervisor trust set. +/// +/// Both files must be present and within bounds, and the bundle must contain +/// the certificate: the supervisor builds the bundle from the system CAs and +/// its own certificate, so a bundle without it would trust every upstream +/// except the interception endpoint the workload must reach. +pub fn validate_pair(ca_cert: &[u8], ca_bundle: &[u8]) -> io::Result<()> { + let invalid = |message: &str| Err(io::Error::new(io::ErrorKind::InvalidInput, message)); + if ca_cert.is_empty() || ca_bundle.is_empty() { + return invalid("supervisor CA certificate and bundle must not be empty"); + } + if ca_cert.len() > MAX_TRUST_FILE_BYTES || ca_bundle.len() > MAX_TRUST_FILE_BYTES { + return invalid("supervisor CA certificate or bundle exceeds the trust file bound"); + } + let certificate = ca_cert.trim_ascii(); + if certificate.is_empty() + || !ca_bundle + .windows(certificate.len()) + .any(|window| window == certificate) + { + return invalid("supervisor CA bundle does not contain the supervisor CA certificate"); + } + Ok(()) +} + +/// One published trust set: the bytes of both files and their backing. +struct TrustPair { + cert: Vec, + bundle: Vec, + cert_file: SealedFile, + bundle_file: SealedFile, +} + +/// The published supervisor trust files, replaced only as a pair. +#[derive(Default)] +pub struct CaTrustFiles { + current: RwLock>>, +} + +impl CaTrustFiles { + fn current(&self) -> Option> { + self.current + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .clone() + } + + /// Publish `ca_cert` and `ca_bundle` as the trust pair, or leave the + /// published pair untouched. + /// + /// The same bytes as the published pair publish nothing new, so a start + /// retried after a failed launch keeps the files earlier opens used. + pub(crate) fn publish(&self, ca_cert: &[u8], ca_bundle: &[u8]) -> io::Result<()> { + validate_pair(ca_cert, ca_bundle)?; + if self + .current() + .is_some_and(|pair| pair.cert == ca_cert && pair.bundle == ca_bundle) + { + return Ok(()); + } + let descriptors = OwnDescriptors::shared()?; + let prepared = Arc::new(TrustPair { + cert_file: SealedFile::create("openshell-ca", FILE_MODE, ca_cert, &descriptors)?, + bundle_file: SealedFile::create( + "openshell-ca-bundle", + FILE_MODE, + ca_bundle, + &descriptors, + )?, + cert: ca_cert.to_vec(), + bundle: ca_bundle.to_vec(), + }); + let replaced = self + .current + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .replace(prepared); + drop(replaced); + Ok(()) + } + + /// Forget the published pair, if any. Descriptors already injected keep + /// the files they opened; later opens of the canonical paths continue to + /// the kernel. + pub(crate) fn clear(&self) { + let replaced = self + .current + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take(); + drop(replaced); + } + + /// The file serving exactly `path`, if it is a canonical trust path and a + /// pair is published. Any other path, including an alias of these, is + /// not served here. + pub(crate) fn lookup(&self, path: &str) -> Option { + let pair = self.current()?; + match path { + CA_CERT_PATH => Some(TrustFile { + pair, + which: Which::Cert, + }), + CA_BUNDLE_PATH => Some(TrustFile { + pair, + which: Which::Bundle, + }), + _ => None, + } + } +} + +#[derive(Clone, Copy)] +enum Which { + Cert, + Bundle, +} + +/// One trust file of a published pair, held for the duration of an open so +/// a concurrent publication cannot close its backing. +pub struct TrustFile { + pair: Arc, + which: Which, +} + +impl TrustFile { + pub(crate) fn backing(&self) -> &SealedFile { + match self.which { + Which::Cert => &self.pair.cert_file, + Which::Bundle => &self.pair.bundle_file, + } + } +} + +#[cfg(test)] +mod tests { + use super::{ + CA_BUNDLE_PATH, CA_CERT_PATH, CaTrustFiles, MAX_TRUST_FILE_BYTES, canonical_paths, + validate_pair, + }; + use std::fs::File; + use std::io::{self, Read as _}; + use std::os::unix::fs::MetadataExt as _; + + const CERT: &[u8] = b"-----BEGIN CERTIFICATE-----\nsupervisor\n-----END CERTIFICATE-----\n"; + + fn bundle_with(system_bytes: usize) -> Vec { + let mut bundle = b"-----BEGIN CERTIFICATE-----\n".to_vec(); + bundle.extend(std::iter::repeat_n(b'A', system_bytes)); + bundle.extend_from_slice(b"\n-----END CERTIFICATE-----\n"); + bundle.extend_from_slice(CERT); + bundle + } + + fn open(files: &CaTrustFiles, path: &str) -> Option { + Some(File::from(files.lookup(path)?.backing().open().unwrap())) + } + + fn read(files: &CaTrustFiles, path: &str) -> Option<(Vec, (u64, u64))> { + let file = open(files, path)?; + let metadata = file.metadata().unwrap(); + let mut content = Vec::new(); + (&file).read_to_end(&mut content).unwrap(); + Some((content, (metadata.dev(), metadata.ino()))) + } + + #[test] + fn canonical_paths_lie_in_the_reserved_runtime_directory() { + let (cert, bundle) = canonical_paths(); + assert!(cert.starts_with(openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR)); + assert!(bundle.starts_with(openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR)); + } + + #[test] + fn a_pair_must_be_complete_consistent_and_bounded() { + assert!(validate_pair(CERT, &bundle_with(16)).is_ok()); + assert!(validate_pair(CERT, &bundle_with(300 * 1024)).is_ok()); + for (cert, bundle) in [ + ( + CERT, + b"-----BEGIN CERTIFICATE-----\nother\n-----END CERTIFICATE-----\n".as_slice(), + ), + (b"".as_slice(), CERT), + (CERT, b"".as_slice()), + (b" \n".as_slice(), CERT), + ] { + assert_eq!( + validate_pair(cert, bundle).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + } + let oversized = bundle_with(MAX_TRUST_FILE_BYTES); + assert_eq!( + validate_pair(CERT, &oversized).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + } + + #[test] + fn a_pair_is_published_whole_and_replaced_whole_or_not_at_all() { + let files = CaTrustFiles::default(); + assert!(files.lookup(CA_CERT_PATH).is_none()); + assert!(files.lookup(CA_BUNDLE_PATH).is_none()); + + let first_bundle = bundle_with(100 * 1024); + files.publish(CERT, &first_bundle).unwrap(); + let (cert, cert_identity) = read(&files, CA_CERT_PATH).unwrap(); + let (bundle, bundle_identity) = read(&files, CA_BUNDLE_PATH).unwrap(); + assert_eq!(cert, CERT); + assert_eq!(bundle, first_bundle); + assert_ne!(cert_identity, bundle_identity); + let mode = open(&files, CA_CERT_PATH) + .unwrap() + .metadata() + .unwrap() + .mode(); + assert_eq!(mode & 0o7777, 0o444); + + // The same bytes, or a refused pair, keep both files. + files.publish(CERT, &first_bundle).unwrap(); + let error = files.publish(CERT, b"unrelated").unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::InvalidInput); + assert_eq!(read(&files, CA_CERT_PATH).unwrap().1, cert_identity); + assert_eq!(read(&files, CA_BUNDLE_PATH).unwrap().1, bundle_identity); + + // A changed bundle replaces both files; a held descriptor keeps its own. + let held = open(&files, CA_BUNDLE_PATH).unwrap(); + let second_bundle = bundle_with(32); + files.publish(CERT, &second_bundle).unwrap(); + let (bundle, new_bundle_identity) = read(&files, CA_BUNDLE_PATH).unwrap(); + assert_eq!(bundle, second_bundle); + assert_ne!(new_bundle_identity, bundle_identity); + assert_ne!(read(&files, CA_CERT_PATH).unwrap().1, cert_identity); + let mut old = Vec::new(); + (&held).read_to_end(&mut old).unwrap(); + assert_eq!(old, first_bundle); + + // Clearing forgets the pair for later opens only. + let held_cert = open(&files, CA_CERT_PATH).unwrap(); + files.clear(); + assert!(files.lookup(CA_CERT_PATH).is_none()); + assert!(files.lookup(CA_BUNDLE_PATH).is_none()); + let mut after_clear = Vec::new(); + (&held_cert).read_to_end(&mut after_clear).unwrap(); + assert_eq!(after_clear, CERT); + files.clear(); + files.publish(CERT, &second_bundle).unwrap(); + assert_eq!(read(&files, CA_BUNDLE_PATH).unwrap().0, second_bundle); + + for alias in [ + "/run/openshell-supervisor-ca/material/../material/ca.crt", + "/run/openshell-supervisor-ca/material/ca.crt/", + "/run/openshell-supervisor-ca/ca.crt", + "/run/openshell-supervisor-ca/material", + "ca.crt", + ] { + assert!(files.lookup(alias).is_none(), "{alias}"); + } + } +} diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index 3c280cc977..b24466254a 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -17,6 +17,8 @@ pub mod boundary_exec; pub mod boundary_io; mod boundary_server; #[cfg(target_os = "linux")] +mod ca_trust_files; +#[cfg(target_os = "linux")] pub mod child_env; pub mod container_log; #[cfg(target_os = "linux")] diff --git a/crates/openshell-sandbox/src/linux/mod.rs b/crates/openshell-sandbox/src/linux/mod.rs index a15b8b6f43..acd8cb74f7 100644 --- a/crates/openshell-sandbox/src/linux/mod.rs +++ b/crates/openshell-sandbox/src/linux/mod.rs @@ -10,6 +10,7 @@ pub mod child_seccomp; pub mod landlock; pub mod proc_fd; pub mod process_signal; +pub(crate) mod sealed_file; pub mod seccomp_notify; pub mod socket_confinement; pub mod socket_registry; diff --git a/crates/openshell-sandbox/src/linux/sealed_file.rs b/crates/openshell-sandbox/src/linux/sealed_file.rs new file mode 100644 index 0000000000..8b1d50f545 --- /dev/null +++ b/crates/openshell-sandbox/src/linux/sealed_file.rs @@ -0,0 +1,400 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Bytes the workload receives as a read-only file. +//! +//! A [`SealedFile`] holds one immutable byte sequence in an anonymous +//! in-memory file. The bytes are written once, the file is sealed against +//! writes, growing, shrinking, and further sealing, and the only descriptor +//! kept is read-only. Each caller open is a new read-only open file +//! description of that file, positioned at its start, so no two holders share +//! a file offset. The reopen walks no path: it goes through this process's +//! own descriptor table in procfs, opened once and verified to be procfs. +//! +//! This module decides nothing about which workload paths such a file +//! serves; it is the backing that provider files and the supervisor CA trust +//! files share. Its metadata, as the kernel reports it for a descriptor of +//! the file, is also what the broker reports for the virtual path. + +#![allow(unsafe_code)] + +use std::fs::File; +use std::io::{self, Write as _}; +use std::mem::size_of; +use std::os::fd::{AsFd, AsRawFd as _, OwnedFd}; +use std::sync::{Arc, OnceLock}; + +use rustix::fs::{Mode, OFlags, SealFlags}; + +use crate::linux::seccomp_notify::NotificationListener; + +/// Every seal a sealed file carries. A file that cannot carry all of them is +/// not published. +const REQUIRED_SEALS: SealFlags = SealFlags::SEAL + .union(SealFlags::WRITE) + .union(SealFlags::GROW) + .union(SealFlags::SHRINK); + +/// This process's own descriptor table in procfs. +/// +/// Opening a descriptor's entry here yields a new open file description of +/// the same file. The directory is opened once and verified to be procfs, so +/// a later reopen walks no path. +pub struct OwnDescriptors(OwnedFd); + +impl OwnDescriptors { + pub(crate) fn open() -> io::Result { + let directory = rustix::fs::open( + "/proc/self/fd", + OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC, + Mode::empty(), + )?; + if rustix::fs::fstatfs(&directory)?.f_type != rustix::fs::PROC_SUPER_MAGIC { + return Err(io::Error::other("/proc/self/fd is not on procfs")); + } + Ok(Self(directory)) + } + + /// The descriptor table shared by every sealed file of this process, + /// opened by the first caller that needs it. + pub(crate) fn shared() -> io::Result> { + static SHARED: OnceLock> = OnceLock::new(); + if let Some(descriptors) = SHARED.get() { + return Ok(Arc::clone(descriptors)); + } + let opened = Arc::new(Self::open()?); + Ok(Arc::clone(SHARED.get_or_init(|| opened))) + } + + /// A new read-only open file description of the file `descriptor` refers + /// to, positioned at its start. Duplicating `descriptor` instead would + /// share one file offset between every holder. + fn reopen_read_only(&self, descriptor: impl AsFd) -> io::Result { + self.reopen(descriptor, OFlags::RDONLY | OFlags::NOCTTY) + } + + fn reopen(&self, descriptor: impl AsFd, flags: OFlags) -> io::Result { + Ok(rustix::fs::openat( + &self.0, + rustix::path::DecInt::new(descriptor.as_fd().as_raw_fd()), + flags | OFlags::CLOEXEC, + Mode::empty(), + )?) + } +} + +/// The size of the native `struct stat` the `stat` family fills: 144 bytes +/// on x86-64 and 128 bytes on `AArch64`, where the libc layout is the kernel's. +pub const STAT_SIZE: usize = size_of::(); +/// The size of `struct statx`, fixed by the UAPI. +pub const STATX_SIZE: usize = 256; + +const _: () = assert!( + (cfg!(target_arch = "x86_64") && STAT_SIZE == 144) + || (cfg!(target_arch = "aarch64") && STAT_SIZE == 128) +); + +/// A kernel-filled `struct stat`, kept as bytes. +/// +/// The kernel zero-fills the whole structure before copying it out, so the +/// bytes carry no padding this process left behind. +#[repr(C, align(8))] +#[derive(Clone, Copy)] +pub struct StatBytes(pub [u8; STAT_SIZE]); + +/// A kernel-filled `struct statx`, kept as bytes. +#[repr(C, align(8))] +#[derive(Clone, Copy)] +pub struct StatxBytes(pub [u8; STATX_SIZE]); + +/// One immutable byte sequence, delivered as a read-only file. +/// +/// The writable descriptor `memfd_create` returns is closed before the value +/// exists, so no code path holds a descriptor that could change the bytes +/// even without the seals. +pub struct SealedFile { + anchor: OwnedFd, + mode: Mode, + descriptors: Arc, +} + +impl SealedFile { + /// Write `content` once and seal it. `name` is the memfd's debugging + /// name, and `mode` is what the file reports to `fstat`; neither + /// protects the content, the seals and the read-only descriptors do. + pub(crate) fn create( + name: &str, + mode: Mode, + content: &[u8], + descriptors: &Arc, + ) -> io::Result { + let writable = rustix::fs::memfd_create( + name, + rustix::fs::MemfdFlags::CLOEXEC | rustix::fs::MemfdFlags::ALLOW_SEALING, + )?; + rustix::fs::fchmod(&writable, mode)?; + let mut writable = File::from(writable); + writable.write_all(content)?; + rustix::fs::fcntl_add_seals(&writable, REQUIRED_SEALS)?; + // Fail closed: the kernel reported the seals applied, and the file + // must agree before anything is served from it. + if !rustix::fs::fcntl_get_seals(&writable)?.contains(REQUIRED_SEALS) { + return Err(io::Error::other("in-memory file did not accept every seal")); + } + let anchor = descriptors.reopen_read_only(&writable)?; + drop(writable); + if rustix::fs::fcntl_getfl(&anchor)? & OFlags::ACCMODE != OFlags::RDONLY { + return Err(io::Error::other( + "retained in-memory file descriptor is not read-only", + )); + } + Ok(Self { + anchor, + mode, + descriptors: Arc::clone(descriptors), + }) + } + + /// A descriptor for one caller open: its own read-only open file + /// description, positioned at the start of the file. + pub(crate) fn open(&self) -> io::Result { + match self.descriptors.reopen_read_only(&self.anchor) { + // Every open shares one file, which the workload owns as well and + // may have made unreadable through a descriptor it holds. Its + // mode is not what protects the content, so restore it rather + // than fail opens until the file is replaced. + Err(error) if error.raw_os_error() == Some(libc::EACCES) => { + rustix::fs::fchmod(&self.anchor, self.mode)?; + self.descriptors.reopen_read_only(&self.anchor) + } + opened => opened, + } + } + + /// Open the file for the notifying workload task and complete its open + /// syscall with the injected descriptor, in one kernel operation. + pub(crate) fn inject( + &self, + listener: &NotificationListener, + notification_id: u64, + close_on_exec: bool, + ) -> io::Result<()> { + let file = self.open()?; + listener.add_fd_and_send(notification_id, file.as_raw_fd(), close_on_exec)?; + Ok(()) + } + + /// The native `struct stat` of the file, exactly what `fstat` reports on + /// an injected descriptor. + pub(crate) fn stat(&self) -> io::Result { + let mut bytes = StatBytes([0; STAT_SIZE]); + // SAFETY: the buffer is a live, 8-byte aligned region of exactly the + // native struct stat size, which the kernel fills whole. + let result = unsafe { + libc::syscall( + libc::SYS_fstat, + self.anchor.as_raw_fd(), + bytes.0.as_mut_ptr(), + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + Ok(bytes) + } + + /// The `struct statx` of the file for `mask`, with `sync_flags` from the + /// caller's `AT_STATX_SYNC_TYPE` bits: exactly what `statx` reports on an + /// injected descriptor. + pub(crate) fn statx(&self, sync_flags: u32, mask: u32) -> io::Result { + let mut bytes = StatxBytes([0; STATX_SIZE]); + // SAFETY: the empty path is NUL-terminated and the buffer is a live, + // 8-byte aligned 256-byte region, which the kernel fills whole. + let result = unsafe { + libc::syscall( + libc::SYS_statx, + self.anchor.as_raw_fd(), + c"".as_ptr(), + libc::AT_EMPTY_PATH | i32::try_from(sync_flags).map_err(io::Error::other)?, + mask, + bytes.0.as_mut_ptr(), + ) + }; + if result < 0 { + return Err(io::Error::last_os_error()); + } + Ok(bytes) + } + + /// The one descriptor retained for the file. + #[cfg(test)] + pub(crate) fn anchor(&self) -> std::os::fd::BorrowedFd<'_> { + self.anchor.as_fd() + } +} + +#[cfg(test)] +mod tests { + use super::{OwnDescriptors, REQUIRED_SEALS, SealedFile}; + use std::fs::File; + use std::io::{self, Read as _, Seek as _, SeekFrom}; + use std::os::fd::AsRawFd as _; + use std::os::unix::fs::{FileExt as _, MetadataExt as _, PermissionsExt as _}; + + const MODE: rustix::fs::Mode = rustix::fs::Mode::from_bits_truncate(0o600); + + fn sealed(content: &[u8]) -> SealedFile { + SealedFile::create("test", MODE, content, &OwnDescriptors::shared().unwrap()).unwrap() + } + + fn identity(file: &File) -> (u64, u64) { + let metadata = file.metadata().unwrap(); + (metadata.dev(), metadata.ino()) + } + + #[test] + fn binary_and_empty_content_round_trip_exactly() { + let binary = (0..=255_u8).cycle().take(70_000).collect::>(); + for content in [binary.as_slice(), b"", b"\0", b"no newline"] { + let mut read = Vec::new(); + File::from(sealed(content).open().unwrap()) + .read_to_end(&mut read) + .unwrap(); + assert_eq!(read, content); + } + } + + #[test] + fn every_open_is_an_independent_read_only_description_of_one_sealed_file() { + let file = sealed(b"0123456789"); + let mut first = File::from(file.open().unwrap()); + let mut second = File::from(file.open().unwrap()); + + let mut head = [0_u8; 4]; + first.read_exact(&mut head).unwrap(); + assert_eq!(&head, b"0123"); + let mut whole = Vec::new(); + second.read_to_end(&mut whole).unwrap(); + assert_eq!(whole, b"0123456789"); + let mut rest = Vec::new(); + first.read_to_end(&mut rest).unwrap(); + assert_eq!(rest, b"456789"); + second.seek(SeekFrom::Start(8)).unwrap(); + assert_eq!(first.stream_position().unwrap(), 10); + + assert_eq!(identity(&first), identity(&second)); + for opened in [&first, &second] { + assert_eq!(opened.metadata().unwrap().mode() & 0o7777, 0o600); + assert_eq!( + rustix::fs::fcntl_getfl(opened).unwrap() & rustix::fs::OFlags::ACCMODE, + rustix::fs::OFlags::RDONLY + ); + assert!( + rustix::fs::fcntl_get_seals(opened) + .unwrap() + .contains(REQUIRED_SEALS) + ); + assert!(opened.write_at(b"changed", 0).is_err()); + assert!(opened.set_len(0).is_err()); + } + assert_eq!( + rustix::fs::fcntl_getfl(file.anchor()).unwrap() & rustix::fs::OFlags::ACCMODE, + rustix::fs::OFlags::RDONLY + ); + } + + #[test] + fn metadata_describes_the_same_file_an_open_descriptor_does() { + use std::mem::offset_of; + let file = sealed(b"0123456789"); + let opened = File::from(file.open().unwrap()); + let expected = opened.metadata().unwrap(); + + let stat = file.stat().unwrap(); + let field = + |offset: usize| u64::from_ne_bytes(stat.0[offset..offset + 8].try_into().unwrap()); + assert_eq!(field(offset_of!(libc::stat, st_ino)), expected.ino()); + assert_eq!(field(offset_of!(libc::stat, st_dev)), expected.dev()); + let size_offset = offset_of!(libc::stat, st_size); + assert_eq!( + i64::from_ne_bytes(stat.0[size_offset..size_offset + 8].try_into().unwrap()), + 10 + ); + let mode_offset = offset_of!(libc::stat, st_mode); + let mode = u32::from_ne_bytes(stat.0[mode_offset..mode_offset + 4].try_into().unwrap()); + assert_eq!(mode, expected.mode()); + assert_eq!(mode & libc::S_IFMT, libc::S_IFREG); + + // statx: mask, mode, inode, and size at their UAPI offsets. + let statx = file.statx(0, 0x0000_07ff).unwrap(); + let word = + |offset: usize| u32::from_ne_bytes(statx.0[offset..offset + 4].try_into().unwrap()); + assert_ne!(word(0) & 0x0000_0200, 0, "STATX_SIZE is reported"); + assert_eq!( + u16::from_ne_bytes(statx.0[28..30].try_into().unwrap()), + u16::try_from(expected.mode()).unwrap() + ); + assert_eq!( + u64::from_ne_bytes(statx.0[32..40].try_into().unwrap()), + expected.ino() + ); + assert_eq!(u64::from_ne_bytes(statx.0[40..48].try_into().unwrap()), 10); + } + + #[test] + fn seals_hold_against_a_writable_reopen_by_the_owner() { + let file = sealed(b"sealed"); + let held = File::from(file.open().unwrap()); + let link = format!("/proc/self/fd/{}", held.as_raw_fd()); + for options in [ + std::fs::OpenOptions::new().write(true).clone(), + std::fs::OpenOptions::new().read(true).write(true).clone(), + std::fs::OpenOptions::new().append(true).clone(), + ] { + if let Ok(writable) = options.open(&link) { + assert!(writable.write_at(b"changed", 0).is_err()); + assert!(io::Write::write_all(&mut &writable, b"changed").is_err()); + assert!(writable.set_len(0).is_err()); + assert!(writable.set_len(1 << 20).is_err()); + assert!( + rustix::fs::fcntl_add_seals(&writable, rustix::fs::SealFlags::empty()).is_err() + ); + } + } + assert!( + std::fs::OpenOptions::new() + .write(true) + .truncate(true) + .open(&link) + .is_err() + ); + let mut content = Vec::new(); + File::from(file.open().unwrap()) + .read_to_end(&mut content) + .unwrap(); + assert_eq!(content, b"sealed"); + } + + #[test] + fn a_cleared_mode_does_not_deny_the_next_open() { + let file = sealed(b"key"); + let held = File::from(file.open().unwrap()); + held.set_permissions(std::fs::Permissions::from_mode(0o000)) + .unwrap(); + // Whether the cleared mode denies the reopen depends on the caller: + // an identity with CAP_DAC_OVERRIDE is never denied, so there is + // nothing to restore and the mode may stay cleared. + let denied = match file.descriptors.reopen_read_only(file.anchor()) { + Ok(_) => false, + Err(error) if error.raw_os_error() == Some(libc::EACCES) => true, + Err(error) => panic!("unexpected reopen failure: {error}"), + }; + let reopened = File::from(file.open().unwrap()); + if denied { + assert_eq!(reopened.metadata().unwrap().mode() & 0o7777, 0o600); + } + let mut content = Vec::new(); + (&reopened).read_to_end(&mut content).unwrap(); + assert_eq!(content, b"key"); + } +} diff --git a/crates/openshell-sandbox/src/linux/seccomp_notify.rs b/crates/openshell-sandbox/src/linux/seccomp_notify.rs index cf513f0ee2..3830231618 100644 --- a/crates/openshell-sandbox/src/linux/seccomp_notify.rs +++ b/crates/openshell-sandbox/src/linux/seccomp_notify.rs @@ -20,6 +20,10 @@ use std::time::Duration; const SECCOMP_SET_MODE_FILTER: libc::c_uint = 1; const SECCOMP_GET_NOTIF_SIZES: libc::c_uint = 3; const SECCOMP_FILTER_FLAG_NEW_LISTENER: libc::c_ulong = 1 << 3; +/// Linux 5.19: once the listener has received a notification, the notifying +/// task waits for the response in a killable sleep instead of an +/// interruptible one. +const SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV: libc::c_ulong = 1 << 5; const SECCOMP_RET_KILL_PROCESS: u32 = 0x8000_0000; const SECCOMP_RET_USER_NOTIF: u32 = 0x7fc0_0000; @@ -196,11 +200,14 @@ impl NotificationProbeReport { /// Owned listener returned by `SECCOMP_FILTER_FLAG_NEW_LISTENER`. /// -/// The listener is installed without `WAIT_KILLABLE_RECV`, so mediation is the -/// same on every kernel. A signal can interrupt a notified syscall, which the -/// kernel then restarts or fails with `EINTR`; broker handlers check the -/// notification is still live before acting and answer a repeated operation -/// as the kernel would. +/// Two kinds of listener exist. [`install_listener`] installs one without +/// `WAIT_KILLABLE_RECV`: a signal can interrupt a notified syscall at any +/// time, which the kernel then restarts or fails with `EINTR`, so broker +/// handlers check the notification is still live before acting and answer a +/// repeated operation as the kernel would. [`install_workload_listener`] +/// installs the production listener with `WAIT_KILLABLE_RECV`, because it +/// writes syscall results into the notifying task's memory; see that +/// function for what the flag changes. pub struct NotificationListener { fd: OwnedFd, } @@ -383,11 +390,22 @@ pub fn install_listener(syscalls: &[i64]) -> io::Result { .into_iter() .map(FilterRule::Notify) .collect::>(); - install_rules(&rules) + install_rules(&rules, ListenerWait::Interruptible) } -/// Install the capability-free workload networking listener on the calling -/// launcher thread. +/// Install a listener for exactly `syscalls` with `WAIT_KILLABLE_RECV`, the +/// production wait mode, so tests can observe that mode in isolation. +#[cfg(test)] +pub(crate) fn install_wait_killable_listener(syscalls: &[i64]) -> io::Result { + let rules = syscalls + .iter() + .map(|syscall| FilterRule::Notify(*syscall)) + .collect::>(); + install_rules(&rules, ListenerWait::KillableOnceReceived) +} + +/// Install the capability-free workload listener on the calling launcher +/// thread. /// /// The filter mediates every syscall that can create, select, or materially /// reconfigure an INET endpoint. Connected `send()`/null-destination @@ -395,18 +413,80 @@ pub fn install_listener(syscalls: &[i64]) -> io::Result { /// natively: the broker binds every INET socket to loopback before injecting /// it, and accepted sockets inherit their listener's binding. `setsockopt` /// allows the [`SETSOCKOPT_FAST_PATH`] pairs in the filter. Every `open`, -/// `openat`, and `openat2` notifies: the broker serves provider files by -/// descriptor injection. +/// `openat`, and `openat2` notifies, and so does every path-based metadata +/// call in [`metadata_syscalls`]: the broker serves provider and CA trust +/// files by descriptor injection and answers their metadata by writing the +/// result into the caller's buffer. +/// +/// That write is why the listener is installed with `WAIT_KILLABLE_RECV`: a +/// task can have only one listener (a second fails with `EBUSY`), so the +/// flag covers every notified syscall. Before the broker receives a +/// notification, signals interrupt the syscall as before. Once received, +/// only a fatal signal ends the wait; a handled signal is delivered after the +/// broker responds. A `connect` awaiting the supervisor's network admission +/// decision therefore ignores handled `SIGALRM` and `SIGINT` until the broker +/// answers or its 30-second decision timeout fails the call; DNS relay waits +/// are bounded at 10 seconds. A blocking loopback `connect` is completed by +/// the broker with a native `libc::connect` on a duplicate of the socket and +/// waits for the kernel's connect timeout, which can exceed 30 seconds. +/// Fatal signals still terminate the task throughout. In return, a received +/// notification ends only by response or death, so a written result can never +/// land in a stack frame the task has left. pub fn install_workload_listener() -> io::Result { - install_rules(&workload_filter_rules()) + install_rules(&workload_filter_rules(), ListenerWait::KillableOnceReceived) +} + +/// How a notifying task waits for the broker's response. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ListenerWait { + /// Any signal interrupts the wait at any time. + Interruptible, + /// `WAIT_KILLABLE_RECV`: only a fatal signal interrupts the wait once the + /// broker has received the notification. + KillableOnceReceived, } /// Qualify the notification ABI, set `no_new_privs` on the calling thread, /// and install `rules` there. -fn install_rules(rules: &[FilterRule]) -> io::Result { +fn install_rules(rules: &[FilterRule], wait: ListenerWait) -> io::Result { verify_notification_sizes()?; set_no_new_privileges()?; - install_notification_filter(rules) + install_notification_filter(rules, wait) +} + +/// The path-based metadata syscalls the workload listener notifies. +/// +/// `newfstatat`, `statx`, `faccessat`, and `faccessat2`, plus the legacy +/// `stat`, `lstat`, and `access` entry points on x86-64. Descriptor based +/// calls (`fstat`, `statx` with an empty path) are not among them: an +/// injected descriptor answers them in the kernel. +pub fn metadata_syscalls() -> Vec { + #[allow(unused_mut)] // The legacy entry points exist only on x86-64. + let mut syscalls = vec![ + libc::SYS_newfstatat, + libc::SYS_statx, + libc::SYS_faccessat, + libc::SYS_faccessat2, + ]; + #[cfg(target_arch = "x86_64")] + syscalls.extend([libc::SYS_stat, libc::SYS_lstat, libc::SYS_access]); + syscalls +} + +/// Whether `syscall` is one of [`metadata_syscalls`]. +#[must_use] +pub fn is_metadata_syscall(syscall: i64) -> bool { + if matches!( + syscall, + libc::SYS_newfstatat | libc::SYS_statx | libc::SYS_faccessat | libc::SYS_faccessat2 + ) { + return true; + } + #[cfg(target_arch = "x86_64")] + if matches!(syscall, libc::SYS_stat | libc::SYS_lstat | libc::SYS_access) { + return true; + } + false } fn workload_filter_rules() -> Vec { @@ -430,6 +510,7 @@ fn workload_filter_rules() -> Vec { ]; #[cfg(target_arch = "x86_64")] syscalls.push(libc::SYS_open); + syscalls.extend(metadata_syscalls()); syscalls .into_iter() .map(|syscall| { @@ -741,7 +822,7 @@ fn probe_scalar_setsockopt_fast_path() -> io::Result<()> { let (sender, receiver) = mpsc::sync_channel(1); let launcher = thread::spawn(move || -> io::Result<()> { - let listener = install_rules(&[FilterRule::ScalarSetsockopt])?; + let listener = install_rules(&[FilterRule::ScalarSetsockopt], ListenerWait::Interruptible)?; sender .send(listener) .map_err(|_| io::Error::other("setsockopt broker disappeared"))?; @@ -863,7 +944,10 @@ fn receive_probe_notification(listener: &NotificationListener) -> io::Result io::Result { +fn install_notification_filter( + rules: &[FilterRule], + wait: ListenerWait, +) -> io::Result { let mut program = build_filter(rules)?; let length = u16::try_from(program.len()) .map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "seccomp filter is too large"))?; @@ -871,7 +955,12 @@ fn install_notification_filter(rules: &[FilterRule]) -> io::Result SECCOMP_FILTER_FLAG_NEW_LISTENER, + ListenerWait::KillableOnceReceived => { + SECCOMP_FILTER_FLAG_NEW_LISTENER | SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV + } + }; // SAFETY: `fprog` points to a live classic-BPF program for the duration of // the syscall. The returned nonnegative value is a newly owned FD. let result = unsafe { @@ -883,7 +972,17 @@ fn install_notification_filter(rules: &[FilterRule]) -> io::Result #[cfg(test)] mod tests { + use std::ffi::CStr; use std::process::Command; use super::*; @@ -1364,6 +1464,166 @@ mod tests { ); } + #[test] + fn workload_filter_notifies_every_metadata_syscall() { + let program = build_filter(&workload_filter_rules()).unwrap(); + for syscall in metadata_syscalls() { + assert!(is_metadata_syscall(syscall)); + assert_eq!( + evaluate(&program, syscall, [0x1000, 0x2000, 0, 0, 0, 0]), + SECCOMP_RET_USER_NOTIF, + "{syscall}" + ); + } + // Descriptor-based metadata stays in the kernel. + assert_eq!( + evaluate(&program, libc::SYS_fstat, [3, 0x1000, 0, 0, 0, 0]), + SECCOMP_RET_ALLOW + ); + assert!(!is_metadata_syscall(libc::SYS_fstat)); + } + + const WAIT_KILLABLE_CHILD: &str = "OPENSHELL_WAIT_KILLABLE_CHILD"; + const WAIT_KILLABLE_MARKER: &CStr = c"/openshell-wait-killable-marker"; + const CHILD_INTERRUPTED: i32 = 7; + static HANDLED_SIGNALS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + + extern "C" fn count_handled_signal(_: libc::c_int) { + HANDLED_SIGNALS.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } + + /// Child of [`wait_killable_listener_defers_handled_signals_until_the_response`]: + /// a notified `faccessat` of the marker path with a `SIGUSR1` handler + /// installed, then a second one that is only ever ended by `SIGKILL`. + fn wait_killable_child() -> ! { + let marker_access = || { + // SAFETY: the marker path is a live NUL-terminated string. + let result = unsafe { + libc::syscall( + libc::SYS_faccessat, + libc::AT_FDCWD, + WAIT_KILLABLE_MARKER.as_ptr(), + libc::F_OK, + 0, + ) + }; + (result, io::Error::last_os_error().raw_os_error()) + }; + // SAFETY: a plain handler without SA_RESTART, so an interruptible + // wait reports EINTR instead of restarting. + let installed = unsafe { + let mut action: libc::sigaction = std::mem::zeroed(); + action.sa_sigaction = count_handled_signal as *const () as usize; + libc::sigemptyset(std::ptr::addr_of_mut!(action.sa_mask)); + libc::sigaction( + libc::SIGUSR1, + std::ptr::addr_of!(action), + std::ptr::null_mut(), + ) + }; + if installed != 0 { + std::process::exit(2); + } + match marker_access() { + (-1, Some(libc::EINTR)) => std::process::exit(CHILD_INTERRUPTED), + (-1, Some(libc::EACCES)) + if HANDLED_SIGNALS.load(std::sync::atomic::Ordering::SeqCst) == 1 => {} + _ => std::process::exit(3), + } + let _ = marker_access(); + std::process::exit(4); + } + + /// Run the child under a `faccessat` listener of the given kind and + /// report `(notification still valid after SIGUSR1, child exit status)`. + fn observe_signal_during_notification(wait_killable: bool) -> (bool, std::process::ExitStatus) { + let executable = std::env::current_exe().expect("test executable"); + let (sender, receiver) = mpsc::sync_channel(1); + thread::spawn(move || { + let listener = if wait_killable { + install_wait_killable_listener(&[libc::SYS_faccessat]) + } else { + install_listener(&[libc::SYS_faccessat]) + } + .expect("install listener"); + let child = Command::new(executable) + .arg("--exact") + .arg("linux::seccomp_notify::tests::wait_killable_listener_defers_handled_signals_until_the_response") + .arg("--nocapture") + .env(WAIT_KILLABLE_CHILD, "1") + .spawn() + .expect("spawn child"); + sender.send((listener, child)).expect("hand over"); + }); + let (listener, mut child) = receiver.recv().expect("listener and child"); + let marker_notification = || { + loop { + let notification = receive_probe_notification(&listener).expect("notification"); + let mut path = [0_u8; 32]; + let marker = WAIT_KILLABLE_MARKER.to_bytes_with_nul(); + if crate::linux::task_memory::read_exact( + notification.tid, + notification.args[1], + &mut path[..marker.len()], + ) + .is_ok() + && &path[..marker.len()] == marker + { + return notification; + } + listener + .respond_continue(notification.id) + .expect("continue startup access"); + } + }; + + let first = marker_notification(); + // SAFETY: a thread-directed signal to the notifying task. + assert_eq!( + unsafe { libc::syscall(libc::SYS_tgkill, child.id(), first.tid, libc::SIGUSR1) }, + 0 + ); + thread::sleep(Duration::from_millis(300)); + let still_valid = listener.validate_id(first.id).is_ok(); + if still_valid { + listener + .respond_errno(first.id, libc::EACCES) + .expect("answer the first access"); + let second = marker_notification(); + child.kill().expect("kill child"); + let status = child.wait().expect("wait child"); + assert_eq!( + listener.validate_id(second.id).unwrap_err().raw_os_error(), + Some(libc::ENOENT), + "a killed task ends its notification" + ); + // A result for a dead notification is neither written nor an + // error: there is no task left to answer. + crate::provider_files::complete_with_result(&listener, &second, 0x1000, &[0; 16]) + .expect("dead notification completes quietly"); + return (true, status); + } + (false, child.wait().expect("wait child")) + } + + #[test] + fn wait_killable_listener_defers_handled_signals_until_the_response() { + use std::os::unix::process::ExitStatusExt as _; + if std::env::var_os(WAIT_KILLABLE_CHILD).is_some() { + wait_killable_child(); + } + // Production listener: the handled signal waits for the response, + // and only SIGKILL ends the second notification. + let (still_valid, status) = observe_signal_during_notification(true); + assert!(still_valid); + assert_eq!(status.signal(), Some(libc::SIGKILL), "{status}"); + // Interruptible listener: the same signal ends the notification and + // the syscall fails with EINTR. + let (still_valid, status) = observe_signal_during_notification(false); + assert!(!still_valid); + assert_eq!(status.code(), Some(CHILD_INTERRUPTED), "{status}"); + } + #[test] fn filter_rejects_a_syscall_named_twice() { let error = build_filter(&[ diff --git a/crates/openshell-sandbox/src/linux/task_memory.rs b/crates/openshell-sandbox/src/linux/task_memory.rs index 63c8f0fc9e..4c9fdddf04 100644 --- a/crates/openshell-sandbox/src/linux/task_memory.rs +++ b/crates/openshell-sandbox/src/linux/task_memory.rs @@ -6,6 +6,12 @@ //! Seccomp user-notification arguments contain addresses in the notifying //! task. Callers must copy pointer-bearing inputs once into trusted memory and //! must never treat a partial copy as valid. +//! +//! Reads go through `process_vm_readv`, or `/proc//mem` where a runtime +//! profile denies that syscall. Writes go through a [`PinnedMemory`] only: +//! an open `/proc//mem` is bound to one address space for its lifetime, +//! so a write can never reach a task that reused the number, which +//! `process_vm_writev` resolves anew on every call. #![allow(unsafe_code)] @@ -70,6 +76,62 @@ pub fn read_exact(tid: u32, address: u64, destination: &mut [u8]) -> io::Result< } } +/// One task's address space, pinned for writes. +/// +/// `/proc//mem` binds to the address space of the task that holds `tid` +/// when it is opened and never follows the number afterwards. The caller +/// proves which task that was: open the file, then validate the notification +/// it serves. A notification stays valid only while its task lives, so a +/// valid one after the open means the task never died, its number was never +/// freed or reused, and the pinned space is its own. A sibling thread's +/// `execve` kills every other thread before replacing the address space, +/// which also ends the notification. `process_vm_writev` offers no such +/// proof: it resolves the number again on every call. +/// +/// Writes here use the kernel's `FOLL_FORCE`, which differs from the +/// `copy_to_user` behind a native `stat` in one case: a private copy-on-write +/// mapping the task holds read-only is written (a private copy is made) +/// instead of failing with `EFAULT`. The kernel refuses forced writes to +/// shared and non-`VM_MAYWRITE` mappings, so this reaches only memory the +/// task could already make writable with `mprotect`; the deviation is +/// confined to a task that passes its own read-only private memory as a +/// result buffer. Unmapped addresses still fail with `EFAULT` or a short +/// count, and a space already torn down accepts nothing. +#[derive(Debug)] +pub struct PinnedMemory { + file: std::fs::File, + tid: u32, +} + +impl PinnedMemory { + /// Pin the address space of the task currently holding `tid`, for + /// writing. Fails with `ESRCH` when no such task exists and with + /// `EACCES` or `EPERM` when the active Yama, LSM, or seccomp posture + /// refuses the attach-mode access the write needs. + pub fn open(tid: u32) -> io::Result { + if tid == 0 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "task-memory TID must be nonzero", + )); + } + let file = std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(format!("/proc/{tid}/mem"))?; + Ok(Self { file, tid }) + } + + /// Write exactly `source` at `address`. Empty and oversized requests and + /// null addresses fail closed; a short write is + /// [`io::ErrorKind::InvalidData`]. + pub fn write_exact(&self, address: u64, source: &[u8]) -> io::Result<()> { + validate_request(self.tid, address, source.len())?; + let written = self.file.write_at(source, address)?; + require_exact(written, source.len(), "task-memory write") + } +} + #[cfg(feature = "perf-harness")] static PROCESS_VM_READS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); #[cfg(feature = "perf-harness")] @@ -122,14 +184,20 @@ fn read_exact_from_proc_mem(tid: u32, address: u64, destination: &mut [u8]) -> i require_exact(copied, destination.len(), "proc task-memory read") } -/// Prove same-UID parent-to-child read access under the active Yama, LSM, and -/// outer seccomp posture. The broker only reads workload memory; it never -/// writes it. +/// Prove same-UID parent-to-child read and write access under the active +/// Yama, LSM, and outer seccomp posture. +/// +/// The parent reads a value the child stored, writes another through a +/// pinned `/proc//mem`, and the child confirms it observed it. /// /// Call this only from a single-threaded probe process. The child executes /// raw, allocation-free syscalls between `fork` and `_exit`. pub fn probe_child_access() -> io::Result<()> { const INITIAL: u64 = 0x1122_3344_5566_7788; + const WRITTEN: u64 = 0x8877_6655_4433_2211; + const CHILD_SAW_WRITE: i32 = 0; + const CHILD_SETUP_FAILED: i32 = 1; + const CHILD_MISSED_WRITE: i32 = 2; // SAFETY: mmap creates one private anonymous page owned by this process. let mapping = unsafe { libc::mmap( @@ -183,35 +251,54 @@ pub fn probe_child_access() -> io::Result<()> { || write_eventfd(&ready).is_err() || read_eventfd(&proceed).is_err() { - libc::_exit(1); + libc::_exit(CHILD_SETUP_FAILED); + } + // A volatile read: the parent changed this through another + // address space, which the compiler cannot see. + if mapping.cast::().read_volatile() == WRITTEN { + libc::_exit(CHILD_SAW_WRITE); } - libc::_exit(0); + libc::_exit(CHILD_MISSED_WRITE); } } let outcome = (|| { read_eventfd(&ready)?; + let child_tid = + u32::try_from(child).map_err(|_| io::Error::other("child PID does not fit u32"))?; let mut observed = [0_u8; size_of::()]; - read_exact( - u32::try_from(child).map_err(|_| io::Error::other("child PID does not fit u32"))?, - mapping_address, - &mut observed, - )?; + read_exact(child_tid, mapping_address, &mut observed)?; if u64::from_ne_bytes(observed) != INITIAL { return Err(io::Error::other( "cross-child memory read returned wrong data", )); } + PinnedMemory::open(child_tid) + .and_then(|memory| memory.write_exact(mapping_address, &WRITTEN.to_ne_bytes())) + .map_err(|error| { + io::Error::new( + error.kind(), + format!( + "cross-child memory write failed: {error}. The sandbox answers stat and access calls on its virtual provider and CA trust files by writing the result into the workload through /proc//mem; the runtime's Yama, LSM, or mount posture must permit a same-UID ancestor to open a dumpable child's memory file for writing" + ), + ) + })?; write_eventfd(&proceed)?; let mut status = 0; // SAFETY: child is a live direct child and status points to storage. if unsafe { libc::waitpid(child, std::ptr::addr_of_mut!(status), 0) } != child { return Err(io::Error::last_os_error()); } - if !libc::WIFEXITED(status) || libc::WEXITSTATUS(status) != 0 { - return Err(io::Error::other("cross-child memory probe failed in child")); + if !libc::WIFEXITED(status) { + return Err(io::Error::other("cross-child memory probe child died")); + } + match libc::WEXITSTATUS(status) { + CHILD_SAW_WRITE => Ok(()), + CHILD_MISSED_WRITE => Err(io::Error::other( + "cross-child memory write reported success but the child did not observe the written value", + )), + _ => Err(io::Error::other("cross-child memory probe failed in child")), } - Ok(()) })(); if outcome.is_err() { @@ -297,6 +384,8 @@ fn require_exact(copied: usize, expected: usize, operation: &str) -> io::Result< #[cfg(test)] mod tests { use super::*; + use std::os::fd::AsRawFd as _; + use std::time::Duration; #[test] fn reads_exact_same_process_memory() { @@ -312,6 +401,180 @@ mod tests { assert_eq!(u64::from_ne_bytes(bytes), source); } + #[test] + fn pinned_memory_writes_exactly_and_refuses_what_is_not_mapped() { + let memory = PinnedMemory::open(std::process::id()).expect("pin own memory"); + let mut target = 0_u64; + let value = 0x0f0e_0d0c_0b0a_0908_u64; + memory + .write_exact(std::ptr::addr_of_mut!(target) as u64, &value.to_ne_bytes()) + .expect("write target"); + // SAFETY: target is a live, aligned u64 on this stack. + assert_eq!(unsafe { std::ptr::addr_of!(target).read_volatile() }, value); + + // SAFETY: a fresh private page, unmapped again below. + let page = unsafe { + libc::mmap( + std::ptr::null_mut(), + 4096, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) + }; + assert_ne!(page, libc::MAP_FAILED); + // SAFETY: the page is mapped and nothing else refers to it. + assert_eq!(unsafe { libc::munmap(page, 4096) }, 0); + let error = memory + .write_exact(page as u64, &[1, 2, 3, 4]) + .expect_err("unmapped address must fail"); + assert!( + error.raw_os_error() == Some(libc::EFAULT) + || error.raw_os_error() == Some(libc::EIO) + || error.kind() == io::ErrorKind::InvalidData, + "{error}" + ); + assert_eq!( + memory.write_exact(0, &[1]).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + memory.write_exact(8, &[]).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + assert_eq!( + PinnedMemory::open(0).unwrap_err().kind(), + io::ErrorKind::InvalidInput + ); + } + + /// Fork a child that stores `INITIAL` in a fresh page, reports ready on + /// a pipe, waits on another, then runs `after`. + fn pinned_child(after: unsafe fn()) -> (libc::pid_t, u64, std::fs::File, std::fs::File) { + let (ready_read, ready_write) = nix::unistd::pipe().unwrap(); + let (go_read, go_write) = nix::unistd::pipe().unwrap(); + // SAFETY: a fresh private page shared by inheritance only. + let page = unsafe { + libc::mmap( + std::ptr::null_mut(), + 4096, + libc::PROT_READ | libc::PROT_WRITE, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) + }; + assert_ne!(page, libc::MAP_FAILED); + // SAFETY: the child runs only raw syscalls before exit or exec. + let child = unsafe { libc::fork() }; + assert!(child >= 0); + if child == 0 { + // SAFETY: inherited descriptors and page; nothing else runs here. + unsafe { + libc::prctl(libc::PR_SET_DUMPABLE, 1, 0, 0, 0); + page.cast::().write_volatile(0x1122_3344_5566_7788); + libc::write(ready_write.as_raw_fd(), [1_u8].as_ptr().cast(), 1); + let mut byte = 0_u8; + libc::read(go_read.as_raw_fd(), std::ptr::addr_of_mut!(byte).cast(), 1); + after(); + libc::_exit(0); + } + } + let mut ready = std::fs::File::from(ready_read); + io::Read::read_exact(&mut ready, &mut [0_u8; 1]).unwrap(); + (child, page as u64, std::fs::File::from(go_write), ready) + } + + unsafe fn exit_child() { + // SAFETY: ends the forked child only. + unsafe { libc::_exit(0) } + } + + unsafe fn exec_child() { + // SAFETY: replaces the forked child's image with a short sleep. + unsafe { + libc::execl( + c"/bin/sleep".as_ptr(), + c"sleep".as_ptr(), + c"5".as_ptr(), + std::ptr::null::(), + ); + libc::_exit(9); + } + } + + #[test] + fn pinned_memory_never_follows_the_task_number_after_exit_or_exec() { + use std::os::fd::AsRawFd as _; + // Exit: the pinned space goes away with the task; a write reaches + // nothing, even though a new task may hold the number by now. + let (child, page, mut go, _ready) = pinned_child(exit_child); + let memory = PinnedMemory::open(u32::try_from(child).unwrap()).unwrap(); + io::Write::write_all(&mut go, &[1]).unwrap(); + let mut status = 0; + // SAFETY: child is this test's direct child. + assert_eq!( + unsafe { libc::waitpid(child, std::ptr::addr_of_mut!(status), 0) }, + child + ); + assert!(memory.write_exact(page, &[7; 8]).is_err()); + + // Exec: the pinned space is the old image's, not the new one's. + let (child, page, mut go, _ready) = pinned_child(exec_child); + let memory = PinnedMemory::open(u32::try_from(child).unwrap()).unwrap(); + io::Write::write_all(&mut go, &[1]).unwrap(); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while !matches!( + std::fs::read_link(format!("/proc/{child}/exe")), + Ok(exe) if exe.ends_with("sleep") + ) { + assert!(std::time::Instant::now() < deadline, "child did not exec"); + std::thread::sleep(Duration::from_millis(10)); + } + let outcome = memory.write_exact(page, &[7; 8]); + // SAFETY: child is this test's direct child. + unsafe { + libc::kill(child, libc::SIGKILL); + libc::waitpid(child, std::ptr::addr_of_mut!(status), 0); + } + assert!(outcome.is_err(), "write reached the replacement image"); + let _ = go.as_raw_fd(); + } + + #[test] + fn pinned_write_into_read_only_private_memory_is_the_documented_deviation() { + let memory = PinnedMemory::open(std::process::id()).unwrap(); + // SAFETY: a fresh private read-only page, unmapped below. + let page = unsafe { + libc::mmap( + std::ptr::null_mut(), + 4096, + libc::PROT_READ, + libc::MAP_PRIVATE | libc::MAP_ANONYMOUS, + -1, + 0, + ) + }; + assert_ne!(page, libc::MAP_FAILED); + match memory.write_exact(page as u64, &[5; 8]) { + // FOLL_FORCE: a private copy is written where copy_to_user + // would fail. Only memory this process could mprotect writable. + // SAFETY: the page is mapped readable. + Ok(()) => assert_eq!( + unsafe { page.cast::().read_volatile() }, + 0x0505_0505_0505_0505 + ), + // A kernel with proc_mem.force_override=never behaves natively. + Err(error) => assert!( + matches!(error.raw_os_error(), Some(libc::EFAULT | libc::EIO)), + "{error}" + ), + } + // SAFETY: the page is mapped and nothing else refers to it. + assert_eq!(unsafe { libc::munmap(page, 4096) }, 0); + } + #[test] fn proc_mem_fallback_reads_exact_memory() { let source = 0x0102_0304_0506_0708_u64; diff --git a/crates/openshell-sandbox/src/linux/workload_launcher.rs b/crates/openshell-sandbox/src/linux/workload_launcher.rs index 51b3af629e..77d06a566d 100644 --- a/crates/openshell-sandbox/src/linux/workload_launcher.rs +++ b/crates/openshell-sandbox/src/linux/workload_launcher.rs @@ -143,10 +143,13 @@ mod tests { if syscall == libc::SYS_socket { break notification; } - if syscall == libc::SYS_openat || syscall == libc::SYS_openat2 { + if syscall == libc::SYS_openat + || syscall == libc::SYS_openat2 + || crate::linux::seccomp_notify::is_metadata_syscall(syscall) + { listener .respond_continue(notification.id) - .expect("continue ordinary file open"); + .expect("continue ordinary file operation"); continue; } #[cfg(target_arch = "x86_64")] diff --git a/crates/openshell-sandbox/src/network_broker.rs b/crates/openshell-sandbox/src/network_broker.rs index 1947dabcbe..9b2439d45a 100644 --- a/crates/openshell-sandbox/src/network_broker.rs +++ b/crates/openshell-sandbox/src/network_broker.rs @@ -747,6 +747,11 @@ fn dispatch_notification( if syscall == libc::SYS_open { return queues.provider_files.handle_open(listener, notification); } + if crate::linux::seccomp_notify::is_metadata_syscall(syscall) { + return queues + .provider_files + .handle_metadata(listener, notification); + } if matches!(syscall, libc::SYS_kill | libc::SYS_rt_sigqueueinfo) { return crate::linux::process_signal::mediate_process_signal( listener, @@ -2498,6 +2503,143 @@ mod tests { assert_eq!(detached.unwrap_err().raw_os_error(), Some(libc::ENOENT)); } + #[test] + fn ca_trust_files_are_served_at_their_canonical_paths_through_the_listener() { + use crate::ca_trust_files::{CA_BUNDLE_PATH, CA_CERT_PATH}; + use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; + const CERT: &str = "-----BEGIN CERTIFICATE-----\nlistener\n-----END CERTIFICATE-----\n"; + + let (launcher, listener) = + crate::linux::workload_launcher::start().expect("start workload launcher"); + let broker = NetworkBroker::start_for_test(listener).expect("start broker"); + // Before publication the canonical path continues to the kernel, + // where nothing exists. + let before = launcher.execute(|| std::fs::read(CA_CERT_PATH)).unwrap(); + assert_eq!(before.unwrap_err().raw_os_error(), Some(libc::ENOENT)); + let before = launcher + .execute(|| std::fs::metadata(CA_CERT_PATH)) + .unwrap(); + assert_eq!(before.unwrap_err().raw_os_error(), Some(libc::ENOENT)); + + // A bundle well beyond the provider file bound. + let bundle = format!("{}{CERT}", "S".repeat(150 * 1024)); + broker + .provider_files() + .publish_ca_trust(CERT.as_bytes(), bundle.as_bytes()) + .unwrap(); + let served_cert = launcher + .execute(|| std::fs::read(CA_CERT_PATH)) + .unwrap() + .expect("canonical certificate open"); + assert_eq!(served_cert, CERT.as_bytes()); + let served_bundle = launcher + .execute(|| std::fs::read(CA_BUNDLE_PATH)) + .unwrap() + .expect("canonical bundle open"); + assert_eq!(served_bundle, bundle.as_bytes()); + + // What a TLS client checks before opening: the path is a regular + // read-only file of the served size, the same file an open yields. + let opened = launcher + .execute(|| std::fs::File::open(CA_BUNDLE_PATH)) + .unwrap() + .unwrap(); + let by_path = launcher + .execute(|| std::fs::metadata(CA_BUNDLE_PATH)) + .unwrap() + .expect("stat of the canonical bundle"); + let by_descriptor = opened.metadata().unwrap(); + assert!(by_path.is_file()); + assert_eq!(by_path.len(), bundle.len() as u64); + assert_eq!( + (by_path.dev(), by_path.ino()), + (by_descriptor.dev(), by_descriptor.ino()) + ); + assert_eq!(by_path.mode() & 0o7777, 0o444); + let symlink_stat = launcher + .execute(|| std::fs::symlink_metadata(CA_CERT_PATH)) + .unwrap() + .unwrap(); + assert!(symlink_stat.is_file()); + let readable = launcher + .execute(|| nix::unistd::access(CA_BUNDLE_PATH, nix::unistd::AccessFlags::R_OK)) + .unwrap(); + assert!(readable.is_ok()); + let writable = launcher + .execute(|| nix::unistd::access(CA_BUNDLE_PATH, nix::unistd::AccessFlags::W_OK)) + .unwrap(); + assert_eq!(writable.unwrap_err(), nix::errno::Errno::EACCES); + + // Every other form of open is decided here, never continued. + let open_with = |flags: i32| { + launcher + .execute(move || { + std::fs::OpenOptions::new() + .read(true) + .custom_flags(flags) + .open(CA_CERT_PATH) + }) + .unwrap() + }; + assert_eq!( + open_with(libc::O_DIRECTORY).unwrap_err().raw_os_error(), + Some(libc::ENOTDIR) + ); + assert_eq!( + open_with(libc::O_PATH).unwrap_err().raw_os_error(), + Some(libc::EINVAL) + ); + let write_attempt = launcher + .execute(|| std::fs::OpenOptions::new().write(true).open(CA_BUNDLE_PATH)) + .unwrap() + .expect_err("a write-capable open is refused"); + assert_eq!(write_attempt.raw_os_error(), Some(libc::EACCES)); + + // The directory does not exist, and an alias is not the canonical + // path: both continue to the kernel and find nothing. + let directory = std::path::Path::new(CA_CERT_PATH).parent().unwrap(); + for path in [ + directory.to_path_buf(), + directory.join("../material").join("ca.crt"), + ] { + let result = launcher.execute(move || std::fs::metadata(path)).unwrap(); + assert_eq!(result.unwrap_err().raw_os_error(), Some(libc::ENOENT)); + } + + // Provider updates, including detaching everything, leave the trust + // files served. + broker + .provider_files() + .replace(HashMap::from([( + "/run/openshell/providers/acme/client.toml".to_string(), + "version = 1\n".to_string(), + )])) + .unwrap(); + broker.provider_files().replace(HashMap::new()).unwrap(); + let still_served = launcher + .execute(|| std::fs::read(CA_CERT_PATH)) + .unwrap() + .expect("certificate open after provider updates"); + assert_eq!(still_served, CERT.as_bytes()); + + // A held descriptor outlives the pair; after an explicit clear the + // canonical path is gone again. + let mut held = launcher + .execute(|| std::fs::File::open(CA_CERT_PATH)) + .unwrap() + .expect("hold the cached certificate"); + broker.provider_files().clear_ca_trust(); + let after_clear = launcher.execute(|| std::fs::read(CA_CERT_PATH)).unwrap(); + assert_eq!(after_clear.unwrap_err().raw_os_error(), Some(libc::ENOENT)); + let after_clear = launcher + .execute(|| std::fs::metadata(CA_CERT_PATH)) + .unwrap(); + assert_eq!(after_clear.unwrap_err().raw_os_error(), Some(libc::ENOENT)); + let mut held_content = String::new(); + io::Read::read_to_string(&mut held, &mut held_content).unwrap(); + assert_eq!(held_content, CERT); + } + /// `openat2` with the given `open_how` fields. fn raw_openat2( path: &std::ffi::CStr, @@ -2680,11 +2822,11 @@ mod tests { (libc::O_RDONLY | libc::O_CREAT, libc::EACCES), (libc::O_RDONLY | libc::O_TRUNC, libc::EACCES), (libc::O_RDONLY | libc::O_APPEND, libc::EACCES), - // O_TMPFILE includes the O_DIRECTORY bit, so a directory - // open is refused as a creating one. - (libc::O_RDONLY | libc::O_DIRECTORY, libc::EACCES), + (libc::O_RDONLY | libc::O_TMPFILE, libc::EACCES), + (libc::O_RDONLY | libc::O_DIRECTORY, libc::ENOTDIR), + (libc::O_PATH | libc::O_DIRECTORY, libc::ENOTDIR), (libc::O_RDONLY | libc::O_DIRECT, libc::EINVAL), - (libc::O_RDONLY | libc::O_PATH, libc::EINVAL), + (libc::O_PATH, libc::EINVAL), ] { check( errno_of(raw_openat(provider_path.as_ptr(), flags)) == Some(errno), @@ -2707,10 +2849,110 @@ mod tests { check( errno_of(raw_openat(missing.as_ptr(), libc::O_WRONLY)) == Some(libc::ENOENT) && errno_of(raw_openat2(missing, read_only, 0, 0x08, 24)) + == Some(libc::ENOENT) + && errno_of(std::fs::metadata(missing.to_str().unwrap())) == Some(libc::ENOENT), "unpublished provider file", )?; + // Metadata by path is the injected descriptor's, written + // into the caller's buffer; the directory does not exist. + let by_path = std::fs::metadata(provider).map_err(|e| e.to_string())?; + check( + (by_path.dev(), by_path.ino(), by_path.mode() & 0o7777) == identity(&first) + && by_path.len() == 10 + && by_path.is_file(), + "stat by path differs from fstat", + )?; + check( + errno_of(std::fs::metadata("/run/openshell/providers/acme")) + == Some(libc::ENOENT), + "provider directory exists", + )?; + let statx = |flags: i32, mask: u32, buffer: *mut u8| -> Option { + // SAFETY: the path is NUL-terminated; buffer is either a + // live 256-byte region or deliberately invalid. + let result = unsafe { + libc::syscall( + libc::SYS_statx, + libc::AT_FDCWD, + provider_path.as_ptr(), + flags, + mask, + buffer, + ) + }; + (result < 0).then(|| io::Error::last_os_error().raw_os_error().unwrap()) + }; + let mut statx_buffer = [0_u64; 32]; + check( + statx(0, 0x7ff, statx_buffer.as_mut_ptr().cast()).is_none() + && u64::from_ne_bytes(statx_buffer[5].to_ne_bytes()) == 10, + "statx size", + )?; + check( + statx(0x1_0000, 0x7ff, statx_buffer.as_mut_ptr().cast()) == Some(libc::EINVAL) + && statx(0, 0x8000_0000, statx_buffer.as_mut_ptr().cast()) + == Some(libc::EINVAL), + "statx with an unsupported flag or reserved mask bit", + )?; + // An unwritable result buffer is EFAULT, and the broker + // stays healthy. + check( + statx(0, 0x7ff, std::ptr::null_mut()) == Some(libc::EFAULT) + && statx(0, 0x7ff, 0x10 as *mut u8) == Some(libc::EFAULT), + "statx into an invalid buffer", + )?; + let fstatat = |flags: i32| -> Option { + let mut stat = std::mem::MaybeUninit::::zeroed(); + // SAFETY: path and buffer are live for the call. + let result = unsafe { + libc::syscall( + libc::SYS_newfstatat, + libc::AT_FDCWD, + provider_path.as_ptr(), + stat.as_mut_ptr(), + flags, + ) + }; + (result < 0).then(|| io::Error::last_os_error().raw_os_error().unwrap()) + }; + check( + fstatat(libc::AT_SYMLINK_NOFOLLOW).is_none() + && fstatat(0x4000) == Some(libc::EINVAL) + && statx(0x4000, 0x7ff, statx_buffer.as_mut_ptr().cast()).is_none() + && fstatat(0x8000) == Some(libc::EINVAL), + "fstatat and statx flag sets", + )?; + let access = |mode: i32, flags: i32| -> Option { + // SAFETY: the path is NUL-terminated. + let result = unsafe { + libc::syscall( + libc::SYS_faccessat2, + libc::AT_FDCWD, + provider_path.as_ptr(), + mode, + flags, + ) + }; + (result < 0).then(|| io::Error::last_os_error().raw_os_error().unwrap()) + }; + // (mode, flags, expected errno) for a read-only file. + let access_cases: [(i32, i32, Option); 6] = [ + (libc::F_OK, 0, None), + (libc::R_OK, 0x200, None), + (libc::W_OK, 0, Some(libc::EACCES)), + (libc::X_OK, 0, Some(libc::EACCES)), + (0o10, 0, Some(libc::EINVAL)), + (libc::R_OK, 0x8000, Some(libc::EINVAL)), + ]; + for (mode, flags, expected) in access_cases { + check( + access(mode, flags) == expected, + &format!("access mode {mode:#o} flags {flags:#x}"), + )?; + } + // Ordinary opens are the kernel's, with every open_how field. let ordinary = std::ffi::CString::new(ordinary.to_str().unwrap()).unwrap(); let created = std::ffi::CString::new(created.to_str().unwrap()).unwrap(); diff --git a/crates/openshell-sandbox/src/perf.rs b/crates/openshell-sandbox/src/perf.rs index 558b50c094..ac50142f61 100644 --- a/crates/openshell-sandbox/src/perf.rs +++ b/crates/openshell-sandbox/src/perf.rs @@ -76,6 +76,8 @@ pub enum Protocol { FileOpenRead, /// Read `payload-bytes` from one descriptor opened before measurement. FileRead, + /// `stat` a file by path; the broker answers a provider path. + FileStat, /// `file-open-read` and `tcp-connect` workers running at the same time. MixedFileTcpConnect, /// `file-open-read` and `tcp-stream` workers running at the same time. @@ -93,6 +95,7 @@ impl Protocol { Self::FileOpenClose, Self::FileOpenRead, Self::FileRead, + Self::FileStat, ]), _ => Ok(vec![ Self::from_str(value, true).map_err(|error| anyhow::anyhow!("{error}"))?, @@ -418,7 +421,10 @@ impl Fixture { Protocol::TcpConnect | Protocol::MixedFileTcpConnect => start_tcp_fixture(false), Protocol::TcpStream | Protocol::MixedFileTcpStream => start_tcp_fixture(true), // File protocols need no peer; the worker ignores the target. - Protocol::FileOpenClose | Protocol::FileOpenRead | Protocol::FileRead => Ok(Self { + Protocol::FileOpenClose + | Protocol::FileOpenRead + | Protocol::FileRead + | Protocol::FileStat => Ok(Self { target: SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0), }), } @@ -716,6 +722,7 @@ fn throughput_mbit_per_second( // A mixed protocol is reported per class, never as a whole. Protocol::TcpConnect | Protocol::FileOpenClose + | Protocol::FileStat | Protocol::MixedFileTcpConnect | Protocol::MixedFileTcpStream => 0, Protocol::TcpStream => payload_bytes.saturating_mul(2), @@ -732,6 +739,7 @@ impl Protocol { Self::FileOpenClose => "read-only file open and close", Self::FileOpenRead => "read-only file open, one read, and close", Self::FileRead => "read on an already-open descriptor; no open is measured", + Self::FileStat => "stat by path; no descriptor is created", Self::MixedFileTcpConnect | Self::MixedFileTcpStream => { "file opens and TCP operations at the same time" } @@ -760,7 +768,7 @@ impl Protocol { const fn is_file(self) -> bool { matches!( self, - Self::FileOpenClose | Self::FileOpenRead | Self::FileRead + Self::FileOpenClose | Self::FileOpenRead | Self::FileRead | Self::FileStat ) } } @@ -913,6 +921,16 @@ fn one_operation( require_data(read)?; require_expected(&prepared.response[..read], prepared.expected.as_deref())?; } + Protocol::FileStat => { + let metadata = std::fs::metadata(file_path)?; + if let Some(expected) = prepared.expected.as_deref() + && metadata.len() < expected.len() as u64 + { + return Err(content_mismatch( + "stat reports a shorter file than expected", + )); + } + } Protocol::MixedFileTcpConnect | Protocol::MixedFileTcpStream => { unreachable!("a mixed protocol runs as its classes") } @@ -1184,6 +1202,7 @@ mod tests { Protocol::FileOpenClose, Protocol::FileOpenRead, Protocol::FileRead, + Protocol::FileStat, ] { let report = one_report(&file_run(protocol, &path)); // Two threads, 64 iterations each. @@ -1206,6 +1225,7 @@ mod tests { Protocol::FileOpenClose, Protocol::FileOpenRead, Protocol::FileRead, + Protocol::FileStat, ] { assert!(run_worker(&file_run(protocol, &missing)).is_err()); } diff --git a/crates/openshell-sandbox/src/process.rs b/crates/openshell-sandbox/src/process.rs index 4967d20086..2700a91117 100644 --- a/crates/openshell-sandbox/src/process.rs +++ b/crates/openshell-sandbox/src/process.rs @@ -78,41 +78,11 @@ pub(crate) fn ocsf_launcher_process() -> openshell_ocsf::Process { pub(crate) fn prepare_child_sandbox( policy: &SandboxPolicy, workdir: Option<&str>, - runtime_read_only: &[PathBuf], ) -> Result> { - let effective_policy = policy_with_runtime_read_only(policy, runtime_read_only); - let prepared = sandbox::linux::prepare_capability_free(&effective_policy, workdir)?; + let prepared = sandbox::linux::prepare_capability_free(policy, workdir)?; Ok(Some(prepared)) } -#[cfg(target_os = "linux")] -fn policy_with_runtime_read_only( - policy: &SandboxPolicy, - runtime_read_only: &[PathBuf], -) -> SandboxPolicy { - let mut effective_policy = policy.clone(); - for path in runtime_read_only { - if !effective_policy.filesystem.read_only.contains(path) { - effective_policy.filesystem.read_only.push(path.clone()); - } - } - effective_policy -} - -#[cfg(target_os = "linux")] -pub(crate) fn ca_runtime_read_only_paths(ca_paths: Option<&(PathBuf, PathBuf)>) -> Vec { - let Some((certificate, bundle)) = ca_paths else { - return Vec::new(); - }; - let mut paths = Vec::with_capacity(3); - if let Some(directory) = certificate.parent() { - paths.push(directory.to_path_buf()); - } - paths.push(certificate.clone()); - paths.push(bundle.clone()); - paths -} - /// Prefix of environment variable names reserved for `OpenShell`. pub(crate) const RESERVED_ENV_PREFIX: &str = "OPENSHELL_"; @@ -611,9 +581,7 @@ impl ProcessHandle { // Prepare the Landlock ruleset as the workload UID. Inaccessible paths // are already unavailable to the child and remain omitted. - #[cfg(target_os = "linux")] - let runtime_read_only = ca_runtime_read_only_paths(ca_paths); - let prepared_sandbox = prepare_child_sandbox(policy, workspace.root(), &runtime_read_only) + let prepared_sandbox = prepare_child_sandbox(policy, workspace.root()) .map_err(|err| miette::miette!("Failed to prepare sandbox: {err}"))?; #[cfg(target_os = "linux")] let mut child_hardening = crate::linux::child_seccomp::prepare(std::process::id()) @@ -1933,85 +1901,6 @@ mod tests { } } - #[cfg(target_os = "linux")] - #[test] - fn runtime_ca_paths_are_added_to_the_effective_read_only_policy() { - let mut policy = policy_with_process(ProcessPolicy::default()); - policy.filesystem.read_only = vec![PathBuf::from("/usr")]; - let certificate = PathBuf::from(format!( - "{}/ca.crt", - openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR - )); - let bundle = PathBuf::from(format!( - "{}/ca-bundle.crt", - openshell_sandbox_backend::SUPERVISOR_CA_RUNTIME_DIR - )); - - let effective = policy_with_runtime_read_only( - &policy, - &[certificate.clone(), bundle.clone(), certificate.clone()], - ); - - assert_eq!(policy.filesystem.read_only, vec![PathBuf::from("/usr")]); - assert_eq!( - effective.filesystem.read_only, - vec![PathBuf::from("/usr"), certificate, bundle] - ); - } - - #[cfg(target_os = "linux")] - #[test] - #[allow(unsafe_code)] - fn runtime_ca_material_remains_readable_after_landlock_for_non_root_workload() { - let root = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(root.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); - let ca_directory = root.path().join("openshell-supervisor-ca"); - std::fs::create_dir(&ca_directory).unwrap(); - std::fs::set_permissions(&ca_directory, std::fs::Permissions::from_mode(0o755)).unwrap(); - let certificate = ca_directory.join("ca.crt"); - let bundle = ca_directory.join("ca-bundle.crt"); - let denied = root.path().join("not-authorized"); - for path in [&certificate, &bundle, &denied] { - std::fs::write(path, b"public certificate material").unwrap(); - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o444)).unwrap(); - } - - let mut policy = policy_with_process(ProcessPolicy::default()); - policy.landlock = LandlockPolicy { - compatibility: openshell_core::policy::LandlockCompatibility::HardRequirement, - }; - let runtime_paths = - ca_runtime_read_only_paths(Some(&(certificate.clone(), bundle.clone()))); - let Ok(Some(prepared)) = prepare_child_sandbox(&policy, None, &runtime_paths) else { - return; - }; - - match unsafe { fork() }.expect("fork should succeed") { - ForkResult::Child => { - let dropped = if nix::unistd::geteuid().is_root() { - unsafe { - libc::setgroups(0, std::ptr::null()) == 0 - && libc::setgid(42_235) == 0 - && libc::setuid(42_234) == 0 - } - } else { - true - }; - let valid = dropped - && sandbox::linux::enforce(prepared).is_ok() - && std::fs::read(&certificate).is_ok() - && std::fs::read(&bundle).is_ok() - && std::fs::read(&denied).is_err(); - unsafe { libc::_exit(i32::from(!valid)) }; - } - ForkResult::Parent { child } => assert_eq!( - waitpid(child, None).expect("waitpid should succeed"), - WaitStatus::Exited(child, 0), - "Landlock must preserve non-root access only to admitted public CA material" - ), - } - } - #[tokio::test] async fn inject_provider_env_skips_supervisor_identity_material() { let mut cmd = Command::new("/usr/bin/env"); diff --git a/crates/openshell-sandbox/src/provider_files.rs b/crates/openshell-sandbox/src/provider_files.rs index 376b210077..d9fd4beb4e 100644 --- a/crates/openshell-sandbox/src/provider_files.rs +++ b/crates/openshell-sandbox/src/provider_files.rs @@ -1,26 +1,41 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Read-only provider files. +//! Read-only virtual files: provider files and the supervisor CA trust pair. //! -//! Every workload file open notifies the broker, which serves a path beneath -//! the provider prefix by injecting a read-only descriptor for a sealed -//! in-memory file and continues every other open. +//! Neither kind exists on any filesystem. Every workload file open notifies +//! the broker, which serves a path beneath the provider prefix, or one of the +//! two canonical CA trust paths, by injecting a read-only descriptor for a +//! sealed in-memory file, and continues every other open. Every path-based +//! `stat`, `statx`, and `access` call notifies as well; for a virtual path the +//! broker writes the result the injected descriptor would report into the +//! caller's buffer, and continues every other path. +//! +//! What a virtual path supports is exactly this: an absolute, canonical path +//! (no `..`, no trailing slash, no symlink, no relative lookup through a +//! directory descriptor), opened read-only, and queried through `stat`, +//! `lstat`, `fstatat`, `statx`, and `access`. The directories above a virtual +//! file do not exist: `chdir`, `opendir`, and `stat` of the directory are +//! `ENOENT`, and a relative or aliased path is left to the kernel, which finds +//! nothing there. A write-capable open is `EACCES`; `O_DIRECTORY` is +//! `ENOTDIR`; `O_PATH` is `EINVAL`. //! //! Two things are kept apart. A [`Revision`] is one immutable version of one //! file, and a [`Snapshot`] is the complete set a publication replaces at //! once; neither knows how bytes reach the workload. What a revision prepares -//! at publication is its backing, a [`SealedMemfd`], created once when the -//! revision is published and reopened for each workload open. +//! at publication is its backing, a [`SealedFile`], created once when the +//! revision is published and reopened for each workload open. The CA trust +//! pair from [`crate::ca_trust_files`] shares the backing but is published +//! separately from, and never removed by, a provider file update. use std::collections::HashMap; -use std::fs::File; -use std::io::{self, Write as _}; -use std::os::fd::{AsFd, AsRawFd as _, OwnedFd}; -use std::sync::{Arc, OnceLock, RwLock}; +use std::io; +use std::sync::{Arc, RwLock}; -use rustix::fs::{Mode, OFlags, SealFlags}; +use rustix::fs::Mode; +use crate::ca_trust_files::{CaTrustFiles, TrustFile}; +use crate::linux::sealed_file::{OwnDescriptors, SealedFile}; use crate::linux::seccomp_notify::{Notification, NotificationListener}; use crate::linux::task_memory; @@ -30,6 +45,22 @@ const MAX_FILE_BYTES: usize = 65_536; pub const MAX_TOTAL_BYTES: usize = 262_144; const MAX_PATH_BYTES: usize = 4_096; +/// The flags `fstatat` accepts; `statx` also accepts `AT_STATX_SYNC_TYPE`. +const STAT_FLAGS: u32 = + (libc::AT_SYMLINK_NOFOLLOW | libc::AT_NO_AUTOMOUNT | libc::AT_EMPTY_PATH) as u32; +/// `AT_STATX_SYNC_TYPE`: the UAPI mask of the `statx` sync flags, which the +/// `libc` crate does not expose on every target. +const AT_STATX_SYNC_TYPE: u32 = 0x6000; +const STATX_FLAGS: u32 = STAT_FLAGS | AT_STATX_SYNC_TYPE; +/// `AT_EACCESS`, also absent from `libc` on some targets. +const AT_EACCESS: u32 = 0x200; +/// The flags `faccessat2` accepts. +const ACCESS_FLAGS: u32 = AT_EACCESS | (libc::AT_SYMLINK_NOFOLLOW | libc::AT_EMPTY_PATH) as u32; +/// The `access` modes: `R_OK | W_OK | X_OK`. +const ACCESS_MODES: u32 = (libc::R_OK | libc::W_OK | libc::X_OK) as u32; +/// `STATX__RESERVED`: a mask with this bit set is refused. +const STATX_RESERVED_MASK: u32 = 0x8000_0000; + /// The mode a provider file reports. It keeps the file's metadata private /// to the sandbox identity; the seals and the read-only descriptor protect /// its content. @@ -71,109 +102,31 @@ fn next_snapshot( .collect() } -/// This process's own descriptor table in procfs. -/// -/// Opening a descriptor's entry here yields a new open file description of -/// the same file. The directory is opened once and verified to be procfs, so -/// a later reopen walks no path. -struct OwnDescriptors(OwnedFd); - -impl OwnDescriptors { - fn open() -> io::Result { - let directory = rustix::fs::open( - "/proc/self/fd", - OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC, - Mode::empty(), - )?; - if rustix::fs::fstatfs(&directory)?.f_type != rustix::fs::PROC_SUPER_MAGIC { - return Err(io::Error::other("/proc/self/fd is not on procfs")); - } - Ok(Self(directory)) - } - - /// A new read-only open file description of the file `descriptor` refers - /// to, positioned at its start. Duplicating `descriptor` instead would - /// share one file offset between every holder. - fn reopen_read_only(&self, descriptor: impl AsFd) -> io::Result { - Ok(rustix::fs::openat( - &self.0, - rustix::path::DecInt::new(descriptor.as_fd().as_raw_fd()), - OFlags::RDONLY | OFlags::CLOEXEC | OFlags::NOCTTY, - Mode::empty(), - )?) - } -} - -/// The sealed in-memory file behind one revision. -/// -/// The content is written once and sealed against writes, resizing, and -/// further sealing, so no descriptor of the file, however it was opened, can -/// change it. The only descriptor kept here is itself read-only: the -/// writable one `memfd_create` returns is closed before the revision is -/// published. -struct SealedMemfd { - anchor: OwnedFd, - descriptors: Arc, -} - -impl SealedMemfd { - fn create(content: &[u8], descriptors: &Arc) -> io::Result { - let writable = rustix::fs::memfd_create( - "openshell-provider", - rustix::fs::MemfdFlags::CLOEXEC | rustix::fs::MemfdFlags::ALLOW_SEALING, - )?; - rustix::fs::fchmod(&writable, FILE_MODE)?; - let mut writable = File::from(writable); - writable.write_all(content)?; - rustix::fs::fcntl_add_seals( - &writable, - SealFlags::SEAL | SealFlags::WRITE | SealFlags::GROW | SealFlags::SHRINK, - )?; - let anchor = descriptors.reopen_read_only(&writable)?; - Ok(Self { - anchor, - descriptors: Arc::clone(descriptors), - }) - } - - /// A descriptor for one workload open: its own read-only open file - /// description, positioned at the start of the file. - fn open(&self) -> io::Result { - match self.descriptors.reopen_read_only(&self.anchor) { - // Every open of a revision shares one file, which the workload - // owns as well and may have made unreadable through a descriptor - // it holds. Its mode is not what protects the content, so - // restore it rather than fail opens until the next revision. - Err(error) if error.raw_os_error() == Some(libc::EACCES) => { - rustix::fs::fchmod(&self.anchor, FILE_MODE)?; - self.descriptors.reopen_read_only(&self.anchor) - } - opened => opened, - } - } +/// The sealed in-memory file behind one provider revision. +fn prepare_backing(content: &[u8]) -> io::Result { + SealedFile::create( + "openshell-provider", + FILE_MODE, + content, + &OwnDescriptors::shared()?, + ) } -/// The published generation of provider files. +/// The published generation of provider files, and the supervisor trust +/// files served with them. #[derive(Default)] struct VirtualFiles { /// Replaced as a whole. Open handlers clone the selected revision before /// dropping the lock, so an update never waits on a workload open and a /// revision outlives its generation for as long as an open uses it. - current: RwLock>>, - /// Opened by the first publication that needs it. - descriptors: OnceLock>, + current: RwLock>>, + /// Published once the CA material is installed, before the workload + /// starts. A provider file update does not touch it. + ca_trust: CaTrustFiles, } impl VirtualFiles { - fn descriptors(&self) -> io::Result<&Arc> { - if let Some(descriptors) = self.descriptors.get() { - return Ok(descriptors); - } - let opened = Arc::new(OwnDescriptors::open()?); - Ok(self.descriptors.get_or_init(|| opened)) - } - - fn snapshot(&self) -> Arc> { + fn snapshot(&self) -> Arc> { Arc::clone( &self .current @@ -182,7 +135,7 @@ impl VirtualFiles { ) } - fn revision(&self, path: &str) -> Option>> { + fn revision(&self, path: &str) -> Option>> { self.current .read() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -191,9 +144,7 @@ impl VirtualFiles { } fn publish(&self, desired: HashMap) -> io::Result<()> { - self.publish_with(desired, |content| { - SealedMemfd::create(content, self.descriptors()?) - }) + self.publish_with(desired, prepare_backing) } /// Replace the generation with exactly `desired`, or leave it untouched. @@ -205,7 +156,7 @@ impl VirtualFiles { fn publish_with( &self, desired: HashMap, - prepare: impl FnMut(&[u8]) -> io::Result, + prepare: impl FnMut(&[u8]) -> io::Result, ) -> io::Result<()> { let next = Arc::new(next_snapshot(&self.snapshot(), desired, prepare)?); let replaced = std::mem::replace( @@ -253,6 +204,35 @@ impl ProviderFiles { self.files.publish(desired) } + /// Publish the supervisor CA certificate and combined trust bundle the + /// workload opens at their canonical paths. + /// + /// Call this after the real files are installed and before the workload + /// starts. The pair is cached and plain read-only opens of exactly those + /// paths receive a sealed copy. A refused pair fails closed and a + /// previously published pair stays as it was. + pub(crate) fn publish_ca_trust(&self, ca_cert: &[u8], ca_bundle: &[u8]) -> io::Result<()> { + self.files.ca_trust.publish(ca_cert, ca_bundle) + } + + /// Forget any published trust pair, so every open of the canonical paths + /// continues to the kernel again. + /// + /// Call this before a workload starts without CA material: a pair cached + /// by an earlier, failed start must not outlive the installation it + /// mirrored. Independent of provider file updates, which never touch + /// the pair. + pub(crate) fn clear_ca_trust(&self) { + self.files.ca_trust.clear(); + } + + /// The cached trust pair. + #[cfg(test)] + pub(crate) fn ca_trust(&self) -> &CaTrustFiles { + &self.files.ca_trust + } + + /// Answer a workload `open`, `openat`, or `openat2`. pub(crate) fn handle_open( &self, listener: &NotificationListener, @@ -264,75 +244,321 @@ impl ProviderFiles { } else { notification.args[0] }; - // Every workload open reaches the listener, and nearly all of them - // are ordinary paths the kernel decides. One exact copy of the - // path's first six bytes tells those apart from the two kinds of path - // served here; an ordinary open costs nothing else, and in - // particular no read of an `openat2` argument structure. - // - // Six bytes are enough. `/proc/` is six bytes, and every provider - // path starts with the first six bytes of its prefix and is longer - // than that, so both kinds always have six readable bytes. A copy - // that fails is an unreadable pointer or a shorter path that ends - // before an unmapped page: neither is served here, and the kernel - // resolves the path or reports its own EFAULT. - // - // The copy only classifies. Both served kinds read the whole path - // again and decide from that, and neither is ever continued: the - // kernel and Landlock authorize every open that is. + let file = match self.classify(listener, ¬ification, path_address)? { + Classified::Served(file) => file, + Classified::Done => return Ok(()), + Classified::Proc => { + if handle_thread_comm_open(listener, notification, path_address)? { + return Ok(()); + } + return listener.respond_continue(notification.id); + } + }; + inject_read_only(listener, ¬ification, file.backing()) + } + + /// Answer a workload path-based `stat`, `statx`, or `access` call. + pub(crate) fn handle_metadata( + &self, + listener: &NotificationListener, + notification: Notification, + ) -> io::Result<()> { + let request = MetadataRequest::parse(¬ification)?; + let file = match self.classify(listener, ¬ification, request.path_address())? { + Classified::Served(file) => file, + Classified::Done => return Ok(()), + Classified::Proc => return listener.respond_continue(notification.id), + }; + let backing = file.backing(); + match request { + MetadataRequest::Stat { flags, buffer, .. } => { + if flags & !STAT_FLAGS != 0 { + return listener.respond_errno(notification.id, libc::EINVAL); + } + complete_with_result(listener, ¬ification, buffer, &backing.stat()?.0) + } + MetadataRequest::Statx { + flags, + mask, + buffer, + .. + } => { + if flags & !STATX_FLAGS != 0 || mask & STATX_RESERVED_MASK != 0 { + return listener.respond_errno(notification.id, libc::EINVAL); + } + let sync = flags & AT_STATX_SYNC_TYPE; + complete_with_result( + listener, + ¬ification, + buffer, + &backing.statx(sync, mask)?.0, + ) + } + MetadataRequest::Access { mode, flags, .. } => { + if mode & !ACCESS_MODES != 0 || flags & !ACCESS_FLAGS != 0 { + return listener.respond_errno(notification.id, libc::EINVAL); + } + // The file is readable and nothing else, whichever identity + // the call checks. + if mode & ACCESS_MODES != libc::R_OK as u32 && mode != libc::F_OK as u32 { + return listener.respond_errno(notification.id, libc::EACCES); + } + listener.respond_value(notification.id, 0) + } + } + } + + /// Decide whether the path at `path_address` is a virtual file. + /// + /// Every workload open and metadata call reaches the listener, and + /// nearly all of them name ordinary paths the kernel decides. One exact + /// copy of the path's first six bytes tells those apart from the paths + /// served here, at no other cost to an ordinary call. Six bytes are + /// enough: `/proc/` is six bytes, and every provider path and both CA + /// trust paths share the first six bytes of the provider prefix and are + /// longer than that. A copy that fails is an unreadable pointer or a + /// shorter path ending before an unmapped page: neither is served, and + /// the kernel resolves the path or reports its own EFAULT. + /// + /// The copy only classifies. A candidate has its whole path read and is + /// decided from that copy alone: a published virtual file is served + /// whatever the flags, an unpublished path beneath the provider prefix is + /// `ENOENT`, and everything else, including an alias of a virtual path, + /// continues to the kernel, which finds nothing of ours there. + fn classify( + &self, + listener: &NotificationListener, + notification: &Notification, + path_address: u64, + ) -> io::Result { let mut start = [0_u8; PROC_PREFIX.len()]; if task_memory::read_exact(notification.tid, path_address, &mut start).is_err() { - return listener.respond_continue(notification.id); + listener.respond_continue(notification.id)?; + return Ok(Classified::Done); } if start == PROC_PREFIX.as_bytes() { - if handle_thread_comm_open(listener, notification, path_address)? { - return Ok(()); + return Ok(Classified::Proc); + } + if start == PREFIX.as_bytes()[..PROC_PREFIX.len()] + && let Ok(path) = read_path(notification.tid, path_address) + { + if let Some(file) = self.files.lookup(&path) { + return Ok(Classified::Served(file)); + } + if path.starts_with(PREFIX) { + listener.respond_errno(notification.id, libc::ENOENT)?; + return Ok(Classified::Done); } - return listener.respond_continue(notification.id); } - if start != PREFIX.as_bytes()[..PROC_PREFIX.len()] { - return listener.respond_continue(notification.id); + listener.respond_continue(notification.id)?; + Ok(Classified::Done) + } +} + +enum Classified { + /// A published virtual file; the caller answers from it. + Served(VirtualFile), + /// The notification was answered here. + Done, + /// A `/proc/` path, which only the open handler looks at further. + Proc, +} + +/// One published file, held so a concurrent publication cannot close its +/// backing while it is served. +enum VirtualFile { + Provider(Arc>), + Trust(TrustFile), +} + +impl VirtualFile { + fn backing(&self) -> &SealedFile { + match self { + Self::Provider(revision) => &revision.backing, + Self::Trust(file) => file.backing(), } - // A failed or non-absolute lookup is left to the kernel. In particular, - // this preserves its normal EFAULT result for an invalid path pointer. - let Ok(path) = read_path(notification.tid, path_address) else { - return listener.respond_continue(notification.id); - }; - if !path.starts_with(PREFIX) { - return listener.respond_continue(notification.id); + } +} + +impl VirtualFiles { + fn lookup(&self, path: &str) -> Option { + if path.starts_with(PREFIX) { + return self.revision(path).map(VirtualFile::Provider); } - let Some(revision) = self.files.revision(&path) else { - return listener.respond_errno(notification.id, libc::ENOENT); - }; - let flags = match open_flags(¬ification) { - Ok(flags) => flags, - Err(error) => { - return listener.respond_errno( - notification.id, - error.raw_os_error().unwrap_or(libc::EINVAL), - ); + self.ca_trust.lookup(path).map(VirtualFile::Trust) + } +} + +/// A notified metadata call, with the arguments the broker checks. +enum MetadataRequest { + Stat { + path: u64, + buffer: u64, + flags: u32, + }, + Statx { + path: u64, + flags: u32, + mask: u32, + buffer: u64, + }, + Access { + path: u64, + mode: u32, + flags: u32, + }, +} + +impl MetadataRequest { + /// The call's arguments. `int` arguments keep their low 32 bits, as the + /// kernel reads them. + fn parse(notification: &Notification) -> io::Result { + let args = notification.args; + #[allow(clippy::cast_possible_truncation)] + let word = |index: usize| args[index] as u32; + let syscall = i64::from(notification.syscall); + if syscall == libc::SYS_newfstatat { + return Ok(Self::Stat { + path: args[1], + buffer: args[2], + flags: word(3), + }); + } + if syscall == libc::SYS_statx { + return Ok(Self::Statx { + path: args[1], + flags: word(2), + mask: word(3), + buffer: args[4], + }); + } + if syscall == libc::SYS_faccessat { + return Ok(Self::Access { + path: args[1], + mode: word(2), + flags: 0, + }); + } + if syscall == libc::SYS_faccessat2 { + return Ok(Self::Access { + path: args[1], + mode: word(2), + flags: word(3), + }); + } + #[cfg(target_arch = "x86_64")] + if syscall == libc::SYS_stat || syscall == libc::SYS_lstat { + return Ok(Self::Stat { + path: args[0], + buffer: args[1], + flags: 0, + }); + } + #[cfg(target_arch = "x86_64")] + if syscall == libc::SYS_access { + return Ok(Self::Access { + path: args[0], + mode: word(1), + flags: 0, + }); + } + Err(io::Error::from_raw_os_error(libc::ENOSYS)) + } + + const fn path_address(&self) -> u64 { + match self { + Self::Stat { path, .. } | Self::Statx { path, .. } | Self::Access { path, .. } => *path, + } + } +} + +/// Write `result` into the notifying task at `buffer` and complete the +/// syscall with 0, or fail it with `EFAULT`. +/// +/// The target is pinned before anything is written: `/proc//mem` is +/// opened and the notification validated afterwards, which proves the pinned +/// address space is the notifying task's (see +/// [`task_memory::PinnedMemory`]). The listener's `WAIT_KILLABLE_RECV` keeps +/// the task in the syscall, so the buffer is still the caller's when the +/// write lands. A buffer the write cannot reach fails the call with `EFAULT`, +/// whatever errno the pinned file reports; every response validates the +/// notification, so a task killed meanwhile is simply no longer answered. +pub fn complete_with_result( + listener: &NotificationListener, + notification: &Notification, + buffer: u64, + result: &[u8], +) -> io::Result<()> { + let outcome = match task_memory::PinnedMemory::open(notification.tid) { + Ok(memory) => { + // The task may have died between the notification and here: the + // pinned memory would then belong to a reused TID. ENOENT means + // the notification is gone; any other ioctl failure is an error. + match listener.validate_id(notification.id) { + Ok(()) => {} + Err(error) if error.raw_os_error() == Some(libc::ENOENT) => return Ok(()), + Err(error) => return Err(error), } - }; - if flags & libc::O_ACCMODE != libc::O_RDONLY - || flags - & (libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC | libc::O_TMPFILE | libc::O_APPEND) - != 0 - { - return listener.respond_errno(notification.id, libc::EACCES); + memory.write_exact(buffer, result) } - if flags & (libc::O_DIRECTORY | libc::O_PATH | libc::O_DIRECT) != 0 { - return listener.respond_errno(notification.id, libc::EINVAL); + // No such task: its procfs entry is gone (ENOENT) or it is exiting + // (ESRCH). The response below reports whether anything is left. + Err(error) if matches!(error.raw_os_error(), Some(libc::ENOENT | libc::ESRCH)) => { + Err(error) } - let file = revision.backing.open()?; - listener.add_fd_and_send( - notification.id, - file.as_raw_fd(), - flags & libc::O_CLOEXEC != 0, - )?; - Ok(()) + Err(error) => return Err(error), + }; + let response = match outcome { + Ok(()) => listener.respond_value(notification.id, 0), + Err(_) => listener.respond_errno(notification.id, libc::EFAULT), + }; + match response { + Err(error) if error.raw_os_error() == Some(libc::ENOENT) => Ok(()), + other => other, } } +/// Complete a workload open of a virtual file, or refuse it. +/// +/// Only a read-only open is served. An open that asks to create, truncate, +/// append, or write is `EACCES`, as on a read-only file; `O_DIRECTORY` is +/// `ENOTDIR`; `O_PATH` and `O_DIRECT` are `EINVAL` (a path-only descriptor +/// cannot be injected: `SECCOMP_IOCTL_NOTIF_ADDFD` refuses `O_PATH` files). +/// Nothing here is continued. +fn inject_read_only( + listener: &NotificationListener, + notification: &Notification, + file: &SealedFile, +) -> io::Result<()> { + let flags = match open_flags(notification) { + Ok(flags) => flags, + Err(error) => { + return listener.respond_errno( + notification.id, + error.raw_os_error().unwrap_or(libc::EINVAL), + ); + } + }; + let close_on_exec = flags & libc::O_CLOEXEC != 0; + // O_TMPFILE includes the O_DIRECTORY bit; a creating open is refused as + // one before the directory bit is read. + if flags & libc::O_TMPFILE == libc::O_TMPFILE { + return listener.respond_errno(notification.id, libc::EACCES); + } + if flags & libc::O_DIRECTORY != 0 { + return listener.respond_errno(notification.id, libc::ENOTDIR); + } + if flags & libc::O_ACCMODE != libc::O_RDONLY + || flags & (libc::O_CREAT | libc::O_EXCL | libc::O_TRUNC | libc::O_APPEND) != 0 + { + return listener.respond_errno(notification.id, libc::EACCES); + } + if flags & (libc::O_PATH | libc::O_DIRECT) != 0 { + return listener.respond_errno(notification.id, libc::EINVAL); + } + file.inject(listener, notification.id, close_on_exec) +} + fn safe_component(value: &str) -> bool { !value.is_empty() && value != "." @@ -420,7 +646,7 @@ fn handle_thread_comm_open( .open(format!("/proc/{caller_group}/task/{target}/comm"))?; listener.add_fd_and_send( notification.id, - file.as_raw_fd(), + std::os::fd::AsRawFd::as_raw_fd(&file), flags & libc::O_CLOEXEC != 0, )?; Ok(true) @@ -499,12 +725,12 @@ fn open_flags(notification: &Notification) -> io::Result { #[cfg(test)] mod tests { - use super::{MAX_FILE_BYTES, ProviderFiles, SealedMemfd, VirtualFiles, comm_target}; + use super::{MAX_FILE_BYTES, ProviderFiles, VirtualFiles, comm_target, prepare_backing}; + use crate::ca_trust_files::{CA_BUNDLE_PATH, CA_CERT_PATH}; use std::collections::HashMap; use std::fs::File; - use std::io::{self, Read as _, Seek as _, SeekFrom}; - use std::os::fd::AsRawFd as _; - use std::os::unix::fs::{FileExt as _, MetadataExt as _, PermissionsExt as _}; + use std::io::{self, Read as _}; + use std::os::unix::fs::{FileExt as _, MetadataExt as _}; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; @@ -582,93 +808,36 @@ mod tests { } #[test] - fn each_open_is_its_own_read_only_description_of_one_sealed_file() { + fn ca_trust_files_are_independent_of_provider_file_updates() { + const CERT: &str = "-----BEGIN CERTIFICATE-----\nsupervisor\n-----END CERTIFICATE-----\n"; + let bundle = format!("{}{CERT}", "B".repeat(MAX_FILE_BYTES)); let provider_files = ProviderFiles::default(); + let trust = provider_files.ca_trust(); + assert!(trust.lookup(CA_CERT_PATH).is_none()); provider_files - .replace(files(&[(CONFIG, "0123456789")])) + .publish_ca_trust(CERT.as_bytes(), bundle.as_bytes()) .unwrap(); - let mut first = open(&provider_files, CONFIG).unwrap(); - let mut second = open(&provider_files, CONFIG).unwrap(); - - // Both opens are positioned at the start, and neither moves the - // other: a duplicated descriptor would share one offset. - let mut head = [0_u8; 4]; - first.read_exact(&mut head).unwrap(); - assert_eq!(&head, b"0123"); - let mut whole = String::new(); - second.read_to_string(&mut whole).unwrap(); - assert_eq!(whole, "0123456789"); - let mut rest = String::new(); - first.read_to_string(&mut rest).unwrap(); - assert_eq!(rest, "456789"); - second.seek(SeekFrom::Start(8)).unwrap(); - assert_eq!(first.stream_position().unwrap(), 10); - - // One file serves every open of a revision. - assert_eq!(file_identity(&first), file_identity(&second)); - for file in [&first, &second] { - assert_eq!(file.metadata().unwrap().mode() & 0o7777, 0o600); - assert_eq!( - rustix::fs::fcntl_getfl(file).unwrap() & rustix::fs::OFlags::ACCMODE, - rustix::fs::OFlags::RDONLY - ); - let seals = rustix::fs::fcntl_get_seals(file).unwrap(); - assert!(seals.contains( - rustix::fs::SealFlags::SEAL - | rustix::fs::SealFlags::WRITE - | rustix::fs::SealFlags::GROW - | rustix::fs::SealFlags::SHRINK - )); - assert!(file.write_at(b"changed", 0).is_err()); - assert!(file.set_len(0).is_err()); - } + let trust_file = |path: &str| { + let file = File::from(trust.lookup(path).unwrap().backing().open().unwrap()); + (file.metadata().unwrap().len(), file_identity(&file)) + }; + let (cert_len, cert_identity) = trust_file(CA_CERT_PATH); + let (bundle_len, bundle_identity) = trust_file(CA_BUNDLE_PATH); + assert_eq!(cert_len, u64::try_from(CERT.len()).unwrap()); + assert_eq!(bundle_len, u64::try_from(bundle.len()).unwrap()); - // The workload owns the file and can reopen its descriptor for - // writing. The seals, not the descriptor's access mode, are what - // keep the content, and they cannot be lifted. - let link = format!("/proc/self/fd/{}", first.as_raw_fd()); - for options in [ - std::fs::OpenOptions::new().write(true).clone(), - std::fs::OpenOptions::new().read(true).write(true).clone(), - std::fs::OpenOptions::new().append(true).clone(), - ] { - if let Ok(writable) = options.open(&link) { - assert!(writable.write_at(b"changed", 0).is_err()); - assert!(io::Write::write_all(&mut &writable, b"changed").is_err()); - assert!(writable.set_len(0).is_err()); - assert!(writable.set_len(1 << 20).is_err()); - } - } + provider_files + .replace(files(&[(CONFIG, "version = 1\n")])) + .unwrap(); + provider_files.replace(HashMap::new()).unwrap(); + assert_eq!(trust_file(CA_CERT_PATH).1, cert_identity); + assert_eq!(trust_file(CA_BUNDLE_PATH).1, bundle_identity); assert!( - std::fs::OpenOptions::new() - .write(true) - .truncate(true) - .open(&link) - .is_err() - ); - assert_eq!(read(&provider_files, CONFIG).unwrap(), "0123456789"); - - // The descriptor the runtime keeps for the revision is read-only too. - let revision = virtual_files(&provider_files).revision(CONFIG).unwrap(); - assert_eq!( - rustix::fs::fcntl_getfl(&revision.backing.anchor).unwrap() - & rustix::fs::OFlags::ACCMODE, - rustix::fs::OFlags::RDONLY + virtual_files(&provider_files) + .revision(CA_CERT_PATH) + .is_none() ); - } - - #[test] - fn a_cleared_mode_does_not_deny_later_opens() { - let provider_files = ProviderFiles::default(); - provider_files.replace(files(&[(KEY, "key-1")])).unwrap(); - let held = open(&provider_files, KEY).unwrap(); - // The workload owns the shared file and can change its mode. - held.set_permissions(std::fs::Permissions::from_mode(0o000)) - .unwrap(); - assert_eq!(read(&provider_files, KEY).unwrap(), "key-1"); - // An identical update keeps the revision; it must stay readable. - provider_files.replace(files(&[(KEY, "key-1")])).unwrap(); - assert_eq!(read(&provider_files, KEY).unwrap(), "key-1"); + assert!(ProviderFiles::validate(&files(&[(CA_BUNDLE_PATH, "x")])).is_err()); } #[test] @@ -797,7 +966,7 @@ mod tests { if prepared == 2 { return Err(io::Error::from_raw_os_error(libc::EMFILE)); } - SealedMemfd::create(content, files_state.descriptors()?) + prepare_backing(content) }, ) .unwrap_err(); diff --git a/docs/about/support-matrix.mdx b/docs/about/support-matrix.mdx index de62a63908..a3995a0b9d 100644 --- a/docs/about/support-matrix.mdx +++ b/docs/about/support-matrix.mdx @@ -180,8 +180,8 @@ when it runs inside a container or microVM: | Module | Requirement | Details | | -------------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | | [Landlock LSM](https://docs.kernel.org/security/landlock.html) | Required | ABI 3 or newer, introduced in Linux 6.2, with Landlock enabled. The mandatory baseline protects private channel and bootstrap files, including against truncation. A filesystem policy's `best_effort` setting never disables this baseline. | -| seccomp | Required | Nested user-notification filters and atomic `SECCOMP_IOCTL_NOTIF_ADDFD` with `SECCOMP_ADDFD_FLAG_SEND`, usable under the runtime's existing seccomp profile without added capabilities. The sandbox actively probes these operations before admitting the workload. | -| Task-memory access | Required | The non-dumpable broker must be able to read a same-UID, dumpable workload child's memory through `process_vm_readv` or `/proc//mem`. The broker never writes workload memory. The sandbox actively probes the production parent-to-child topology before admitting the workload. | +| seccomp | Required | Nested user-notification filters with `SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV` (Linux 5.19) and atomic `SECCOMP_IOCTL_NOTIF_ADDFD` with `SECCOMP_ADDFD_FLAG_SEND`, usable under the runtime's existing seccomp profile without added capabilities. The sandbox actively probes these operations before admitting the workload. Once the broker has received a mediated syscall, a handled signal is delivered after the broker responds: a `connect` awaiting the supervisor's network admission decision ignores handled `SIGALRM` and `SIGINT` for up to 30 seconds, DNS waits for up to 10 seconds, and a blocking loopback `connect` for the kernel's connect timeout, which can be longer; fatal signals still terminate the process. | +| Task-memory access | Required | The non-dumpable broker must be able to read a same-UID, dumpable workload child's memory through `process_vm_readv` or `/proc//mem`, and write through `/proc//mem`. Writes deliver only the results of `stat` and `statx` on the virtual provider and CA trust files, into the buffer the call names. The sandbox actively probes the production parent-to-child topology before admitting the workload. | | Socket device binding | Required | The sandbox binds every workload TCP and UDP socket to the loopback interface with `SO_BINDTODEVICE`, without added capabilities. Accepted sockets inherit the binding, so workloads accept connections only from inside the sandbox network namespace. The sandbox probes this before admitting the workload. | A kernel version alone does not establish support. A disabled Landlock LSM or a diff --git a/docs/how-it-works/policies/default-policy.mdx b/docs/how-it-works/policies/default-policy.mdx index 4c9ab29ff8..1438856165 100644 --- a/docs/how-it-works/policies/default-policy.mdx +++ b/docs/how-it-works/policies/default-policy.mdx @@ -81,9 +81,10 @@ appear in the policy views. OpenShell can reject a replacement policy that removes filesystem paths, so keep the added paths when you replace the complete policy. -The runtime also grants the workload read-only access to the sandbox's TLS CA -certificates under `/run/openshell-supervisor-ca`. This grant is not saved in -the sandbox's policy. +The sandbox's TLS CA certificate and trust bundle appear read-only at +`/run/openshell-supervisor-ca/material` without a filesystem grant: the +runtime serves them from memory to opens and `stat` calls on those exact +paths. The directory itself does not exist. ### GPU Sandboxes diff --git a/e2e/python/test_sandbox_tls_trust.py b/e2e/python/test_sandbox_tls_trust.py new file mode 100644 index 0000000000..fbd24fa331 --- /dev/null +++ b/e2e/python/test_sandbox_tls_trust.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the supervisor CA trust files a default sandbox receives. + +The files exist only in the sandbox runtime, which answers opens and metadata +calls on their exact paths. This verifies what TLS clients actually do with a +CA path: the stat and access checks `requests` and shell guards make, the +plain opens OpenSSL (Python `ssl`, the `openssl` CLI) makes, and that +the loaded bundle contains the supervisor CA alongside the system CAs. +""" + +from __future__ import annotations + +import textwrap +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Callable + + from openshell import Sandbox + +_CA_DIR = "/run/openshell-supervisor-ca/material" + +_CHECK_TRUST_FILES = textwrap.dedent( + f""" + import json, os, ssl, stat, subprocess + + ca_dir = {_CA_DIR!r} + cert_path = os.environ["NODE_EXTRA_CA_CERTS"] + bundle_path = os.environ["SSL_CERT_FILE"] + assert cert_path == os.path.join(ca_dir, "ca.crt"), cert_path + assert bundle_path == os.path.join(ca_dir, "ca-bundle.crt"), bundle_path + for variable in ("REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", "GIT_SSL_CAINFO"): + assert os.environ[variable] == bundle_path, variable + + # What requests checks before it hands the path to OpenSSL. + for path in (cert_path, bundle_path): + assert os.path.exists(path), path + assert not os.path.isdir(path), path + info = os.stat(path) + assert stat.S_ISREG(info.st_mode) and info.st_size > 0, path + assert os.access(path, os.R_OK) and not os.access(path, os.W_OK), path + with open(path, "rb") as handle: + assert os.fstat(handle.fileno()).st_ino == info.st_ino, path + # The directory itself is not provided. + assert not os.path.exists(ca_dir), ca_dir + + with open(cert_path, "rb") as handle: + cert = handle.read() + with open(bundle_path, "rb") as handle: + bundle = handle.read() + assert cert.startswith(b"-----BEGIN CERTIFICATE-----"), cert[:40] + assert cert.strip() in bundle, "bundle must contain the supervisor CA" + assert len(bundle) > len(cert), "bundle must contain more than the supervisor CA" + + # Python ssl loads both through OpenSSL. + supervisor_der = ssl.PEM_cert_to_DER_cert(cert.decode()) + context = ssl.create_default_context(cafile=bundle_path) + loaded = context.get_ca_certs(binary_form=True) + assert supervisor_der in loaded, "bundle must load the supervisor CA" + assert len(loaded) > 1, "bundle must load the system CAs too" + assert ssl.create_default_context(cafile=cert_path).get_ca_certs(binary_form=True) == [supervisor_der] + + # requests takes REQUESTS_CA_BUNDLE from the environment, then requires + # the path to exist and not be a directory before handing it to OpenSSL. + import requests + url = "https://example.invalid/" + verify = requests.Session().merge_environment_settings(url, {{}}, None, None, None)["verify"] + assert verify == bundle_path, verify + class Conn: + pass + conn = Conn() + requests.adapters.HTTPAdapter().cert_verify(conn, url, verify, None) + assert conn.ca_certs == bundle_path, conn.ca_certs + + # The openssl CLI and a shell guard read the same paths. + verified = subprocess.run( + ["openssl", "verify", "-CAfile", bundle_path, cert_path], + capture_output=True, text=True, check=False, + ) + assert verified.returncode == 0, verified.stderr + guard = subprocess.run( + ["sh", "-c", 'test -f "$SSL_CERT_FILE" && test -r "$SSL_CERT_FILE" && ! test -w "$SSL_CERT_FILE"'], + check=False, + ) + assert guard.returncode == 0, "shell file test" + + # Read-only to the workload. + try: + open(bundle_path, "wb") + except PermissionError: + pass + else: + raise AssertionError("trust bundle must not be writable") + + print(json.dumps({{"bundle_cas": len(loaded), "bundle_bytes": len(bundle)}})) + """ +) + + +def test_default_sandbox_exposes_a_consistent_supervisor_trust_bundle( + sandbox: Callable[..., Sandbox], + run_python: Callable[[Sandbox, str], tuple[int, str, str]], +) -> None: + with sandbox(delete_on_exit=True) as sb: + exit_code, stdout, stderr = run_python(sb, _CHECK_TRUST_FILES) + assert exit_code == 0, f"stdout={stdout!r} stderr={stderr!r}" + assert '"bundle_cas"' in stdout, stdout diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 3f00bf27a3..9465fc39f7 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -376,8 +376,21 @@ Common findings: workload container has no provider-file volume or mount, so a missing mount is not a fault; debug it as an enforcement-probe or provider-attachment problem. +- The same interception serves `/run/openshell-supervisor-ca/material/ca.crt` + and `ca-bundle.crt`: opens and `stat`, `statx`, and `access` calls on exactly + those paths are answered from a sealed in-memory pair, and no file, mount, + or directory exists there (`ls` of the directory is `ENOENT` by design). A + sandbox start refused with `publish supervisor CA trust files: ...` carries + the cause after the prefix. A `supervisor CA ...` cause (empty, over the + trust file bound, or a bundle that does not contain the certificate) means + the supervisor sent an invalid pair; check the supervisor's TLS output and + system CA bundle rather than the workload. Any other cause is a preparation + failure in the workload container: `/proc/self/fd` not on procfs, + `memfd_create` or sealing refused, or a descriptor or memory limit reached. + Check the runtime's seccomp profile, procfs mount, and the container's file + descriptor and memory limits. - Sandbox startup reports an `isolation control contract` failure: inspect the supervisor and sandbox-runtime image versions and update them together. An older runtime or incompatible isolation capability set fails before workload startup. See the published Sandbox Runtimes documentation for trusted image override precedence. -- The sandbox fails its enforcement probe: inspect the sandbox log for the exact nested seccomp user-notification, task-memory, Landlock, loopback DNS, or socket-injection check that failed. A runtime may return `ENOSYS` for `process_vm_readv` and `process_vm_writev` while satisfying the production parent-to-workload-child task-memory probe through `/proc//mem`; only failure of both backends is fatal. Do not add capabilities or switch to an unconfined seccomp profile; use a runtime whose default profile permits the unprivileged probe. +- The sandbox fails its enforcement probe: inspect the sandbox log for the exact nested seccomp user-notification, task-memory, Landlock, loopback DNS, or socket-injection check that failed. A runtime may return `ENOSYS` for `process_vm_readv` while satisfying the production parent-to-workload-child task-memory read probe through `/proc//mem`; the write probe always uses `/proc//mem`, and a `cross-child memory write failed` error means the runtime's Yama, LSM, or procfs posture refuses it. Do not add capabilities or switch to an unconfined seccomp profile; use a runtime whose default profile permits the unprivileged probe. - A GPU sandbox fails because Docker reports no discovered NVIDIA CDI devices: verify `.DiscoveredDevices` contains entries such as `nvidia.com/gpu=all`, verify `/etc/cdi` or `/var/run/cdi` contains a generated NVIDIA spec, and check that `nvidia-cdi-refresh.service` and `nvidia-cdi-refresh.path` from NVIDIA Container Toolkit are enabled and healthy. The service is a one-shot unit, so `inactive (dead)` can be normal after a successful run; use `systemctl status` and `journalctl` to distinguish success from a skipped or failed refresh. Restart `nvidia-cdi-refresh.service` to regenerate missing or stale CDI specs, then restart or reload Docker and re-check `docker info`. - Docker or Podman rejects an explicit `cdi_devices` selector: use a CDI qualified name (`/=`), not a host-device path. Syntax validation accepts other vendors; it does not establish that the runtime has the device or that its GPU workload is supported. Consult the published GPU selection documentation for request requirements.