diff --git a/docs/security/cvm-boundaries.md b/docs/security/cvm-boundaries.md index b1bccd72e..b0a106eef 100644 --- a/docs/security/cvm-boundaries.md +++ b/docs/security/cvm-boundaries.md @@ -35,7 +35,7 @@ This is the main configuration file for the application in JSON format: | local_key_provider_enabled | 0.3.1 | boolean | Use a local key provider | | key_provider_id | 0.5.1 | string | Optional pin for the key provider identity (hex-encoded bytes). For `kms` this is the KMS CA public key; for `local` the sealing-provider MR. For `tpm` and `none` it must be an empty string — the TPM app-root public key is instance-specific and is not used as a provider id or measured as one. | | public_logs | 0.3.3 | boolean | Whether logs are publicly visible | -| public_sysinfo | 0.3.3 | boolean | Whether system info is public | +| public_sysinfo | 0.3.3 | boolean | Whether system info is public. Covers the guest dashboard and `/metrics`, including the `dstack_gpu_*` series (since 0.6.0), which expose each GPU's UUID and PCI bus address alongside utilization, memory, temperature and power. | | public_tcbinfo | 0.5.1 | boolean | Whether TCB info is public | | allowed_envs | 0.4.2 | array of string | List of allowed environment variable names | | no_instance_id | 0.4.2 | boolean | Disable instance ID generation | diff --git a/dstack/Cargo.lock b/dstack/Cargo.lock index 25c0e21e2..89e110e04 100644 --- a/dstack/Cargo.lock +++ b/dstack/Cargo.lock @@ -2002,6 +2002,7 @@ dependencies = [ "anyhow", "base64 0.22.1", "bollard", + "cached-cell", "cc-eventlog", "cert-client", "chrono", @@ -2023,6 +2024,7 @@ dependencies = [ "libc", "listenfd", "load_config", + "lspci", "nvattest", "or-panic", "ra-rpc", @@ -2291,11 +2293,13 @@ dependencies = [ "ez-hash", "fs-err", "getrandom 0.3.4", + "guest-api", "hex", "hex_fmt", "host-api", "k256", "libc", + "lspci", "luks2", "nvattest", "nvml-wrapper", @@ -4291,6 +4295,7 @@ version = "0.6.0" dependencies = [ "anyhow", "insta", + "tempfile", ] [[package]] diff --git a/dstack/dstack-util/Cargo.toml b/dstack/dstack-util/Cargo.toml index ef4066831..a38f02f41 100644 --- a/dstack/dstack-util/Cargo.toml +++ b/dstack/dstack-util/Cargo.toml @@ -42,6 +42,7 @@ tpm-attest.workspace = true tpm2.workspace = true tpm-qvl = { workspace = true, features = ["crl-download"] } host-api = { workspace = true, features = ["client"] } +guest-api.workspace = true cmd_lib.workspace = true toml.workspace = true dcap-qvl.workspace = true @@ -60,6 +61,7 @@ sodiumbox.workspace = true libc.workspace = true luks2.workspace = true nvml-wrapper.workspace = true +lspci.workspace = true scopeguard.workspace = true tempfile.workspace = true ez-hash.workspace = true diff --git a/dstack/dstack-util/src/gpu_info.rs b/dstack/dstack-util/src/gpu_info.rs new file mode 100644 index 000000000..ea634b16d --- /dev/null +++ b/dstack/dstack-util/src/gpu_info.rs @@ -0,0 +1,282 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! One-shot NVML sampler behind `dstack-util gpu-info`. +//! +//! NVML calls cannot be cancelled: a driver that wedges during a GPU reset or a +//! CC handshake blocks the calling thread until it decides to return. Running +//! the sample in a short-lived process lets `dstack-guest-agent` kill it and +//! move on, and keeps `libnvidia-ml` out of the long-lived agent entirely. +//! +//! One process per sample also means `Nvml::init()` runs fresh every time, so a +//! driver that loads after the agent started is picked up on the next sample +//! instead of being cached as "no GPU" for the agent's lifetime. +//! +//! This command owns collection only. Caching, presence gating and timeouts +//! belong to the caller. + +use std::collections::HashSet; + +use anyhow::Result; +use guest_api::{GpuDevice, GpuInfoResponse}; +use nvml_wrapper::enum_wrappers::device::TemperatureSensor; +use nvml_wrapper::error::NvmlError; +use nvml_wrapper::{Device, Nvml}; +use tracing::{debug, warn}; + +/// Per-run state for error log deduplication. +/// +/// A single run touches each (device, field) pair once, so this only collapses +/// the repeats that a multi-GPU box would otherwise produce for the same +/// unsupported counter on every card. +#[derive(Default)] +struct Sampler { + warned: HashSet<(u32, &'static str)>, +} + +/// Samples every NVIDIA GPU visible to NVML and prints the result as one JSON +/// object on stdout. +/// +/// Exits zero even when NVML is unavailable: "no driver" is a result the caller +/// needs to record, not a failure of this command. A non-zero exit is reserved +/// for not being able to produce a result at all. +pub fn cmd_gpu_info() -> Result<()> { + let response = collect(); + serde_json::to_writer(std::io::stdout(), &response)?; + println!(); + Ok(()) +} + +fn collect() -> GpuInfoResponse { + let nvml = match Nvml::init() { + Ok(nvml) => nvml, + Err(err) => { + // Expected on any guest without an NVIDIA driver. The caller gates + // on PCI presence, so reaching here means a card is attached but + // the library is missing -- worth a warning. + warn!("failed to initialize NVML: {err}"); + return unavailable(format!("failed to initialize NVML: {err}")); + } + }; + sample(&mut Sampler::default(), &nvml) +} + +fn unavailable(error: impl Into) -> GpuInfoResponse { + GpuInfoResponse { + gpus: vec![], + error: error.into(), + cc_ready: None, + cc_enabled: None, + sample_age_ms: None, + } +} + +fn sample(sampler: &mut Sampler, nvml: &Nvml) -> GpuInfoResponse { + let count = match nvml.device_count() { + Ok(count) => count, + Err(e) => { + let error = format!("failed to get NVML GPU count: {e}"); + warn!("{error}"); + return unavailable(error); + } + }; + + // Both CC queries are system-wide despite hanging off `Device`, so ask once + // through device 0 rather than once per card. + let (cc_ready, cc_enabled) = query_cc_state(sampler, nvml, count); + GpuInfoResponse { + gpus: (0..count) + .map(|index| collect_device(sampler, nvml, index)) + .collect(), + error: String::new(), + cc_ready, + cc_enabled, + sample_age_ms: None, + } +} + +/// Reads the system-wide confidential-computing state. +/// +/// `get_confidential_compute_state` is `nvmlSystemGetConfComputeGpusReadyState` +/// and `is_cc_enabled` is `nvmlSystemGetConfComputeSettings`; neither uses the +/// device handle it is called on. Any device will do, so device 0 is used. +fn query_cc_state(sampler: &mut Sampler, nvml: &Nvml, count: u32) -> (Option, Option) { + if count == 0 { + return (None, None); + } + let device = match nvml.device_by_index(0) { + Ok(device) => device, + Err(e) => { + log_query_error(sampler, 0, "cc_state", &e); + return (None, None); + } + }; + let ready = match device.get_confidential_compute_state() { + Ok(ready) => Some(ready), + Err(e) => { + log_query_error(sampler, 0, "cc_ready", &e); + None + } + }; + let enabled = match device.is_cc_enabled() { + Ok(enabled) => Some(enabled), + Err(e) => { + log_query_error(sampler, 0, "cc_enabled", &e); + None + } + }; + (ready, enabled) +} + +fn collect_device(sampler: &mut Sampler, nvml: &Nvml, index: u32) -> GpuDevice { + match nvml.device_by_index(index) { + Ok(device) => collect_device_fields(sampler, index, &device), + Err(e) => { + log_query_error(sampler, index, "device", &e); + GpuDevice { + index, + errors: vec![format!("device: {e}")], + ..Default::default() + } + } + } +} + +fn collect_device_fields(sampler: &mut Sampler, index: u32, device: &Device<'_>) -> GpuDevice { + let mut errors = Vec::new(); + + let uuid = match device.uuid() { + Ok(uuid) => uuid, + Err(e) => { + push_error(sampler, index, "uuid", e, &mut errors); + String::new() + } + }; + let pci_bus_id = match device.pci_info() { + Ok(pci) => pci.bus_id, + Err(e) => { + push_error(sampler, index, "pci_bus_id", e, &mut errors); + String::new() + } + }; + + let (utilization_gpu, utilization_memory) = match device.utilization_rates() { + Ok(util) => (Some(util.gpu), Some(util.memory)), + Err(e) => { + push_error(sampler, index, "utilization", e, &mut errors); + (None, None) + } + }; + + let (memory_total_bytes, memory_used_bytes, memory_free_bytes) = match device.memory_info() { + Ok(mem) => (Some(mem.total), Some(mem.used), Some(mem.free)), + Err(e) => { + push_error(sampler, index, "memory", e, &mut errors); + (None, None, None) + } + }; + + let temperature_c = match device.temperature(TemperatureSensor::Gpu) { + Ok(temp) => Some(temp), + Err(e) => { + push_error(sampler, index, "temperature", e, &mut errors); + None + } + }; + + let power_usage_mw = match device.power_usage() { + Ok(power) => Some(power), + Err(e) => { + push_error(sampler, index, "power", e, &mut errors); + None + } + }; + + GpuDevice { + index, + uuid, + pci_bus_id, + utilization_gpu, + utilization_memory, + memory_total_bytes, + memory_used_bytes, + memory_free_bytes, + temperature_c, + power_usage_mw, + errors, + } +} + +fn push_error( + sampler: &mut Sampler, + index: u32, + field: &'static str, + err: NvmlError, + errors: &mut Vec, +) { + errors.push(format!("{field}: {err}")); + log_query_error(sampler, index, field, &err); +} + +fn log_query_error(sampler: &mut Sampler, index: u32, field: &'static str, err: &NvmlError) { + let first = sampler.warned.insert((index, field)); + if matches!(err, NvmlError::NotSupported) { + // CC mode disables some counters by design; that is not an incident. + debug!("GPU {index} {field} not supported: {err}"); + } else if first { + warn!("failed to query GPU {index} {field}: {err}"); + } else { + debug!("failed to query GPU {index} {field}: {err}"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// NVML may or may not be present on the machine running this test. + /// Unavailability must be an error with no devices, not a silent empty + /// success (which would mean "NVML worked, zero GPUs"). + #[test] + fn collect_reports_nvml_unavailability_without_panicking() { + let info = collect(); + if info.error.is_empty() { + return; + } + assert!(info.gpus.is_empty()); + assert_eq!(info.cc_ready, None); + assert_eq!(info.cc_enabled, None); + } + + /// The guest agent parses stdout as one JSON object. A response must + /// survive that round trip with `None` distinct from `Some(0)`. + #[test] + fn response_round_trips_through_json() { + let original = GpuInfoResponse { + gpus: vec![GpuDevice { + index: 0, + uuid: "GPU-abc".into(), + pci_bus_id: "00000000:01:00.0".into(), + utilization_gpu: Some(0), + utilization_memory: None, + memory_total_bytes: Some(8 << 30), + memory_used_bytes: Some(0), + memory_free_bytes: None, + temperature_c: Some(40), + power_usage_mw: None, + errors: vec!["power: not supported".into()], + }], + error: String::new(), + cc_ready: Some(true), + cc_enabled: Some(true), + sample_age_ms: None, + }; + let encoded = serde_json::to_string(&original).expect("encode"); + assert!(!encoded.contains('\n'), "one JSON line per sample"); + let decoded: GpuInfoResponse = serde_json::from_str(&encoded).expect("decode"); + assert_eq!(decoded, original); + assert_eq!(decoded.gpus[0].utilization_gpu, Some(0)); + assert_eq!(decoded.gpus[0].utilization_memory, None); + } +} diff --git a/dstack/dstack-util/src/main.rs b/dstack/dstack-util/src/main.rs index d4c56a93d..10373f94d 100644 --- a/dstack/dstack-util/src/main.rs +++ b/dstack/dstack-util/src/main.rs @@ -32,6 +32,7 @@ use utils::AppKeys; mod crypto; mod docker_compose; mod gateway_checker; +mod gpu_info; mod host_api; mod host_shared; mod parse_env_file; @@ -101,6 +102,8 @@ enum Commands { Decrypt(DecryptArgs), /// Encrypt data for an app using its KMS-provided environment encryption key Encrypt(EncryptArgs), + /// Sample NVIDIA GPU telemetry through NVML and print it as JSON + GpuInfo, } #[derive(Parser)] @@ -1567,14 +1570,20 @@ async fn cmd_tpm_verify(args: TpmVerifyArgs) -> Result<()> { #[tokio::main] async fn main() -> Result<()> { + let cli = Cli::parse(); { use tracing_subscriber::{fmt, EnvFilter}; let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")); - fmt().with_env_filter(filter).with_ansi(false).init(); + let builder = fmt().with_env_filter(filter).with_ansi(false); + // `gpu-info` writes a machine-readable JSON document to stdout, so its + // logs must go to stderr or an NVML warning would corrupt the output. + // Every other subcommand keeps the historical stdout behaviour. + match cli.command { + Commands::GpuInfo => builder.with_writer(std::io::stderr).init(), + _ => builder.init(), + } } - let cli = Cli::parse(); - match cli.command { Commands::Quote => cmd_quote()?, Commands::Eventlog => cmd_eventlog()?, @@ -1655,6 +1664,9 @@ async fn main() -> Result<()> { Commands::Encrypt(args) => { cmd_encrypt(args).await?; } + Commands::GpuInfo => { + gpu_info::cmd_gpu_info()?; + } } Ok(()) diff --git a/dstack/dstack-util/src/system_setup.rs b/dstack/dstack-util/src/system_setup.rs index 46c9d6a5f..66839b4fa 100644 --- a/dstack/dstack-util/src/system_setup.rs +++ b/dstack/dstack-util/src/system_setup.rs @@ -1361,11 +1361,7 @@ mod gpu { /// Bound Rego evaluation so a runaway application policy cannot hang boot. const POLICY_TIMEOUT: Duration = Duration::from_secs(10); - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) struct GpuInventory { - pub(super) total: u32, - pub(super) nvidia: u32, - } + use lspci::sysfs::GpuInventory; #[derive(Debug, Serialize)] struct GpuAttestationEvent { @@ -1457,33 +1453,11 @@ mod gpu { /// the NVIDIA driver did not bind cannot be hidden from the gate. Reading /// the inventory is fail-closed: a mixed NVIDIA/non-NVIDIA set must not be /// represented by an attestation result for only the NVIDIA subset. + /// + /// The scan itself lives in `lspci::sysfs` because the guest agent's GPU + /// telemetry gate needs the same answer with a different failure policy. pub(super) fn gpu_inventory() -> Result { - gpu_inventory_at(Path::new("/sys/bus/pci/devices")) - } - - fn gpu_inventory_at(devices_path: &Path) -> Result { - let entries = fs::read_dir(devices_path).context("failed to enumerate PCI devices")?; - let mut inventory = GpuInventory { - total: 0, - nvidia: 0, - }; - for entry in entries { - let device = entry.context("failed to read PCI device entry")?; - let class_path = device.path().join("class"); - let class = fs::read_to_string(&class_path) - .with_context(|| format!("failed to read {}", class_path.display()))?; - if !matches!(class.trim().get(..6), Some("0x0300") | Some("0x0302")) { - continue; - } - inventory.total += 1; - let vendor_path = device.path().join("vendor"); - let vendor = fs::read_to_string(&vendor_path) - .with_context(|| format!("failed to read {}", vendor_path.display()))?; - if vendor.trim() == "0x10de" { - inventory.nvidia += 1; - } - } - Ok(inventory) + lspci::sysfs::gpu_inventory() } pub(super) fn nvidia_gpu_count(inventory: GpuInventory) -> Result { @@ -1726,13 +1700,6 @@ mod gpu { mod tests { use super::*; - fn add_pci_device(root: &Path, name: &str, vendor: &str, class: &str) { - let device = root.join(name); - fs::create_dir_all(&device).unwrap(); - fs::write(device.join("vendor"), vendor).unwrap(); - fs::write(device.join("class"), class).unwrap(); - } - fn nvattest_output(nonce: &str, claims: usize) -> Vec { let claims = (0..claims) .map(|_| { @@ -1801,21 +1768,6 @@ mod gpu { ); } - #[test] - fn inventory_counts_nvidia_and_non_nvidia_gpus() { - let root = tempfile::tempdir().unwrap(); - add_pci_device(root.path(), "0000:01:00.0", "0x10de\n", "0x030200\n"); - add_pci_device(root.path(), "0000:02:00.0", "0x1234\n", "0x030000\n"); - add_pci_device(root.path(), "0000:03:00.0", "0x1af4\n", "0x020000\n"); - assert_eq!( - gpu_inventory_at(root.path()).unwrap(), - GpuInventory { - total: 2, - nvidia: 1 - } - ); - } - #[test] fn gpu_count_rejects_non_nvidia_gpus() { let mixed = GpuInventory { diff --git a/dstack/guest-agent/Cargo.toml b/dstack/guest-agent/Cargo.toml index ffdfa269e..9b028a6cc 100644 --- a/dstack/guest-agent/Cargo.toml +++ b/dstack/guest-agent/Cargo.toml @@ -58,6 +58,8 @@ or-panic.workspace = true cc-eventlog.workspace = true listenfd.workspace = true libc.workspace = true +cached-cell.workspace = true +lspci.workspace = true [dev-dependencies] # The test-only mock platform builds attestations from a fixture, which needs diff --git a/dstack/guest-agent/src/gpu_info.rs b/dstack/guest-agent/src/gpu_info.rs new file mode 100644 index 000000000..6e62a9641 --- /dev/null +++ b/dstack/guest-agent/src/gpu_info.rs @@ -0,0 +1,478 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! GPU telemetry, sampled out of process and cached. +//! +//! Three properties drive the shape of this module: +//! +//! 1. **A guest without an NVIDIA card pays nothing.** PCI topology is fixed +//! for a CVM's lifetime -- the VMM assigns GPUs through VFIO when QEMU +//! starts and never hot-plugs -- so the scan runs once and every later +//! request is an atomic load returning a constant. +//! 2. **NVML never runs in this process.** `dstack-util gpu-info` samples in a +//! short-lived child that can be killed when a driver call wedges. Nothing +//! stays resident between samples, and each sample re-initializes NVML, so a +//! driver that loads late is picked up instead of being cached as "no GPU". +//! 3. **A request never waits on the GPU.** Callers get the last known snapshot +//! with its age attached and a refresh is kicked off behind them. Returning +//! a placeholder on expiry would mean a Prometheus scrape slower than the +//! TTL -- which is every realistic scrape interval -- never sees a single +//! GPU series. +//! +//! The cache is not here because sampling is slow. One `dstack-util gpu-info` +//! run, fork to exit, measured 94-99 ms on a single H200 in CC mode. It is here +//! because that number describes the healthy path only, and three things do not +//! follow from it: +//! +//! - `/metrics` also carries CPU, memory, disk and container state. Sampling +//! inline would put all of it behind a driver call that can wedge for +//! [`SAMPLE_TIMEOUT`], well past a default Prometheus scrape timeout, so one +//! stuck card would erase every metric this CVM reports rather than just the +//! GPU ones. +//! - `/metrics`, the dashboard and the `GpuInfo` RPC can arrive together, and +//! each would otherwise fork its own collector to repeat the same `dlopen` +//! and `nvmlInit_v2`. [`REFRESH_LOCK`] collapses a burst into one sample. +//! - Enumeration cost grows with card count, and only one card has been +//! measured. +//! +//! Refreshing is lazy: nothing resamples unless someone asks. A served snapshot +//! is therefore roughly one scrape interval old, not [`SAMPLE_TTL`] old -- the +//! TTL decides when a request triggers the next sample, not how fresh the +//! answer is. `sample_age_ms`, and `dstack_gpu_sample_age_seconds` on +//! `/metrics`, carry that age so a consumer can correct for it. A background +//! ticker would cap the age at the TTL instead, at the price of a permanent +//! ~2% of a core on every GPU guest whether or not anyone is watching, to +//! publish a number the consumer can already derive. + +use std::path::Path; +use std::process::Stdio; +use std::sync::{LazyLock, OnceLock}; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; +use cached_cell::TtlCell; +use guest_api::GpuInfoResponse; +#[cfg(test)] +use or_panic::ResultOrPanic; +use tokio::process::Command; +use tokio::sync::Mutex; +use tokio::time::timeout; +use tracing::{debug, warn}; + +/// How long a successful snapshot is served before a request triggers a +/// refresh. Older snapshots are still served, just with a refresh started +/// behind them, so this bounds when resampling starts and not how old a served +/// sample can be. +const SAMPLE_TTL: Duration = Duration::from_secs(5); +/// Backoff after a failed sample. +/// +/// A collector that hangs burns [`SAMPLE_TIMEOUT`] and is then killed. Retrying +/// that on the success cadence would keep a guest whose driver has wedged in a +/// near-continuous spawn-and-kill loop, which is both useless and the state +/// most likely to leave a process stuck in an uninterruptible driver call. +/// Long enough to stop hammering, short enough that a recovered driver is +/// picked up while an operator is still looking at the dashboard. +const FAILURE_BACKOFF: Duration = Duration::from_secs(30); +/// Upper bound on one `dstack-util gpu-info` run. Finite because the whole +/// point of the child process is that a wedged driver can be abandoned. +/// +/// A measured run on a single H200 in CC mode takes 94-99 ms, so this is two +/// orders of magnitude of headroom. The margin stays until a multi-GPU CVM has +/// been measured: `nvmlInit_v2` enumerates every card, and trading the untested +/// case for faster recovery in the tested one is the wrong direction. +const SAMPLE_TIMEOUT: Duration = Duration::from_secs(10); + +/// The collector, installed into the rootfs alongside this agent. +const DSTACK_UTIL: &str = "/usr/bin/dstack-util"; +/// Set by tests to point at a stub collector. +#[cfg(test)] +static COLLECTOR_OVERRIDE: std::sync::RwLock> = std::sync::RwLock::new(None); + +/// Holds the last outcome, successful or not. The cell's own TTL is unused: +/// serving is always allowed and [`refresh_interval`] decides when to resample. +static SNAPSHOT: LazyLock> = LazyLock::new(|| TtlCell::new(SAMPLE_TTL)); +/// Serializes refreshes so a burst of scrapes spawns one collector, not N. +static REFRESH_LOCK: LazyLock> = LazyLock::new(|| Mutex::new(())); + +/// Why this guest cannot produce GPU telemetry, if it cannot. +enum Gate { + /// No NVIDIA device on the PCI bus. Nothing to report, ever. + NoGpu, + /// A card is present but the kernel module is not loaded yet. + NoDriver, + /// Sampling is worth attempting. + Sample, +} + +fn gate() -> Gate { + if !nvidia_on_pci() { + return Gate::NoGpu; + } + // Unlike PCI presence this is not fixed: the module can load after the + // agent starts, so it is re-checked every time. It is one `stat`. + if !Path::new("/sys/module/nvidia").exists() { + return Gate::NoDriver; + } + Gate::Sample +} + +/// True when an NVIDIA display device is attached to the PCI bus. +/// +/// Cached for the process lifetime. A CVM's GPUs are fixed at launch by the +/// VMM's VFIO assignment; there is no hot-plug path that could change this +/// answer, and paying a directory scan on every scrape of every GPU-less guest +/// is exactly the cost this gate exists to avoid. +/// +/// Fails open to "no GPU" where the boot attestation gate in `dstack-util` +/// fails closed on the same inventory: refusing to report telemetry is the +/// safe direction here, and spawning a collector forever on a guest whose +/// sysfs cannot be read is not. +fn nvidia_on_pci() -> bool { + static PRESENT: OnceLock = OnceLock::new(); + *PRESENT.get_or_init(|| match lspci::sysfs::gpu_inventory() { + Ok(inventory) => inventory.has_nvidia(), + Err(error) => { + warn!("failed to scan PCI for GPUs, assuming none: {error:#}"); + false + } + }) +} + +/// Returns the best answer available without ever blocking. +/// +/// Serves the last snapshot whatever its age, annotated with `sample_age_ms`, +/// and starts a refresh behind the caller when that snapshot has aged past the +/// TTL. Used by `/metrics` and the dashboard, where a wedged GPU must not +/// delay unrelated guest data. Only the very first call on a GPU guest returns +/// "not sampled yet". +pub(crate) fn gpu_info() -> GpuInfoResponse { + match gate() { + Gate::NoGpu => return no_gpus(), + Gate::NoDriver => return unavailable("NVIDIA driver is not loaded"), + Gate::Sample => {} + } + + let cached = SNAPSHOT.get_allow_stale().ok(); + if is_due(cached.as_ref()) { + tokio::spawn(refresh_if_free()); + } + serve(cached) +} + +/// Like [`gpu_info`], but waits for a first sample when the cache is cold. +/// +/// The `GpuInfo` RPC is a direct question from an operator or the control +/// plane, so "ask again in five seconds" is a worse answer than a bounded +/// wait. The wait is bounded twice: by [`SAMPLE_TIMEOUT`] here and by the RPC +/// timeout at the call site. +pub(crate) async fn gpu_info_awaited() -> GpuInfoResponse { + match gate() { + Gate::NoGpu => return no_gpus(), + Gate::NoDriver => return unavailable("NVIDIA driver is not loaded"), + Gate::Sample => {} + } + + if SNAPSHOT.get_allow_stale().is_err() { + // Cold cache. Queue behind any in-flight sample rather than reporting + // nothing; whoever wins the lock fills the cache for both. + let _guard = REFRESH_LOCK.lock().await; + if SNAPSHOT.get_allow_stale().is_err() { + sample_into_cache().await; + } + } + gpu_info() +} + +/// How long this outcome is kept before resampling. +fn refresh_interval(response: &GpuInfoResponse) -> Duration { + if response.error.is_empty() { + SAMPLE_TTL + } else { + FAILURE_BACKOFF + } +} + +/// True when the cached outcome has outlived its refresh interval. +fn is_due(cached: Option<&cached_cell::Snapshot>) -> bool { + cached.is_none_or(|snapshot| snapshot.age() >= refresh_interval(snapshot.value())) +} + +fn serve(cached: Option>) -> GpuInfoResponse { + match cached { + Some(snapshot) => { + let mut response = snapshot.value().clone(); + response.sample_age_ms = Some(snapshot.age().as_millis() as u64); + response + } + None => unavailable("GPU sample is not available yet"), + } +} + +/// Runs one collection unless another one is already in flight. +/// +/// Dropping the refresh when the lock is held is deliberate: a burst of scrapes +/// must spawn one collector, not one per scrape, and the waiting callers are +/// already being served the previous snapshot. +async fn refresh_if_free() { + let Ok(_guard) = REFRESH_LOCK.try_lock() else { + debug!("GPU sample already in progress, skipping refresh"); + return; + }; + // Another task may have refreshed between the staleness check and here. + if !is_due(SNAPSHOT.get_allow_stale().ok().as_ref()) { + return; + } + sample_into_cache().await; +} + +/// Collects once and stores the outcome, success or failure. +/// +/// Failures are cached like successes so a guest whose driver is broken reports +/// the reason instead of an empty device list, and so a hard-failing collector +/// is not re-spawned on every single scrape. +/// +/// Caller must hold [`REFRESH_LOCK`]. +async fn sample_into_cache() { + match timeout(SAMPLE_TIMEOUT, collect()).await { + Ok(Ok(response)) => { + SNAPSHOT.set(response); + } + Ok(Err(error)) => { + warn!("failed to sample GPU telemetry: {error:#}"); + SNAPSHOT.set(unavailable(format!("GPU sampling failed: {error:#}"))); + } + Err(_) => { + warn!("GPU sampling timed out after {SAMPLE_TIMEOUT:?}"); + SNAPSHOT.set(unavailable("GPU sampling timed out")); + } + } +} + +/// Spawns `dstack-util gpu-info` and parses its stdout. +/// +/// `kill_on_drop` matters: the timeout above drops this future, and a wedged +/// NVML call inside the child must not outlive it. +async fn collect() -> Result { + let collector = collector_path(); + if !Path::new(&collector).exists() { + bail!("{collector} is not installed"); + } + let output = Command::new(&collector) + .arg("gpu-info") + .stdin(Stdio::null()) + .kill_on_drop(true) + .output() + .await + .with_context(|| format!("failed to run {collector} gpu-info"))?; + // The collector logs to stderr and reserves stdout for the document, so + // anything here is diagnostics worth keeping rather than protocol noise. + if !output.stderr.is_empty() { + warn!( + "gpu-info collector: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + if !output.status.success() { + bail!("gpu-info collector exited with {}", output.status); + } + serde_json::from_slice(&output.stdout).context("invalid gpu-info output") +} + +fn collector_path() -> String { + #[cfg(test)] + if let Some(path) = COLLECTOR_OVERRIDE + .read() + .or_panic("collector override poisoned") + .clone() + { + return path; + } + DSTACK_UTIL.to_string() +} + +/// The guest has no NVIDIA hardware. Empty devices with no error is the +/// documented encoding for "the collector ran and found nothing". +fn no_gpus() -> GpuInfoResponse { + GpuInfoResponse::default() +} + +fn unavailable(error: impl Into) -> GpuInfoResponse { + GpuInfoResponse { + error: error.into(), + ..Default::default() + } +} + +/// Test-only hooks. Production callers go through [`gpu_info`]. +#[cfg(test)] +mod test_support { + use super::*; + + /// Serializes the tests that reach into process-global state. + /// + /// [`SNAPSHOT`], [`REFRESH_LOCK`] and [`COLLECTOR_OVERRIDE`] are one per + /// process while the harness runs tests on parallel threads, so without + /// this two tests race over the same cache entry and the same collector + /// path -- one pointing at a stub that sleeps, the other at a path that + /// does not exist. + static EXCLUSION: LazyLock> = LazyLock::new(|| Mutex::new(())); + + /// Held for the whole test body, not just the mutation, and restores the + /// collector override on the way out so a stub cannot leak into a test + /// that expects the real path. + pub(super) struct Exclusive { + _exclusion: tokio::sync::MutexGuard<'static, ()>, + _refresh: tokio::sync::MutexGuard<'static, ()>, + } + + impl Drop for Exclusive { + fn drop(&mut self) { + *COLLECTOR_OVERRIDE + .write() + .or_panic("collector override poisoned") = None; + } + } + + pub(super) async fn exclusive() -> Exclusive { + let exclusion = EXCLUSION.lock().await; + // Waits rather than try_lock: a refresh spawned by an earlier test may + // still be finishing, and failing a test for losing that race tests + // the harness rather than the code. + let refresh = REFRESH_LOCK.lock().await; + Exclusive { + _exclusion: exclusion, + _refresh: refresh, + } + } + + pub(super) fn set_snapshot(response: GpuInfoResponse) { + SNAPSHOT.set(response); + } + + pub(super) fn set_collector(path: &str) { + *COLLECTOR_OVERRIDE + .write() + .or_panic("collector override poisoned") = Some(path.to_string()); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use guest_api::GpuDevice; + use std::os::unix::fs::PermissionsExt; + + fn sample_response() -> GpuInfoResponse { + GpuInfoResponse { + gpus: vec![GpuDevice { + index: 0, + uuid: "GPU-abc".into(), + ..Default::default() + }], + ..Default::default() + } + } + + /// A guest with no NVIDIA device must not scan, spawn, or cache anything. + /// The distinction that matters downstream is "no GPUs" (empty, no error) + /// versus "could not tell" (error set). + #[test] + fn a_guest_without_a_card_reports_no_gpus_rather_than_an_error() { + let response = no_gpus(); + assert!(response.gpus.is_empty()); + assert!(response.error.is_empty()); + assert_eq!(response.sample_age_ms, None); + } + + /// A failed sample must not be retried on the success cadence. A collector + /// that burns the whole timeout and gets killed would otherwise be respawned + /// on every scrape, which is the state most likely to strand a process in an + /// uninterruptible driver call. + #[test] + fn a_failed_sample_backs_off_further_than_a_successful_one() { + assert_eq!(refresh_interval(&sample_response()), SAMPLE_TTL); + assert_eq!( + refresh_interval(&unavailable("GPU sampling timed out")), + FAILURE_BACKOFF + ); + assert!(FAILURE_BACKOFF > SAMPLE_TTL); + } + + /// An empty cache is always due; that is the cold-start path. + #[test] + fn an_empty_cache_is_due() { + assert!(is_due(None)); + } + + /// The bug this design exists to prevent: a scrape interval longer than the + /// TTL must still see the GPU series. Serving a placeholder on expiry meant + /// `/metrics` reported zero GPUs forever at Prometheus' default 15s. + #[tokio::test] + async fn a_snapshot_older_than_the_ttl_is_still_served() { + // Hold the globals so no refresh, and no other test, can overwrite the + // snapshot mid-assertion on a machine that does have a GPU. + let _guard = test_support::exclusive().await; + test_support::set_snapshot(sample_response()); + + let served = serve(SNAPSHOT.get_allow_stale().ok()); + + assert!( + served.error.is_empty(), + "stale data must not become an error" + ); + assert_eq!(served.gpus.len(), 1); + assert!(served.sample_age_ms.is_some(), "age must be reported"); + } + + /// The reason sampling lives in a child process: when a driver call wedges, + /// the timeout must actually reclaim it. Nothing may be left running + /// between samples. + #[tokio::test] + async fn a_collector_that_hangs_is_killed_when_the_sample_times_out() { + let dir = tempfile::tempdir().expect("tempdir"); + let script = dir.path().join("stub-collector"); + let pid_file = dir.path().join("pid"); + std::fs::write( + &script, + format!("#!/bin/sh\necho $$ > {}\nsleep 60\n", pid_file.display()), + ) + .expect("write stub"); + std::fs::set_permissions(&script, std::fs::Permissions::from_mode(0o755)) + .expect("chmod stub"); + + let _guard = test_support::exclusive().await; + test_support::set_collector(&script.to_string_lossy()); + + let outcome = timeout(Duration::from_millis(500), collect()).await; + assert!(outcome.is_err(), "the stub sleeps far past the timeout"); + + let pid: i32 = std::fs::read_to_string(&pid_file) + .expect("stub recorded its pid") + .trim() + .parse() + .expect("pid is a number"); + + // SIGKILL is asynchronous; give the kernel a moment to reap. + for _ in 0..50 { + if !Path::new(&format!("/proc/{pid}")).exists() { + return; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + panic!("collector {pid} survived the sample timeout"); + } + + /// A collector that is not installed must degrade to a recorded error, not + /// a panic and not a silent "no GPUs". + #[tokio::test] + async fn a_missing_collector_is_reported_as_an_error() { + let _guard = test_support::exclusive().await; + test_support::set_collector("/nonexistent/dstack-util"); + let error = collect().await.expect_err("must fail"); + assert!( + error.to_string().contains("not installed"), + "unexpected error: {error:#}" + ); + } +} diff --git a/dstack/guest-agent/src/guest_api_service.rs b/dstack/guest-agent/src/guest_api_service.rs index 22f64ada5..cf3e96163 100644 --- a/dstack/guest-agent/src/guest_api_service.rs +++ b/dstack/guest-agent/src/guest_api_service.rs @@ -12,8 +12,8 @@ use dstack_types::SysConfig; use fs_err as fs; use guest_api::{ guest_api_server::{GuestApiRpc, GuestApiServer}, - Container, DiskInfo, Gateway, GuestInfo, Interface, IpAddress, ListContainersResponse, - NetworkInformation, SystemInfo, + Container, DiskInfo, Gateway, GpuInfoResponse, GuestInfo, Interface, IpAddress, + ListContainersResponse, NetworkInformation, SystemInfo, }; use host_api::Notification; use ra_rpc::{CallContext, RpcCall}; @@ -27,6 +27,7 @@ const DOCKER_API_TIMEOUT: Duration = Duration::from_secs(15); const SHUTDOWN_NOTIFY_TIMEOUT: Duration = Duration::from_secs(5); const POWEROFF_COMMAND_TIMEOUT: Duration = Duration::from_secs(10); const WG_COMMAND_TIMEOUT: Duration = Duration::from_secs(5); +const GPU_INFO_TIMEOUT: Duration = Duration::from_secs(15); pub struct GuestApiHandler { state: AppState, @@ -101,6 +102,12 @@ impl GuestApiRpc for GuestApiHandler { .context("SysInfo worker failed") } + async fn gpu_info(self) -> Result { + timeout(GPU_INFO_TIMEOUT, crate::gpu_info::gpu_info_awaited()) + .await + .context("GpuInfo request timed out") + } + async fn list_containers(self) -> Result { timeout(GUEST_API_HANDLER_TIMEOUT, list_containers()) .await diff --git a/dstack/guest-agent/src/http_routes.rs b/dstack/guest-agent/src/http_routes.rs index d95f81fb7..3d0f662f4 100644 --- a/dstack/guest-agent/src/http_routes.rs +++ b/dstack/guest-agent/src/http_routes.rs @@ -58,6 +58,13 @@ async fn index(state: &State) -> Result, String> { let handler = GuestApiHandler::construct(context) .map_err(|e| format!("Failed to construct RPC handler: {}", e))?; let system_info = handler.sys_info().await.unwrap_or_default(); + // Only sampled when the page will actually render it. Asking otherwise + // would spawn a collector on guests that keep their sysinfo private. + let gpu_info = if public_sysinfo { + crate::gpu_info::gpu_info() + } else { + Default::default() + }; let containers = list_containers().await.unwrap_or_default().containers; let model = crate::models::Dashboard { @@ -74,6 +81,7 @@ async fn index(state: &State) -> Result, String> { public_tcbinfo, cloud_vendor, cloud_product, + gpu_info, }; match model.render() { Ok(html) => Ok(RawHtml(html)), @@ -93,7 +101,11 @@ async fn metrics(state: &State) -> Result { .map_err(|e| format!("Failed to construct RPC handler: {}", e))?; let system_info = handler.sys_info().await.unwrap_or_default(); - let model = crate::models::Metrics { system_info }; + let gpu_info = crate::gpu_info::gpu_info(); + let model = crate::models::Metrics { + system_info, + gpu_info, + }; match model.render() { Ok(body) => Ok(body), Err(err) => Err(format!("Failed to render template: {err}")), diff --git a/dstack/guest-agent/src/lib.rs b/dstack/guest-agent/src/lib.rs index 0aee897be..7c70e192a 100644 --- a/dstack/guest-agent/src/lib.rs +++ b/dstack/guest-agent/src/lib.rs @@ -9,6 +9,7 @@ pub mod backend; pub mod config; mod container_health; mod gpu_attest; +mod gpu_info; mod guest_api_service; mod health; mod http_routes; diff --git a/dstack/guest-agent/src/models.rs b/dstack/guest-agent/src/models.rs index 1d91bab21..97a537251 100644 --- a/dstack/guest-agent/src/models.rs +++ b/dstack/guest-agent/src/models.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 -use guest_api::{Container, SystemInfo}; +use guest_api::{Container, GpuInfoResponse, SystemInfo}; use rinja::Template; mod filters { @@ -33,6 +33,37 @@ mod filters { Ok(hex::encode(s)) } + /// Drops a zero PCI domain. NVML reports `00000000:01:00.0` where lspci and + /// the kernel print `0000:01:00.0` or just `01:00.0`. + /// + /// Only a zero domain is dropped, and only from the three-field form: a + /// non-zero domain distinguishes two cards that would otherwise display the + /// same address, and `00:00.0` is a bus, not a domain. + /// + /// Display only. `/metrics` keeps the full string, because that is what + /// host-side tooling joins on. + pub fn short_bdf(s: &str) -> Result<&str, rinja::Error> { + let Some((domain, rest)) = s.split_once(':') else { + return Ok(s); + }; + if rest.contains(':') && !domain.is_empty() && domain.bytes().all(|b| b == b'0') { + return Ok(rest); + } + Ok(s) + } + + /// The label set every `dstack_gpu_*` series carries, matching + /// dcgm-exporter so host-side tooling that speaks BDF can correlate. + /// Written once here rather than repeated in the template for each metric. + pub fn gpu_labels(gpu: &guest_api::GpuDevice) -> Result { + Ok(format!( + "{{index=\"{}\", uuid=\"{}\", pci_bus_id=\"{}\"}}", + gpu.index, + prometheus_label(&gpu.uuid)?, + prometheus_label(&gpu.pci_bus_id)?, + )) + } + pub fn prometheus_label(s: &str) -> Result { Ok(s.replace('\\', "\\\\") .replace('\n', "\\n") @@ -56,10 +87,251 @@ pub struct Dashboard { pub public_tcbinfo: bool, pub cloud_vendor: String, pub cloud_product: String, + pub gpu_info: GpuInfoResponse, } #[derive(Template)] #[template(path = "metrics.tpl", escape = "none")] pub struct Metrics { pub system_info: SystemInfo, + pub gpu_info: GpuInfoResponse, +} + +#[cfg(test)] +mod tests { + use super::*; + use guest_api::GpuDevice; + use rinja::Template; + + fn render(gpu_info: GpuInfoResponse) -> String { + Metrics { + system_info: Default::default(), + gpu_info, + } + .render() + .expect("render") + } + + fn gpu(index: u32) -> GpuDevice { + GpuDevice { + index, + uuid: format!("GPU-{index}"), + pci_bus_id: format!("00000000:0{index}:00.0"), + ..Default::default() + } + } + + /// The regression this whole design exists for: a sample older than the + /// collection TTL is still exposed. Emitting nothing meant every scrape at + /// Prometheus' default interval saw zero GPU series on a healthy card. + #[test] + fn a_stale_sample_still_produces_series() { + let body = render(GpuInfoResponse { + gpus: vec![GpuDevice { + utilization_gpu: Some(37), + ..gpu(0) + }], + sample_age_ms: Some(60_000), + ..Default::default() + }); + assert!(body.contains( + r#"dstack_gpu_utilization_percent{index="0", uuid="GPU-0", pci_bus_id="00000000:00:00.0"} 37"# + )); + assert!(body.contains("dstack_gpu_nvml_up 1")); + assert!(body.contains("dstack_gpu_sample_age_seconds 60")); + } + + /// A field NVML could not answer must be absent, not zero: a scraped 0 is + /// indistinguishable from an idle GPU. + #[test] + fn a_field_that_failed_to_sample_emits_no_series() { + let body = render(GpuInfoResponse { + gpus: vec![GpuDevice { + utilization_gpu: None, + temperature_c: Some(0), + ..gpu(0) + }], + ..Default::default() + }); + assert!(!body.contains("dstack_gpu_utilization_percent{")); + assert!(body.contains(r#"dstack_gpu_temperature_celsius{index="0""#)); + } + + /// Failed queries are counted from a list. When they were a `"; "`-joined + /// string, a message containing that separator inflated the count. + #[test] + fn query_errors_counts_entries_not_separators() { + let body = render(GpuInfoResponse { + gpus: vec![GpuDevice { + errors: vec![ + "power: not supported".into(), + "memory: unknown error; retry advised".into(), + ], + ..gpu(0) + }], + ..Default::default() + }); + assert!(body.contains( + r#"dstack_gpu_query_errors{index="0", uuid="GPU-0", pci_bus_id="00000000:00:00.0"} 2"# + )); + } + + /// A guest with no GPU reports `nvml_up 1` with no devices: the collector + /// ran and found nothing. `nvml_up 0` is reserved for "could not tell". + #[test] + fn a_guest_without_gpus_is_not_reported_as_a_collection_failure() { + let body = render(GpuInfoResponse::default()); + assert!(body.contains("dstack_gpu_nvml_up 1")); + assert!(!body.contains("dstack_gpu_utilization_percent{")); + assert!(!body.contains("dstack_gpu_cc_ready")); + + let failed = render(GpuInfoResponse { + error: "NVIDIA driver is not loaded".into(), + ..Default::default() + }); + assert!(failed.contains("dstack_gpu_nvml_up 0")); + } + + /// Label values are escaped, so a hostile UUID cannot inject a series. + #[test] + fn label_values_are_escaped() { + let body = render(GpuInfoResponse { + gpus: vec![GpuDevice { + uuid: r#"GPU-"injected" \ end"#.into(), + utilization_gpu: Some(1), + ..gpu(0) + }], + ..Default::default() + }); + assert!(body.contains(r#"uuid="GPU-\"injected\" \\ end""#), "{body}"); + } + + /// Watts are rendered from milliwatts. Raw f32 Display would print + /// 70.123001 for a card drawing 70.123 W. + #[test] + fn dashboard_power_is_formatted_to_one_decimal() { + let dashboard = Dashboard { + app_name: String::new(), + app_id: vec![], + instance_id: vec![], + device_id: vec![], + key_provider_info: String::new(), + tcb_info: String::new(), + containers: vec![], + system_info: Default::default(), + public_sysinfo: true, + public_logs: false, + public_tcbinfo: false, + cloud_vendor: String::new(), + cloud_product: String::new(), + gpu_info: GpuInfoResponse { + gpus: vec![GpuDevice { + power_usage_mw: Some(70_123), + ..gpu(0) + }], + cc_enabled: Some(true), + sample_age_ms: Some(2_500), + ..Default::default() + }, + }; + let html = dashboard.render().expect("render"); + assert!(html.contains("70.1 W"), "{html}"); + } + + /// The CC fields are labelled rows, so the value carries no prose of its + /// own. What must not come back is the old run-on line, and the age has to + /// round like every other number on the page. + #[test] + fn dashboard_renders_the_gpu_header_as_labelled_rows() { + let html = dashboard_with(GpuInfoResponse { + gpus: vec![gpu(0)], + cc_enabled: Some(true), + cc_ready: Some(false), + sample_age_ms: Some(1_049), + ..Default::default() + }); + assert!(html.contains(">Confidential Computing<"), "{html}"); + assert!(html.contains(">true<"), "{html}"); + assert!(html.contains(">GPU Ready State<"), "{html}"); + assert!(html.contains(">false<"), "{html}"); + assert!(html.contains("1.0 s"), "{html}"); + assert!(!html.contains("enabled = true"), "{html}"); + } + + /// An unset field is a third state, not a false. Reporting a GPU whose CC + /// status could not be read as `false` would invert the claim. + #[test] + fn dashboard_distinguishes_unknown_cc_state_from_false() { + let html = dashboard_with(GpuInfoResponse { + gpus: vec![gpu(0)], + cc_enabled: None, + cc_ready: None, + ..Default::default() + }); + assert!(html.contains(">unknown<"), "{html}"); + assert!(!html.contains(">false<"), "{html}"); + } + + /// Every other block on the page is a card. A bare paragraph on the page + /// background is what this section used to render into. + #[test] + fn dashboard_keeps_the_gpu_section_inside_a_card() { + let html = dashboard_with(GpuInfoResponse::default()); + assert!( + html.contains(r#"
No NVIDIA GPUs
"#), + "{html}" + ); + } + + /// The zero domain is display noise, but only the noise may go. A non-zero + /// domain is what tells two cards on a multi-domain host apart, and + /// `00:00.0` is bus zero, not a domain that can be dropped. + #[test] + fn the_pci_domain_is_dropped_only_where_it_carries_nothing() { + use super::filters::short_bdf; + + assert_eq!(short_bdf("00000000:01:00.0").unwrap(), "01:00.0"); + assert_eq!(short_bdf("0000:01:00.0").unwrap(), "01:00.0"); + assert_eq!(short_bdf("00010000:01:00.0").unwrap(), "00010000:01:00.0"); + assert_eq!(short_bdf("00:00.0").unwrap(), "00:00.0"); + assert_eq!(short_bdf("01:00.0").unwrap(), "01:00.0"); + } + + /// The dashboard drops the UUID column and shortens the address. Neither + /// may reach `/metrics`: those labels are what a host-side exporter joins + /// on, so they carry the identifiers verbatim. + #[test] + fn metrics_labels_keep_the_full_identifiers() { + let body = render(GpuInfoResponse { + gpus: vec![GpuDevice { + uuid: "GPU-b2880e21-86ab".into(), + pci_bus_id: "00000000:01:00.0".into(), + ..gpu(0) + }], + ..Default::default() + }); + assert!(body.contains(r#"uuid="GPU-b2880e21-86ab""#), "{body}"); + assert!(body.contains(r#"pci_bus_id="00000000:01:00.0""#), "{body}"); + } + + fn dashboard_with(gpu_info: GpuInfoResponse) -> String { + Dashboard { + app_name: String::new(), + app_id: vec![], + instance_id: vec![], + device_id: vec![], + key_provider_info: String::new(), + tcb_info: String::new(), + containers: vec![], + system_info: Default::default(), + public_sysinfo: true, + public_logs: false, + public_tcbinfo: false, + cloud_vendor: String::new(), + cloud_product: String::new(), + gpu_info, + } + .render() + .expect("render") + } } diff --git a/dstack/guest-agent/templates/dashboard.html b/dstack/guest-agent/templates/dashboard.html index 4cfd5494d..e4eccda03 100644 --- a/dstack/guest-agent/templates/dashboard.html +++ b/dstack/guest-agent/templates/dashboard.html @@ -213,6 +213,111 @@

Node Information

+ {% if public_sysinfo %} +

GPUs

+ {% if !gpu_info.error.is_empty() %} +
{{ gpu_info.error }}
+ {% else if gpu_info.gpus.is_empty() %} +
No NVIDIA GPUs
+ {% else %} +
+
+
+
Confidential Computing
+
+ {%- match gpu_info.cc_enabled -%} + {%- when Some with (enabled) -%} + {{- enabled -}} + {%- when None -%} + unknown + {%- endmatch -%} +
+
+
+
GPU Ready State
+
+ {%- match gpu_info.cc_ready -%} + {%- when Some with (ready) -%} + {{- ready -}} + {%- when None -%} + unknown + {%- endmatch -%} +
+
+ {%- match gpu_info.sample_age_ms %} + {%- when Some with (age) %} +
+
Sample Age
+
{{ "{:.1}"|format(age as f32 / 1000.0) }} s
+
+ {%- when None %} + {%- endmatch %} +
+
+ + + + + + + + + + + + + + + {% for gpu in gpu_info.gpus %} + + + + + + + + + + + {% endfor %} + +
IndexPCIGPU %Mem %MemoryTempPowerError
{{ gpu.index }}{{ gpu.pci_bus_id|short_bdf }} + {% match gpu.utilization_gpu %} + {% when Some with (value) %}{{ value }}% + {% when None %}- + {% endmatch %} + + {% match gpu.utilization_memory %} + {% when Some with (value) %}{{ value }}% + {% when None %}- + {% endmatch %} + + {% match gpu.memory_used_bytes %} + {% when Some with (used) %} + {% match gpu.memory_total_bytes %} + {% when Some with (total) %}{{ used|hsize }} / {{ total|hsize }} + {% when None %}{{ used|hsize }} + {% endmatch %} + {% when None %} + {% match gpu.memory_total_bytes %} + {% when Some with (total) %}- / {{ total|hsize }} + {% when None %}- + {% endmatch %} + {% endmatch %} + + {% match gpu.temperature_c %} + {% when Some with (value) %}{{ value }} C + {% when None %}- + {% endmatch %} + + {% match gpu.power_usage_mw %} + {% when Some with (value) %}{{ "{:.1}"|format(value as f32 / 1000.0) }} W + {% when None %}- + {% endmatch %} + {{ gpu.errors.join("; ") }}
+ {% endif %} + {% endif %} +

Deployed Containers

diff --git a/dstack/guest-agent/templates/metrics.tpl b/dstack/guest-agent/templates/metrics.tpl index 00d94f8f2..f80a8e915 100644 --- a/dstack/guest-agent/templates/metrics.tpl +++ b/dstack/guest-agent/templates/metrics.tpl @@ -74,6 +74,79 @@ dstack_guest_disk_used_bytes{name="{{disk.name|prometheus_label}}", mount_point= dstack_guest_disk_used_ratio{name="{{disk.name|prometheus_label}}", mount_point="{{disk.mount_point|prometheus_label}}"} {% if disk.total_size > 0 %}{{(disk.total_size - disk.free_size) as f64 / disk.total_size as f64}}{% else %}0{% endif %} {% endfor %} +# HELP dstack_gpu_nvml_up 1 if the last GPU sample succeeded, 0 otherwise. +# TYPE dstack_gpu_nvml_up gauge +dstack_gpu_nvml_up {% if gpu_info.error.is_empty() %}1{% else %}0{% endif %} + +{%- match gpu_info.sample_age_ms %}{% when Some with (age) %} + +# HELP dstack_gpu_sample_age_seconds Age of the GPU sample being served. +# TYPE dstack_gpu_sample_age_seconds gauge +dstack_gpu_sample_age_seconds {{ age as f64 / 1000.0 }} +{%- when None %}{% endmatch %} + +{%- match gpu_info.cc_enabled %}{% when Some with (enabled) %} + +# HELP dstack_gpu_cc_enabled 1 if NVIDIA confidential computing is enabled. +# TYPE dstack_gpu_cc_enabled gauge +dstack_gpu_cc_enabled {% if enabled %}1{% else %}0{% endif %} +{%- when None %}{% endmatch %} + +{%- match gpu_info.cc_ready %}{% when Some with (ready) %} + +# HELP dstack_gpu_cc_ready 1 if NVIDIA CC GPUs are accepting client requests. +# TYPE dstack_gpu_cc_ready gauge +dstack_gpu_cc_ready {% if ready %}1{% else %}0{% endif %} +{%- when None %}{% endmatch %} + +# HELP dstack_gpu_utilization_percent GPU SM duty cycle, percent. +# TYPE dstack_gpu_utilization_percent gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.utilization_gpu %}{% when Some with (value) %} +dstack_gpu_utilization_percent{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_memory_utilization_percent GPU memory-bus duty cycle, percent. +# TYPE dstack_gpu_memory_utilization_percent gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.utilization_memory %}{% when Some with (value) %} +dstack_gpu_memory_utilization_percent{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_memory_total_bytes GPU framebuffer size in bytes. +# TYPE dstack_gpu_memory_total_bytes gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.memory_total_bytes %}{% when Some with (value) %} +dstack_gpu_memory_total_bytes{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_memory_used_bytes GPU framebuffer used in bytes. +# TYPE dstack_gpu_memory_used_bytes gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.memory_used_bytes %}{% when Some with (value) %} +dstack_gpu_memory_used_bytes{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_memory_free_bytes GPU framebuffer free in bytes. +# TYPE dstack_gpu_memory_free_bytes gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.memory_free_bytes %}{% when Some with (value) %} +dstack_gpu_memory_free_bytes{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_temperature_celsius GPU temperature in Celsius. +# TYPE dstack_gpu_temperature_celsius gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.temperature_c %}{% when Some with (value) %} +dstack_gpu_temperature_celsius{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_power_usage_milliwatts GPU power usage in milliwatts. +# TYPE dstack_gpu_power_usage_milliwatts gauge +{%- for gpu in gpu_info.gpus %}{% match gpu.power_usage_mw %}{% when Some with (value) %} +dstack_gpu_power_usage_milliwatts{{ gpu|gpu_labels }} {{ value }} +{%- when None %}{% endmatch %}{% endfor %} + +# HELP dstack_gpu_query_errors Number of NVML field queries that failed on this GPU. +# TYPE dstack_gpu_query_errors gauge +{%- for gpu in gpu_info.gpus %} +dstack_gpu_query_errors{{ gpu|gpu_labels }} {{ gpu.errors.len() }} +{%- endfor %} + # Everything below is the pre-rename exposition, kept verbatim so existing # dashboards keep working through one release cycle. Deprecated: use the # dstack_guest_* series above; these will be removed in a future release. diff --git a/dstack/guest-api/proto/guest_api.proto b/dstack/guest-api/proto/guest_api.proto index 5d5868cb9..9e2a9f990 100644 --- a/dstack/guest-api/proto/guest_api.proto +++ b/dstack/guest-api/proto/guest_api.proto @@ -129,12 +129,70 @@ message DiskInfo { uint64 free_size = 5; } +// One NVIDIA GPU sampled through NVML. +// +// Optional numeric fields are unset when that query failed, so a consumer can +// tell a genuine zero from a missing sample. `errors` lists the failed queries +// for this card, one entry each, so counting them does not mean re-parsing a +// joined string. +// +// `uuid` is the stable identity; `index` and `pci_bus_id` are ordinal/location +// helpers. `uuid` and `pci_bus_id` are plain strings rather than `optional` +// because an empty value is already unambiguous: NVML never returns one, and +// the matching `errors` entry names the query that failed. `index` is always +// populated, so a device is identifiable even when both lookups fail. +message GpuDevice { + // NVML device index (PCI order) + uint32 index = 1; + string uuid = 2; + // PCI BDF from nvmlPciInfo.busId + string pci_bus_id = 3; + // SM duty cycle, percent + optional uint32 utilization_gpu = 4; + // Memory-bus duty cycle, percent + optional uint32 utilization_memory = 5; + optional uint64 memory_total_bytes = 6; + optional uint64 memory_used_bytes = 7; + optional uint64 memory_free_bytes = 8; + optional uint32 temperature_c = 9; + optional uint32 power_usage_mw = 10; + // Failed queries for this card, one per entry; empty if all fields succeeded + repeated string errors = 11; +} + +// GPU inventory collected through NVML. +// +// `error` is non-empty when NVML itself is unavailable, or when the sample +// could not be taken at all. In that case `gpus` is empty. An empty `gpus` +// with an empty `error` means the collector ran and the guest has no NVIDIA +// GPUs. +// +// `cc_ready` and `cc_enabled` are both system-wide, not per device: NVML +// exposes them as `nvmlSystemGetConfComputeGpusReadyState` and +// `nvmlSystemGetConfComputeSettings`, which nvml-wrapper hangs off `Device` +// without using the device handle. `cc_enabled` says the CC feature is turned +// on; `cc_ready` says the SPDM handshake finished and the GPUs are accepting +// client work. Both unset when the query is unsupported or failed. +// +// `sample_age_ms` is how old the returned sample is. The agent serves the last +// known snapshot rather than nothing, so consumers must decide for themselves +// how stale is too stale. Unset when the response carries no sample. +message GpuInfoResponse { + repeated GpuDevice gpus = 1; + string error = 2; + optional bool cc_ready = 3; + optional bool cc_enabled = 4; + optional uint64 sample_age_ms = 5; +} + // Direct gRPC surface exposed by the in-guest agent. service GuestApi { // Returns attestation material and identifiers for the calling guest. rpc Info(google.protobuf.Empty) returns (GuestInfo); // Reports the guest's OS/kernel and resource statistics. rpc SysInfo(google.protobuf.Empty) returns (SystemInfo); + // Reports per-GPU utilization, memory, temperature, power, and CC ready state. + rpc GpuInfo(google.protobuf.Empty) returns (GpuInfoResponse); // Dumps NIC/Gateway configuration so operators can debug connectivity. rpc NetworkInfo(google.protobuf.Empty) returns (NetworkInformation); // Enumerates the containers running under the guest supervisor. @@ -147,6 +205,7 @@ service GuestApi { service ProxiedGuestApi { rpc Info(Id) returns (GuestInfo); rpc SysInfo(Id) returns (SystemInfo); + rpc GpuInfo(Id) returns (GpuInfoResponse); rpc NetworkInfo(Id) returns (NetworkInformation); rpc ListContainers(Id) returns (ListContainersResponse); rpc Shutdown(Id) returns (google.protobuf.Empty); diff --git a/dstack/guest-api/src/lib.rs b/dstack/guest-api/src/lib.rs index eb5524134..36aae1ad1 100644 --- a/dstack/guest-api/src/lib.rs +++ b/dstack/guest-api/src/lib.rs @@ -10,3 +10,92 @@ mod generated; #[cfg(feature = "client")] pub mod client; + +#[cfg(test)] +mod tests { + use super::*; + use prost::Message; + + /// Every numeric field is `optional` so a consumer can tell "the query + /// failed" from a genuine zero. That distinction only holds if it survives + /// the wire, which is what this pins. + #[test] + fn gpu_info_response_roundtrips_optional_fields() { + let original = GpuInfoResponse { + error: String::new(), + cc_ready: Some(true), + cc_enabled: Some(true), + sample_age_ms: Some(4200), + gpus: vec![GpuDevice { + index: 0, + uuid: "GPU-abc".into(), + pci_bus_id: "00000000:01:00.0".into(), + utilization_gpu: Some(12), + utilization_memory: None, + memory_total_bytes: Some(8 << 30), + memory_used_bytes: Some(0), + memory_free_bytes: None, + temperature_c: Some(0), + power_usage_mw: None, + errors: vec!["temperature: timeout".into()], + }], + }; + let bytes = original.encode_to_vec(); + let decoded = GpuInfoResponse::decode(bytes.as_slice()).expect("decode"); + assert_eq!(decoded, original); + assert_eq!(decoded.gpus[0].utilization_memory, None); + assert_eq!(decoded.gpus[0].memory_used_bytes, Some(0)); + assert_eq!(decoded.cc_ready, Some(true)); + assert_eq!(decoded.sample_age_ms, Some(4200)); + } + + /// The NVML-unavailable shape: an error, no devices, and no CC state at + /// all. `cc_ready` unset must not decode as `Some(false)`, which would + /// read as "the handshake failed" rather than "we could not ask". + #[test] + fn gpu_info_response_omits_unset_optionals() { + let original = GpuInfoResponse { + error: "failed to initialize NVML: driver not loaded".into(), + cc_ready: None, + cc_enabled: None, + sample_age_ms: None, + gpus: vec![], + }; + let decoded = GpuInfoResponse::decode(original.encode_to_vec().as_slice()).expect("decode"); + assert_eq!(decoded, original); + assert!(decoded.gpus.is_empty()); + assert!(!decoded.error.is_empty()); + assert_eq!(decoded.cc_ready, None); + assert_eq!(decoded.cc_enabled, None); + } + + /// "No GPUs" is the default response: empty devices, empty error. It must + /// stay distinguishable from every failure shape above. + #[test] + fn the_default_response_means_no_gpus_rather_than_a_failure() { + let decoded = + GpuInfoResponse::decode(GpuInfoResponse::default().encode_to_vec().as_slice()) + .expect("decode"); + assert!(decoded.gpus.is_empty()); + assert!(decoded.error.is_empty()); + assert_eq!(decoded.sample_age_ms, None); + } + + /// Per-field failures are a list, so counting them for + /// `dstack_gpu_query_errors` does not mean re-parsing a joined string. + #[test] + fn per_device_errors_are_counted_not_parsed() { + let device = GpuDevice { + index: 0, + errors: vec![ + "power: not supported".into(), + // A message containing the old "; " separator would have + // inflated the count when errors were a joined string. + "memory: unknown error; retry advised".into(), + ], + ..Default::default() + }; + let decoded = GpuDevice::decode(device.encode_to_vec().as_slice()).expect("decode"); + assert_eq!(decoded.errors.len(), 2); + } +} diff --git a/dstack/lspci/Cargo.toml b/dstack/lspci/Cargo.toml index bbb6f35cd..00b33724c 100644 --- a/dstack/lspci/Cargo.toml +++ b/dstack/lspci/Cargo.toml @@ -14,3 +14,4 @@ anyhow.workspace = true [dev-dependencies] insta.workspace = true +tempfile.workspace = true diff --git a/dstack/lspci/src/lib.rs b/dstack/lspci/src/lib.rs index ef3565ede..10c84e70e 100644 --- a/dstack/lspci/src/lib.rs +++ b/dstack/lspci/src/lib.rs @@ -6,6 +6,8 @@ use std::process::Command; use anyhow::{Context, Result}; +pub mod sysfs; + /// Represents a PCI device with the specified fields. #[derive(Debug)] pub struct Device { diff --git a/dstack/lspci/src/sysfs.rs b/dstack/lspci/src/sysfs.rs new file mode 100644 index 000000000..b866a28fc --- /dev/null +++ b/dstack/lspci/src/sysfs.rs @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: © 2025 Phala Network +// +// SPDX-License-Identifier: Apache-2.0 + +//! GPU inventory read straight from sysfs. +//! +//! Distinct from the `lspci` parser in this crate: no subprocess, no text +//! parsing, and it sees devices the NVIDIA driver never bound. That matters in +//! two places with opposite failure policies -- the boot attestation gate must +//! fail closed when the inventory cannot be read, while a telemetry collector +//! wants to shrug and report no GPUs -- so this returns the raw counts and lets +//! each caller decide. + +use std::path::Path; + +use anyhow::{Context, Result}; + +/// Where the kernel exposes the PCI bus. +pub const PCI_DEVICES: &str = "/sys/bus/pci/devices"; + +const NVIDIA_VENDOR_ID: &str = "0x10de"; +/// PCI class prefixes for VGA and 3D controllers. +const DISPLAY_CLASS_PREFIXES: [&str; 2] = ["0x0300", "0x0302"]; + +/// Display-class PCI devices, split by vendor. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct GpuInventory { + /// All display-class devices, whatever the vendor. + pub total: u32, + /// The subset made by NVIDIA. + pub nvidia: u32, +} + +impl GpuInventory { + /// True when at least one NVIDIA display device is attached. + pub fn has_nvidia(&self) -> bool { + self.nvidia > 0 + } +} + +/// Counts display-class GPUs on the live PCI bus. +pub fn gpu_inventory() -> Result { + gpu_inventory_at(Path::new(PCI_DEVICES)) +} + +/// Counts display-class GPUs under an arbitrary sysfs root, for tests. +pub fn gpu_inventory_at(devices_path: &Path) -> Result { + let entries = std::fs::read_dir(devices_path) + .with_context(|| format!("failed to enumerate {}", devices_path.display()))?; + let mut inventory = GpuInventory::default(); + for entry in entries { + let device = entry.context("failed to read PCI device entry")?; + let class_path = device.path().join("class"); + let class = std::fs::read_to_string(&class_path) + .with_context(|| format!("failed to read {}", class_path.display()))?; + if !class + .trim() + .get(..6) + .is_some_and(|prefix| DISPLAY_CLASS_PREFIXES.contains(&prefix)) + { + continue; + } + inventory.total += 1; + let vendor_path = device.path().join("vendor"); + let vendor = std::fs::read_to_string(&vendor_path) + .with_context(|| format!("failed to read {}", vendor_path.display()))?; + if vendor.trim() == NVIDIA_VENDOR_ID { + inventory.nvidia += 1; + } + } + Ok(inventory) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn add_pci_device(root: &Path, name: &str, vendor: &str, class: &str) { + let dir = root.join(name); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("vendor"), vendor).unwrap(); + std::fs::write(dir.join("class"), class).unwrap(); + } + + #[test] + fn inventory_counts_nvidia_and_non_nvidia_gpus() { + let root = tempfile::tempdir().unwrap(); + add_pci_device(root.path(), "0000:01:00.0", "0x10de\n", "0x030200\n"); + add_pci_device(root.path(), "0000:02:00.0", "0x1234\n", "0x030000\n"); + // A virtio NIC is not a display device and must not be counted. + add_pci_device(root.path(), "0000:03:00.0", "0x1af4\n", "0x020000\n"); + assert_eq!( + gpu_inventory_at(root.path()).unwrap(), + GpuInventory { + total: 2, + nvidia: 1 + } + ); + } + + /// The common CVM: virtio devices only. `has_nvidia` is the gate that keeps + /// GPU telemetry from costing such a guest anything. + #[test] + fn a_guest_without_a_display_device_reports_no_gpus() { + let root = tempfile::tempdir().unwrap(); + add_pci_device(root.path(), "0000:01:00.0", "0x1af4\n", "0x020000\n"); + add_pci_device(root.path(), "0000:02:00.0", "0x1af4\n", "0x010000\n"); + let inventory = gpu_inventory_at(root.path()).unwrap(); + assert_eq!(inventory, GpuInventory::default()); + assert!(!inventory.has_nvidia()); + } + + /// Callers must be able to tell "no GPUs" from "could not look". + #[test] + fn an_unreadable_root_is_an_error_not_an_empty_inventory() { + assert!(gpu_inventory_at(Path::new("/nonexistent/pci/devices")).is_err()); + } +} diff --git a/dstack/vmm/src/guest_api_service.rs b/dstack/vmm/src/guest_api_service.rs index 1fc089f8f..439df00d1 100644 --- a/dstack/vmm/src/guest_api_service.rs +++ b/dstack/vmm/src/guest_api_service.rs @@ -6,7 +6,7 @@ use crate::App as AppState; use anyhow::Result; use guest_api::{ proxied_guest_api_server::{ProxiedGuestApiRpc, ProxiedGuestApiServer}, - GuestInfo, Id, ListContainersResponse, NetworkInformation, SystemInfo, + GpuInfoResponse, GuestInfo, Id, ListContainersResponse, NetworkInformation, SystemInfo, }; use ra_rpc::{CallContext, RpcCall}; use std::ops::Deref; @@ -42,6 +42,10 @@ impl ProxiedGuestApiRpc for GuestApiHandler { self.guest_agent_client(&request.id)?.sys_info().await } + async fn gpu_info(self, request: Id) -> Result { + self.guest_agent_client(&request.id)?.gpu_info().await + } + async fn network_info(self, request: Id) -> Result { self.guest_agent_client(&request.id)?.network_info().await }