From 4e6d4f587035eaa6071e4ab27a9c1320412c2720 Mon Sep 17 00:00:00 2001 From: Jorge Prendes Date: Thu, 19 Nov 2026 21:58:09 +0000 Subject: [PATCH 1/2] Cache snapshot memory hashes and add checked saving Make save reuse content-addressed blobs without rereading them and add checked_save for full hashing and existing-blob verification. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Jorge Prendes --- docs/snapshot-oci-format.md | 23 ++++ .../src/sandbox/snapshot/file/digest.rs | 11 ++ .../src/sandbox/snapshot/file/fsutil.rs | 47 +++++--- .../src/sandbox/snapshot/file/mod.rs | 64 +++++++++-- .../src/sandbox/snapshot/file_tests.rs | 102 +++++++++++++++--- .../src/sandbox/snapshot/memory/mod.rs | 4 +- 6 files changed, 215 insertions(+), 36 deletions(-) diff --git a/docs/snapshot-oci-format.md b/docs/snapshot-oci-format.md index bc5c4e575..f2e43a5a0 100644 --- a/docs/snapshot-oci-format.md +++ b/docs/snapshot-oci-format.md @@ -57,6 +57,29 @@ Manifest layers are ordered data, then transport. Blob filenames are the sha256 of the blob bytes, so identical blobs across tags are stored once. +### Immutable blob reuse + +`Snapshot::save` caches each shared memory layer's SHA-256. Existing regular +blob files with the expected length are trusted by their content-addressed +filenames without reading or hashing their contents, even at a new destination. +There is no backing-file cache. Missing blobs are written atomically; symlinks +and wrong-length files are replaced. +`Snapshot::checked_load` seeds the digest cache during verification; unchecked +`load` does not. Metadata and transport blobs retain their normal write path. + +Callers must keep loaded and published memory blobs immutable for the lifetime +of these snapshots and outside writable guest mounts. Host-side mutation and +silent storage corruption are outside the save-time threat boundary. Saving +does not detect same-length corruption; use `Snapshot::checked_load` to verify +persisted contents explicitly. The persisted OCI format is unchanged. + +`Snapshot::checked_save` takes the fully verifying path: it recomputes all +memory-layer digests without using the cache and hashes existing memory blob +files before reusing them. Mismatched contents are atomically replaced from +the snapshot's memory. It otherwise has the same layout and tag semantics as +`save`. This is not protection against concurrent modification of loaded +backing files, which must remain immutable while snapshots use them. + ## Application metadata `Snapshot::with_metadata` creates a snapshot that shares the source snapshot's diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/digest.rs b/src/hyperlight_host/src/sandbox/snapshot/file/digest.rs index a0448714e..80ca6d66d 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/digest.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/digest.rs @@ -6,6 +6,13 @@ use std::io::{Read, Seek, SeekFrom}; use oci_spec::image::Digest; use sha2::{Digest as _, Sha256}; +#[cfg(test)] +thread_local! { + // Counts hashed bytes to test save's cache reuse versus checked_save's full hashing. + // Thread-local storage isolates parallel tests; this is absent from production builds. + pub(super) static HASHED_BYTES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + /// A `sha256:` digest as recorded in OCI manifests. The bare hex /// (without prefix) is also the blob's filename inside `blobs/sha256/`. #[derive(Clone)] @@ -16,6 +23,8 @@ pub(super) struct Digest256 { impl Digest256 { pub(super) fn from_bytes(bytes: &[u8]) -> Self { + #[cfg(test)] + HASHED_BYTES.update(|count| count + bytes.len()); let arr: [u8; 32] = Sha256::digest(bytes).into(); Self::from_digest_array(arr) } @@ -86,6 +95,8 @@ pub(super) fn verify_blob_file( break; } hasher.update(&buf[..n]); + #[cfg(test)] + HASHED_BYTES.update(|count| count + n); } file.seek(SeekFrom::Start(0)) .map_err(|e| crate::new_error!("failed to rewind {} blob: {}", label, e))?; diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/fsutil.rs b/src/hyperlight_host/src/sandbox/snapshot/file/fsutil.rs index abd74e87e..ea07f0d19 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/fsutil.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/fsutil.rs @@ -40,27 +40,44 @@ pub(super) fn put_blob(blobs_dir: &Path, digest: &Digest256, bytes: &[u8]) -> cr } /// Write a content-addressed blob into `blobs_dir`, reusing the -/// existing file at `blobs_dir/` only if it is present, has the -/// expected length, AND hashes to `digest`. A wrong-content file of -/// the right length (corruption, partial copy, foreign tool) is -/// overwritten. -/// -/// Intended for the large snapshot blob, where the cost of one full -/// re-hash of the existing file is far less than the cost of an -/// unconditional rewrite. +/// existing regular file at `blobs_dir/` if it has the expected +/// length. Unless `verify_existing` is set, the content-addressed +/// filename is trusted without reading the file. With verification, +/// mismatched contents are replaced. Callers must keep published blobs +/// immutable while snapshots use them. pub(super) fn put_blob_if_absent( blobs_dir: &Path, digest: &Digest256, bytes: &[u8], + verify_existing: bool, ) -> crate::Result<()> { let target = blobs_dir.join(&digest.hex); - if let Ok(meta) = std::fs::symlink_metadata(&target) - && meta.is_file() - && meta.len() == bytes.len() as u64 - && let Ok(mut file) = std::fs::File::open(&target) - && verify_blob_file("existing snapshot", &mut file, &digest.hex).is_ok() - { - return Ok(()); + match std::fs::symlink_metadata(&target) { + Ok(meta) if meta.is_file() && meta.len() == bytes.len() as u64 => { + if !verify_existing { + return Ok(()); + } + let mut file = open_no_follow(&target)?; + match verify_blob_file("existing snapshot", &mut file, &digest.hex) { + Ok(()) => return Ok(()), + Err(error) => { + tracing::warn!( + ?target, + %error, + "Replacing snapshot blob after failed content verification" + ); + } + } + } + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(crate::new_error!( + "failed to stat snapshot blob {:?}: {}", + target, + error + )); + } } replace_file_atomic(&target, bytes) } diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs b/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs index ffc731508..fa5309af9 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs @@ -41,6 +41,11 @@ use crate::mem::shared_mem::{ReadonlySharedMemory, SharedMemory}; pub(super) const OCI_LAYOUT_VERSION: &str = "1.0.0"; +#[cfg(test)] +pub(super) fn test_hashed_bytes() -> usize { + digest::HASHED_BYTES.get() +} + /// Short golden-tag token for the host CPU vendor, or `None` if the /// goldens do not cover it. Exposed for the snapshot golden tests so /// they share the host crate's vendor detection. @@ -333,6 +338,19 @@ impl Snapshot { /// digest but not by this tag. Snapshots under other tags are /// untouched. /// + /// # Immutable blob store + /// + /// Memory-layer digests are cached across snapshots sharing those layers. + /// Existing regular blob files with the expected length are trusted by + /// their content-addressed filenames without reading or hashing them. + /// + /// The caller must keep loaded and saved memory blobs immutable for the + /// lifetime of these snapshots and outside writable guest mounts. Saving + /// does not detect same-length corruption or host-side modifications. + /// Use [`Snapshot::checked_save`] to verify existing blobs before reuse, + /// or [`Snapshot::checked_load`] to verify persisted contents explicitly. + /// Metadata and transport blobs follow the normal write path. + /// /// # Portability /// /// Snapshot images are bound to the specific CPU architecture, @@ -368,8 +386,30 @@ impl Snapshot { /// # } /// ``` pub fn save(&self, path: impl AsRef, tag: &OciTag) -> crate::Result { - let path = path.as_ref(); + self.save_inner(path.as_ref(), tag, false) + } + + /// Save this snapshot, recomputing memory-layer digests and verifying + /// existing blob contents before reuse. + /// + /// Unlike [`Self::save`], this method bypasses the memory-layer digest + /// cache and hashes existing memory blob files. Blobs with mismatched + /// contents are atomically replaced with the snapshot's memory contents. + /// It uses the same layout, tag semantics and return value as [`Self::save`]. + /// + /// This does not protect against concurrent modification of the snapshot's + /// backing files. Loaded memory blobs must still remain immutable for the + /// lifetime of the snapshot. + pub fn checked_save(&self, path: impl AsRef, tag: &OciTag) -> crate::Result { + self.save_inner(path.as_ref(), tag, true) + } + fn save_inner( + &self, + path: &Path, + tag: &OciTag, + verify_blobs: bool, + ) -> crate::Result { // Building the config can reject the snapshot. Do it before // writing any file. let cfg = self.build_config()?; @@ -466,7 +506,8 @@ impl Snapshot { Vec::new() }; - let new_desc = self.write_blobs_and_build_descriptor(path, tag, &cfg, &cfg_bytes)?; + let new_desc = + self.write_blobs_and_build_descriptor(path, tag, &cfg, &cfg_bytes, verify_blobs)?; let written_digest = OciDigest::from_oci_spec_digest(new_desc.digest()); // Replacement is by tag, not by digest: a new snapshot may @@ -514,6 +555,7 @@ impl Snapshot { tag: &OciTag, cfg: &OciSnapshotConfig, cfg_bytes: &[u8], + verify_blobs: bool, ) -> crate::Result { let blobs_dir = dir.join("blobs").join("sha256"); std::fs::create_dir_all(&blobs_dir).map_err(|e| { @@ -526,8 +568,13 @@ impl Snapshot { .blobs() .map(|blob| { let bytes = blob.memory().as_slice(); - let digest = Digest256::from_bytes(bytes); - put_blob_if_absent(&blobs_dir, &digest, bytes)?; + let digest = if verify_blobs { + Digest256::from_bytes(bytes) + } else { + let hex = blob.sha256.get_or_init(|| Digest256::from_bytes(bytes).hex); + Digest256 { hex: hex.clone() } + }; + put_blob_if_absent(&blobs_dir, &digest, bytes, verify_blobs)?; blob_descriptor(MT_SNAPSHOT_CURRENT, &digest, bytes.len()) }) .collect::>>()?; @@ -987,15 +1034,18 @@ impl Snapshot { .collect::>>()?; let page_tables = blobs[cfg.page_table_layer].clone(); let data_layers = blobs - .into_iter() + .iter() .filter(|blob| blob.data_len() > 0) + .cloned() .collect(); let memory = Arc::new(SnapshotMemory::new(data_layers, page_tables)?); // Hash after validating the layers so malformed snapshots fail // cheaply. if verify_blobs { - for (desc, file) in data_descs.iter().zip(&mut snap_files) { - verify_blob_file("snapshot", file, &parse_oci_digest(desc.digest())?)?; + for ((desc, file), blob) in data_descs.iter().zip(&mut snap_files).zip(&blobs) { + let digest = parse_oci_digest(desc.digest())?; + verify_blob_file("snapshot", file, &digest)?; + blob.sha256.get_or_init(|| digest); } } diff --git a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs index 1b7b215dd..970bbf3b5 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs @@ -472,11 +472,8 @@ fn layered_snapshot_recapture_round_trip() { ); } -/// A pre-existing snapshot blob with the right length but wrong -/// bytes (corruption, partial copy, foreign tool) must be detected -/// and replaced by `save`, not silently trusted. #[test] -fn save_self_heals_same_length_wrong_content_snapshot_blob() { +fn save_trusts_existing_blob_name_without_verifying_contents() { let snapshot = create_snapshot(); let dir = tempfile::tempdir().unwrap(); @@ -491,19 +488,98 @@ fn save_self_heals_same_length_wrong_content_snapshot_blob() { let len = std::fs::metadata(&snap_path).unwrap().len() as usize; std::fs::write(&snap_path, vec![0xAAu8; len]).unwrap(); - // Re-save. `put_blob_if_absent` must notice the digest mismatch - // and rewrite the blob. + let before = super::file::test_hashed_bytes(); snapshot .save(&oci, &OciTag::new("latest").unwrap()) .unwrap(); + assert!(super::file::test_hashed_bytes() - before < len); + assert_eq!(std::fs::read(&snap_path).unwrap(), vec![0xAAu8; len]); + let error = unwrap_err_snapshot(Snapshot::checked_load(&oci, OciTag::new("latest").unwrap())); + assert!(error.to_string().contains("digest mismatch")); +} - // A checked load succeeds: the rewritten blob matches the - // descriptor digest. - let loaded = Snapshot::checked_load(&oci, OciTag::new("latest").unwrap()).unwrap(); - let mut sbox2 = - Sandbox::from_snapshot(Arc::new(loaded), HostFunctions::default(), None).unwrap(); - let result: String = sbox2.call("Echo", "hello\n".to_string()).unwrap(); - assert_eq!(result, "hello\n"); +#[test] +fn checked_save_rehashes_memory_and_existing_blobs() { + let snapshot = create_snapshot(); + let dir = tempfile::tempdir().unwrap(); + let tag = OciTag::new("latest").unwrap(); + let digest = snapshot.save(dir.path(), &tag).unwrap(); + let memory_bytes: usize = snapshot + .state + .memory + .blobs() + .map(|blob| blob.memory().as_slice().len()) + .sum(); + + for _ in 0..2 { + let before = super::file::test_hashed_bytes(); + let checked_digest = snapshot.checked_save(dir.path(), &tag).unwrap(); + assert!(super::file::test_hashed_bytes() - before >= 2 * memory_bytes); + assert_eq!(checked_digest, digest); + } + Snapshot::checked_load(dir.path(), tag).unwrap(); +} + +#[test] +fn checked_save_repairs_same_length_wrong_content_snapshot_blob() { + let snapshot = create_snapshot(); + let dir = tempfile::tempdir().unwrap(); + let tag = OciTag::new("latest").unwrap(); + let digest = snapshot.checked_save(dir.path(), &tag).unwrap(); + let path = find_snapshot_blob(dir.path()); + let len = std::fs::metadata(&path).unwrap().len() as usize; + std::fs::write(&path, vec![0xAAu8; len]).unwrap(); + + assert_eq!(snapshot.checked_save(dir.path(), &tag).unwrap(), digest); + let loaded = Arc::new(Snapshot::checked_load(dir.path(), tag).unwrap()); + let mut sandbox = Sandbox::from_snapshot(loaded, HostFunctions::default(), None).unwrap(); + assert_eq!( + sandbox.call::("Echo", "hello".to_string()).unwrap(), + "hello" + ); +} + +#[test] +fn save_reuses_hashes_across_saves_and_checked_loads() { + let snapshot = create_snapshot(); + let dir = tempfile::tempdir().unwrap(); + let tag = OciTag::new("latest").unwrap(); + snapshot.save(dir.path(), &tag).unwrap(); + let blob_len = std::fs::metadata(find_snapshot_blob(dir.path())) + .unwrap() + .len() as usize; + + let before = super::file::test_hashed_bytes(); + snapshot.save(dir.path(), &tag).unwrap(); + assert!(super::file::test_hashed_bytes() - before < blob_len); + + let loaded = Snapshot::checked_load(dir.path(), tag.clone()).unwrap(); + let before = super::file::test_hashed_bytes(); + loaded.save(dir.path(), &tag).unwrap(); + assert!(super::file::test_hashed_bytes() - before < blob_len); + let loaded = Arc::new(Snapshot::checked_load(dir.path(), tag).unwrap()); + let mut sandbox = Sandbox::from_snapshot(loaded, HostFunctions::default(), None).unwrap(); + assert_eq!(sandbox.call::("GetStatic", ()).unwrap(), 0); +} + +#[test] +fn save_trusts_replaced_files_and_writes_new_destinations() { + let snapshot = create_snapshot(); + let dir = tempfile::tempdir().unwrap(); + let tag = OciTag::new("latest").unwrap(); + snapshot.save(dir.path(), &tag).unwrap(); + let blob = find_snapshot_blob(dir.path()); + let replacement = dir.path().join("replacement"); + std::fs::copy(&blob, &replacement).unwrap(); + std::fs::rename(replacement, &blob).unwrap(); + let blob_len = std::fs::metadata(&blob).unwrap().len() as usize; + let before = super::file::test_hashed_bytes(); + snapshot.save(dir.path(), &tag).unwrap(); + assert!(super::file::test_hashed_bytes() - before < blob_len); + + let other = tempfile::tempdir().unwrap(); + snapshot.save(other.path(), &tag).unwrap(); + Snapshot::checked_load(other.path(), tag).unwrap(); } #[test] diff --git a/src/hyperlight_host/src/sandbox/snapshot/memory/mod.rs b/src/hyperlight_host/src/sandbox/snapshot/memory/mod.rs index 8aea74eca..359fd2ef7 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/memory/mod.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/memory/mod.rs @@ -17,7 +17,7 @@ //! ``` use std::ops::Range; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use hyperlight_common::vmem::PAGE_SIZE; @@ -56,6 +56,7 @@ pub(crate) struct SnapshotBlob { memory: ReadonlySharedMemory, gpa_start: u64, page_table_len: usize, + pub(super) sha256: OnceLock, } impl SnapshotBlob { @@ -96,6 +97,7 @@ impl SnapshotBlob { memory, gpa_start, page_table_len, + sha256: OnceLock::new(), }) } From a3fb5912ecf3e85600748faae5383948e5bc1cc6 Mon Sep 17 00:00:00 2001 From: Jorge Prendes Date: Thu, 19 Nov 2026 22:33:23 +0000 Subject: [PATCH 2/2] Document snapshot save verification behavior Explain trusted blob reuse in save and direct callers needing verification and corruption repair to checked_save. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: Jorge Prendes --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf2fc4117..f1c98cb6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). * Add `Sandbox::recreate()`, which rebuilds a sandbox's VM and memory from a snapshot. It recovers unrecoverable sandboxes and keeps configuration, host functions, and interrupt handles by @ludfjig in https://github.com/hyperlight-dev/hyperlight/pull/1751 ### Changed +* `Snapshot::save` caches memory-layer hashes and trusts existing same-length + regular blobs by their content-addressed filenames without verifying their + contents. Callers must keep blobs immutable. Use the new + `Snapshot::checked_save` to recompute hashes, verify existing blobs, and + atomically repair mismatched contents. * **Breaking:** Snapshots use ABI 10 and config schema v6. Guests must be rebuilt and existing snapshots regenerated. Guest entry takes the random seed, host page size, and log level.