diff --git a/CHANGELOG.md b/CHANGELOG.md index ac24915f3..54bc21eee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ## [Prerelease] - Unreleased ### Added +* Namespaced application metadata on immutable snapshots. * Per-direction virtqueue configuration through `SandboxConfiguration` and `SandboxBuilder`, with allocations included in scratch sizing. * Shared virtqueue framing with a 12-byte `MsgHeader` and external byte values. diff --git a/docs/snapshot-oci-format.md b/docs/snapshot-oci-format.md index 39fb3d41e..98c9918c5 100644 --- a/docs/snapshot-oci-format.md +++ b/docs/snapshot-oci-format.md @@ -35,8 +35,8 @@ Four blob kinds per tag: * **config** (`application/vnd.hyperlight.snapshot.config.v3+json`). The snapshot descriptor: arch, hypervisor, CPU vendor, ABI version, resume address and captured registers, memory and transport layout, - registered host functions, and snapshot generation counter. Loaded - eagerly and fully parsed. + registered host functions, snapshot generation counter, and namespaced + application metadata. Loaded eagerly and fully parsed. * **layer / memory** (`application/vnd.hyperlight.snapshot.memory.v1`). The raw guest memory image, exactly `memory_size` bytes. mmap'd on restore. @@ -51,6 +51,18 @@ The runtime queue protocol and canonical checkpoint are described in Blob filenames are the sha256 of the blob bytes, so identical blobs across tags are stored once. +## Application metadata + +`Snapshot::with_metadata` creates a snapshot that shares the source snapshot's +sandbox state and stores a serializable value under an application-owned +namespace. `Snapshot::metadata` deserializes the value for that namespace. +The source snapshot remains unchanged. + +Metadata is stored in the config blob as a JSON object with namespaces as +keys. Snapshots without metadata omit the field. The loader treats a missing +field as an empty map, so snapshots written before metadata support remain +loadable. Metadata counts toward the config blob size limit. + ## Transport framing The transport layer is at most 2 MiB. Integers are unsigned and little-endian. diff --git a/docs/snapshot-versioning.md b/docs/snapshot-versioning.md index f78ba7771..368b39e9f 100644 --- a/docs/snapshot-versioning.md +++ b/docs/snapshot-versioning.md @@ -50,6 +50,9 @@ The config blob also records `hyperlight_version`, the `CARGO_PKG_VERSION` of the host crate at write time. This is informational only. The loader records it for diagnostics and does not gate loading on it. +The optional `metadata` field stores application-owned values by namespace. +Writers omit an empty map, and readers treat a missing field as empty. + ## Compatibility cleanup Record compatibility paths here when a future hard snapshot break can remove diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/config.rs b/src/hyperlight_host/src/sandbox/snapshot/file/config.rs index bfae789b9..92978fd35 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/config.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/config.rs @@ -1,6 +1,8 @@ // SPDX-License-Identifier: Apache-2.0 // Copyright 2025 The Hyperlight Authors. +use std::collections::BTreeMap; + use hyperlight_common::flatbuffer_wrappers::function_types::{ParameterType, ReturnType}; use hyperlight_common::flatbuffer_wrappers::host_function_definition::HostFunctionDefinition; use hyperlight_common::vmem::PAGE_SIZE; @@ -199,6 +201,9 @@ pub(super) struct OciSnapshotConfig { /// `SCRATCH_TOP_SNAPSHOT_GENERATION_OFFSET` is continuous across /// save/load. pub(super) snapshot_generation: u64, + /// Application-owned metadata keyed by namespace. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub(super) metadata: BTreeMap, } /// Sizes and permissions of the regions inside the snapshot blob, @@ -852,6 +857,7 @@ mod tests { memory_size: PAGE_SIZE as u64, host_functions: Vec::new(), snapshot_generation: 0, + metadata: BTreeMap::new(), } } diff --git a/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs b/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs index d2b85af1e..d69c954f5 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file/mod.rs @@ -497,7 +497,7 @@ impl Snapshot { cfg: &OciSnapshotConfig, cfg_bytes: &[u8], ) -> crate::Result { - let memory_bytes = self.memory.as_slice(); + let memory_bytes = self.state.memory.as_slice(); let memory_size = memory_bytes.len(); if memory_size == 0 || !memory_size.is_multiple_of(PAGE_SIZE) { return Err(crate::new_error!( @@ -516,7 +516,7 @@ impl Snapshot { put_blob_if_absent(&blobs_dir, &snapshot_digest, memory_bytes)?; // Transport blob: the canonical ring image omitted from memory. - let transport = self.virtq.as_ref().ok_or_else(|| { + let transport = self.state.virtq.as_ref().ok_or_else(|| { crate::new_error!("initialized snapshot has no canonical transport state") })?; let transport_bytes = transport::encode(transport)?; @@ -585,7 +585,7 @@ impl Snapshot { } fn build_config(&self) -> crate::Result { - let (entrypoint_addr, sregs) = match (self.next_action, self.sregs.as_ref()) { + let (entrypoint_addr, sregs) = match (self.state.next_action, self.state.sregs.as_ref()) { (NextAction::Call(addr), Some(sregs)) => (addr, sregs), (NextAction::Call(_), None) => { return Err(crate::new_error!( @@ -605,18 +605,18 @@ impl Snapshot { } }; - if self.virtq.is_none() { + if self.state.virtq.is_none() { return Err(crate::new_error!( "initialized snapshot has no canonical transport state" )); } - let host_functions = match &self.host_functions.host_functions { + let host_functions = match &self.state.host_functions.host_functions { Some(v) => v.iter().map(HostFunction::from).collect(), None => Vec::new(), }; - let l = &self.layout; + let l = &self.state.layout; Ok(OciSnapshotConfig { hyperlight_version: env!("CARGO_PKG_VERSION").to_string(), arch: Arch::current(), @@ -624,12 +624,13 @@ impl Snapshot { hypervisor: Hypervisor::current() .ok_or_else(|| crate::new_error!("no hypervisor available to tag snapshot"))?, cpu_vendor: CpuVendor::current(), - stack_top_gva: self.stack_top_gva, + stack_top_gva: self.state.stack_top_gva, entrypoint_addr, - original_entrypoint_addr: self.original_entrypoint, + original_entrypoint_addr: self.state.original_entrypoint, sregs: *sregs, #[cfg(target_arch = "x86_64")] msrs: self + .state .msrs .as_ref() .ok_or_else(|| crate::new_error!("snapshot has no MSR state"))? @@ -650,9 +651,10 @@ impl Snapshot { snapshot_size: l.snapshot_size(), pt_size: l.pt_size(), }, - memory_size: self.memory.mem_size() as u64, + memory_size: self.state.memory.mem_size() as u64, host_functions, - snapshot_generation: self.snapshot_generation, + snapshot_generation: self.state.snapshot_generation, + metadata: self.metadata.clone(), }) } @@ -957,18 +959,21 @@ impl Snapshot { }; Ok(Snapshot { - layout, - memory, - load_info: crate::mem::exe::LoadInfo::dummy(), - stack_top_gva: cfg.stack_top_gva, - sregs: Some(cfg.sregs), - #[cfg(target_arch = "x86_64")] - msrs: Some(cfg.msrs), - next_action, - original_entrypoint: cfg.original_entrypoint_addr, - snapshot_generation, - host_functions, - virtq: Some(virtq), + state: std::sync::Arc::new(super::SnapshotState { + layout, + memory, + load_info: crate::mem::exe::LoadInfo::dummy(), + stack_top_gva: cfg.stack_top_gva, + sregs: Some(cfg.sregs), + #[cfg(target_arch = "x86_64")] + msrs: Some(cfg.msrs), + next_action, + original_entrypoint: cfg.original_entrypoint_addr, + snapshot_generation, + host_functions, + virtq: Some(virtq), + }), + metadata: cfg.metadata, }) } } diff --git a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs index 3cec4724c..a6df4ba1a 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/file_tests.rs @@ -8,6 +8,7 @@ use std::sync::Arc; use hyperlight_testing::{c_simple_guest_as_pathbuf, simple_guest_as_pathbuf}; +use serde::{Deserialize, Serialize}; use serde_json::Value; use sha2::{Digest as _, Sha256}; @@ -50,6 +51,12 @@ fn create_snapshot() -> Arc { sbox.snapshot().unwrap() } +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +struct TestMetadata { + format_version: u32, + runtime: String, +} + /// `Result::unwrap_err` requires `T: Debug`, but `Snapshot` is not /// `Debug`. This wrapper is the test-side equivalent. #[track_caller] @@ -143,6 +150,45 @@ fn from_snapshot_in_memory_pre_init() { assert_eq!(result, 0); } +#[test] +fn snapshot_metadata_is_immutable_and_shares_state() { + let snapshot = create_snapshot(); + let metadata = TestMetadata { + format_version: 1, + runtime: "test".to_string(), + }; + + let with_metadata = snapshot + .with_metadata("snapshot-metadata-namespace-v1", &metadata) + .unwrap(); + + assert!(Arc::ptr_eq(&snapshot.state, &with_metadata.state)); + assert_eq!( + snapshot + .metadata::("snapshot-metadata-namespace-v1") + .unwrap(), + None + ); + assert_eq!( + with_metadata + .metadata::("snapshot-metadata-namespace-v1") + .unwrap(), + Some(metadata) + ); +} + +#[test] +fn snapshot_metadata_namespaces_are_independent() { + let snapshot = create_snapshot(); + let first = snapshot.with_metadata("first", &1_u32).unwrap(); + let second = first.with_metadata("second", &2_u32).unwrap(); + let replaced = second.with_metadata("first", &3_u32).unwrap(); + + assert_eq!(second.metadata::("first").unwrap(), Some(1)); + assert_eq!(replaced.metadata::("first").unwrap(), Some(3)); + assert_eq!(replaced.metadata::("second").unwrap(), Some(2)); +} + // Round-trip via OCI layout on disk. #[test] @@ -163,6 +209,59 @@ fn round_trip_save_load_call() { assert_eq!(result, "hello\n"); } +#[test] +fn snapshot_metadata_round_trip() { + let metadata = TestMetadata { + format_version: 1, + runtime: "test".to_string(), + }; + let snapshot = create_snapshot() + .with_metadata("snapshot-metadata-namespace-v1", &metadata) + .unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("metadata"); + + snapshot + .save(&path, &OciTag::new("latest").unwrap()) + .unwrap(); + let config: Value = + serde_json::from_slice(&std::fs::read(find_config_blob(&path)).unwrap()).unwrap(); + assert_eq!( + config["metadata"], + serde_json::json!({ + "snapshot-metadata-namespace-v1": { + "format_version": 1, + "runtime": "test", + } + }) + ); + let loaded = Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap(); + + assert_eq!( + loaded + .metadata::("snapshot-metadata-namespace-v1") + .unwrap(), + Some(metadata) + ); +} + +#[test] +fn snapshot_without_metadata_omits_config_field() { + let snapshot = create_snapshot(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("metadata"); + + snapshot + .save(&path, &OciTag::new("latest").unwrap()) + .unwrap(); + let config: Value = + serde_json::from_slice(&std::fs::read(find_config_blob(&path)).unwrap()).unwrap(); + let loaded = Snapshot::checked_load(&path, OciTag::new("latest").unwrap()).unwrap(); + + assert!(config.get("metadata").is_none()); + assert_eq!(loaded.metadata::("missing").unwrap(), None); +} + /// A pre-existing snapshot blob with the right length but wrong /// bytes (corruption, partial copy, foreign tool) must be detected /// and replaced by `save`, not silently trusted. @@ -438,7 +537,8 @@ fn restore_from_loaded_snapshot() { fn restore_missing_transport_preserves_target() { // Remove transport from a snapshot with valid memory and vCPU state. let mut bad_snapshot = create_snapshot(); - Arc::get_mut(&mut bad_snapshot).unwrap().virtq = None; + let snapshot = Arc::get_mut(&mut bad_snapshot).unwrap(); + Arc::get_mut(&mut snapshot.state).unwrap().virtq = None; // Seed guest state and read the mapped file before caching the snapshot. let file = tempfile::NamedTempFile::new().unwrap(); @@ -3423,7 +3523,7 @@ fn save_new_tag_into_loaded_layout_preserves_live_mapping() { // Record the full mapped image and every on-disk blob before the // second save, so any byte change is caught. - let mapping_before = loaded_a.memory.as_slice().to_vec(); + let mapping_before = loaded_a.memory().as_slice().to_vec(); let blobs_dir = path.join("blobs").join("sha256"); let blobs_before = read_blob_dir(&blobs_dir); @@ -3436,7 +3536,7 @@ fn save_new_tag_into_loaded_layout_preserves_live_mapping() { // The live mapping is unchanged, byte for byte. assert_eq!( - loaded_a.memory.as_slice(), + loaded_a.memory().as_slice(), mapping_before.as_slice(), "live snapshot mapping changed after a new tag was written" ); diff --git a/src/hyperlight_host/src/sandbox/snapshot/mod.rs b/src/hyperlight_host/src/sandbox/snapshot/mod.rs index e7c1dfc16..63636a9c5 100644 --- a/src/hyperlight_host/src/sandbox/snapshot/mod.rs +++ b/src/hyperlight_host/src/sandbox/snapshot/mod.rs @@ -6,6 +6,7 @@ mod file_tests; mod tripwires; use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; pub(crate) use file::host_cpu_vendor_golden_tag; pub use file::reference::{OciDigest, OciReference, OciTag}; @@ -55,9 +56,15 @@ pub enum NextAction { None, } -/// A wrapper around a `SharedMemory` reference and a snapshot -/// of the memory therein +/// An immutable snapshot of sandbox state. pub struct Snapshot { + state: Arc, + // Stable key order keeps config bytes deterministic. + metadata: BTreeMap, +} + +/// Immutable sandbox state held by a snapshot. +struct SnapshotState { /// Layout object for the sandbox. TODO: get rid of this and /// replace with something saner and set up from the guest (early /// on?). @@ -118,6 +125,7 @@ pub struct Snapshot { /// Both the images and layout remain immutable afterwards. virtq: Option, } + impl core::convert::AsRef for Snapshot { fn as_ref(&self) -> &Self { self @@ -130,7 +138,7 @@ impl hyperlight_common::vmem::TableReadOps for Snapshot { } unsafe fn read_entry(&self, addr: u64) -> vmem::PageTableEntry { let addr = addr as usize; - let Some(pte_bytes) = self.memory.as_slice().get(addr..addr + PTE_SIZE) else { + let Some(pte_bytes) = self.state.memory.as_slice().get(addr..addr + PTE_SIZE) else { // Attacker-controlled data pointed out-of-bounds. We'll // default to returning 0 in this case, which, for most // architectures (including x86-64 and arm64, the ones we @@ -403,20 +411,23 @@ impl Snapshot { .ok_or_else(|| crate::new_error!("ELF entrypoint GVA overflows"))?; Ok(Self { - memory: ReadonlySharedMemory::from_bytes(&memory, layout.snapshot_size())?, - layout, - load_info, - stack_top_gva: exn_stack_top_gva, - sregs: None, - #[cfg(target_arch = "x86_64")] - msrs: None, - next_action: NextAction::Initialise(entrypoint_gva), - original_entrypoint: entrypoint_gva, - snapshot_generation: 0, - host_functions: HostFunctionDetails { - host_functions: None, - }, - virtq: None, + state: Arc::new(SnapshotState { + memory: ReadonlySharedMemory::from_bytes(&memory, layout.snapshot_size())?, + layout, + load_info, + stack_top_gva: exn_stack_top_gva, + sregs: None, + #[cfg(target_arch = "x86_64")] + msrs: None, + next_action: NextAction::Initialise(entrypoint_gva), + original_entrypoint: entrypoint_gva, + snapshot_generation: 0, + host_functions: HostFunctionDetails { + host_functions: None, + }, + virtq: None, + }), + metadata: BTreeMap::new(), }) } @@ -595,47 +606,137 @@ impl Snapshot { layout.set_snapshot_size(guest_visible_size); Ok(Self { - layout, - memory: ReadonlySharedMemory::from_bytes(&memory, guest_visible_size)?, - load_info, - stack_top_gva, - sregs: Some(sregs), - #[cfg(target_arch = "x86_64")] - msrs: Some(msrs), - next_action, - original_entrypoint, - snapshot_generation, - host_functions, - virtq, + state: Arc::new(SnapshotState { + layout, + memory: ReadonlySharedMemory::from_bytes(&memory, guest_visible_size)?, + load_info, + stack_top_gva, + sregs: Some(sregs), + #[cfg(target_arch = "x86_64")] + msrs: Some(msrs), + next_action, + original_entrypoint, + snapshot_generation, + host_functions, + virtq, + }), + metadata: BTreeMap::new(), }) } + /// Deserializes the metadata stored under `namespace` as `T`. + /// + /// Returns `None` if the namespace has no metadata. + /// + /// # Errors + /// + /// Returns an error if the JSON metadata cannot be deserialized as `T`. + /// + /// # Examples + /// + /// ```no_run + /// # use std::sync::Arc; + /// # use hyperlight_host::sandbox::snapshot::Snapshot; + /// # use serde::{Deserialize, Serialize}; + /// # + /// #[derive(Debug, PartialEq, Serialize, Deserialize)] + /// struct Metadata { + /// version: u32, + /// } + /// + /// # fn example(snapshot: Arc) -> Result<(), Box> { + /// let snapshot = + /// snapshot.with_metadata("snapshot-metadata-namespace-v1", &Metadata { version: 1 })?; + /// let metadata = snapshot + /// .metadata::("snapshot-metadata-namespace-v1")? + /// .expect("metadata should exist"); + /// assert_eq!(metadata, Metadata { version: 1 }); + /// # Ok(()) + /// # } + /// ``` + pub fn metadata(&self, namespace: &str) -> Result> + where + T: serde::de::DeserializeOwned, + { + self.metadata + .get(namespace) + .map(serde::Deserialize::deserialize) + .transpose() + .map_err(Into::into) + } + + /// Adds `metadata` to this snapshot and returns the result as a new + /// snapshot. Metadata already stored under `namespace` is replaced. + /// + /// This snapshot remains unchanged. + /// Metadata is saved and loaded with the snapshot. + /// + /// # Errors + /// + /// Returns an error if `metadata` cannot be serialized as JSON. + /// + /// # Examples + /// + /// ```no_run + /// # use std::sync::Arc; + /// # use hyperlight_host::sandbox::snapshot::Snapshot; + /// # use serde::{Deserialize, Serialize}; + /// # + /// #[derive(Debug, PartialEq, Serialize, Deserialize)] + /// struct Metadata { + /// version: u32, + /// } + /// + /// # fn example(snapshot: Arc) -> Result<(), Box> { + /// let snapshot = + /// snapshot.with_metadata("snapshot-metadata-namespace-v1", &Metadata { version: 1 })?; + /// let metadata = snapshot + /// .metadata::("snapshot-metadata-namespace-v1")? + /// .expect("metadata should exist"); + /// assert_eq!(metadata, Metadata { version: 1 }); + /// # Ok(()) + /// # } + /// ``` + pub fn with_metadata(&self, namespace: impl Into, metadata: &T) -> Result> + where + T: serde::Serialize, + { + let namespace = namespace.into(); + let metadata = serde_json::to_value(metadata)?; + let mut metadata_by_namespace = self.metadata.clone(); + metadata_by_namespace.insert(namespace, metadata); + Ok(Arc::new(Self { + state: Arc::clone(&self.state), + metadata: metadata_by_namespace, + })) + } + /// Generation number assigned to this snapshot when it was taken. pub(crate) fn snapshot_generation(&self) -> u64 { - self.snapshot_generation + self.state.snapshot_generation } /// Return the main memory contents of the snapshot #[instrument(skip_all, parent = Span::current(), level= "Trace")] pub(crate) fn memory(&self) -> &ReadonlySharedMemory { - &self.memory + &self.state.memory } /// Return a copy of the load info for the exe in the snapshot pub(crate) fn load_info(&self) -> LoadInfo { - self.load_info.clone() + self.state.load_info.clone() } pub(crate) fn layout(&self) -> &crate::mem::layout::SandboxMemoryLayout { - &self.layout + &self.state.layout } pub(crate) fn root_pt_gpa(&self) -> u64 { - self.layout.get_pt_base_gpa() + self.state.layout.get_pt_base_gpa() } pub(crate) fn stack_top_gva(&self) -> u64 { - self.stack_top_gva + self.state.stack_top_gva } /// Returns the special registers stored in this snapshot. @@ -644,28 +745,28 @@ impl Snapshot { /// Note: The CR3 value in the returned struct should NOT be used for restore; /// use `root_pt_gpa()` instead since page tables are relocated during snapshot. pub(crate) fn sregs(&self) -> Option<&CommonSpecialRegisters> { - self.sregs.as_ref() + self.state.sregs.as_ref() } /// The MSRs saved in this snapshot. #[cfg(target_arch = "x86_64")] pub(crate) fn msrs(&self) -> Option<&Vec> { - self.msrs.as_ref() + self.state.msrs.as_ref() } pub(crate) fn next_action(&self) -> NextAction { - self.next_action + self.state.next_action } pub(crate) fn virtq(&self) -> Option<&VirtqSnapshot> { - self.virtq.as_ref() + self.state.virtq.as_ref() } /// Guest virtual address of the guest binary's ELF entry point, /// preserved across the `Initialise` -> `Call` transition. Used /// to fill `AT_ENTRY` in guest core dumps. 0 if unknown. pub(crate) fn original_entrypoint(&self) -> u64 { - self.original_entrypoint + self.state.original_entrypoint } /// Validate that `provided` is a superset of the host functions @@ -680,7 +781,7 @@ impl Snapshot { &self, provided: &crate::sandbox::host_funcs::FunctionRegistry, ) -> Result<()> { - let required = match &self.host_functions.host_functions { + let required = match &self.state.host_functions.host_functions { Some(v) => v, None => return Ok(()), };