From a9dbf3162290fc92a8abd2d6a5a3246b1676afc4 Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> Date: Tue, 11 Aug 2026 14:57:48 -0700 Subject: [PATCH 1/2] Reset xcr0 on restore Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> --- CHANGELOG.md | 1 + .../src/hypervisor/hyperlight_vm/x86_64.rs | 8 ++- .../hypervisor/virtual_machine/kvm/x86_64.rs | 31 ++++++++++++ .../src/hypervisor/virtual_machine/mod.rs | 18 +++++++ .../hypervisor/virtual_machine/mshv/x86_64.rs | 16 +++++- .../src/hypervisor/virtual_machine/whp.rs | 17 +++++++ .../src/sandbox/initialized_multi_use.rs | 24 +++++++++ src/tests/rust_guests/simpleguest/src/main.rs | 50 +++++++++++++++++++ 8 files changed, 163 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0546852eb..516928815 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Fixed * Fix symbol resolution in guest core dumps for sandboxes created from snapshots by @ludfjig in https://github.com/hyperlight-dev/hyperlight/pull/1618 * Reject malformed OCI snapshot metadata and non-regular artifact files during load. +* Reset XCR0 during x86 snapshot restore. ## [v0.16.0] - 2026-06-26 diff --git a/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs b/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs index 372483e3c..49abad98a 100644 --- a/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/hyperlight_vm/x86_64.rs @@ -48,7 +48,7 @@ use crate::hypervisor::virtual_machine::mshv::MshvVm; #[cfg(target_os = "windows")] use crate::hypervisor::virtual_machine::whp::WhpVm; use crate::hypervisor::virtual_machine::{ - HypervisorType, RegisterError, VmError, get_available_hypervisor, + HypervisorType, RegisterError, VmError, XCR0_RESET, get_available_hypervisor, }; #[cfg(target_os = "windows")] use crate::hypervisor::{PartitionState, WindowsInterruptHandle}; @@ -371,6 +371,7 @@ impl HyperlightVm { /// - General purpose registers /// - Debug registers /// - XSAVE (includes FPU/SSE state with proper FCW and MXCSR defaults) + /// - XCR0 /// - Special registers (restored from snapshot, with CR3 updated to new page table location) // TODO: check if other state needs to be reset pub(crate) fn reset_vcpu( @@ -384,6 +385,7 @@ impl HyperlightVm { })?; self.vm.set_debug_regs(&CommonDebugRegs::default())?; self.vm.reset_xsave()?; + self.vm.set_xcr0(XCR0_RESET)?; self.apply_sregs(cr3, sregs)?; @@ -1625,6 +1627,7 @@ mod tests { let xsave = dirty_xsave(¤t_xsave); let debug_regs = dirty_debug_regs(); + hyperlight_vm.vm.set_xcr0(3).unwrap(); hyperlight_vm.vm.set_xsave(&xsave).unwrap(); hyperlight_vm.vm.set_regs(®s).unwrap(); hyperlight_vm.vm.set_fpu(&fpu).unwrap(); @@ -1675,6 +1678,7 @@ mod tests { let mut expected_sregs = sregs; normalize_sregs_hidden_cache(&mut expected_sregs, &got_sregs); assert_eq!(got_sregs, expected_sregs); + assert_eq!(hyperlight_vm.vm.xcr0().unwrap(), 3); // Reset the vCPU hyperlight_vm.reset_vcpu(0, &default_sregs()).unwrap(); @@ -1688,6 +1692,8 @@ mod tests { // Verify debug registers are reset to defaults assert_debug_regs_reset(hyperlight_vm.vm.as_ref()); + assert_eq!(hyperlight_vm.vm.xcr0().unwrap(), XCR0_RESET); + // Verify xsave is reset - should be zeroed except for hypervisor-specific fields let reset_xsave = hyperlight_vm.vm.xsave().unwrap(); // Build expected xsave: all zeros with fpu specific defaults. Then copy hypervisor-specific fields from actual diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs index 9a2cc254f..4343a0ee1 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/kvm/x86_64.rs @@ -627,6 +627,37 @@ impl VirtualMachine for KvmVm { Ok(()) } + #[cfg(test)] + fn xcr0(&self) -> std::result::Result { + let xcrs = self + .vcpu_fd + .get_xcrs() + .map_err(|e| RegisterError::GetXcrs(e.into()))?; + xcrs.xcrs + .iter() + .take(xcrs.nr_xcrs as usize) + .find(|xcr| xcr.xcr == 0) + .map(|xcr| xcr.value) + .ok_or(RegisterError::MissingXcr0) + } + + fn set_xcr0(&self, value: u64) -> std::result::Result<(), RegisterError> { + let mut xcrs = self + .vcpu_fd + .get_xcrs() + .map_err(|e| RegisterError::GetXcrs(e.into()))?; + let xcr0 = xcrs + .xcrs + .iter_mut() + .take(xcrs.nr_xcrs as usize) + .find(|xcr| xcr.xcr == 0) + .ok_or(RegisterError::MissingXcr0)?; + xcr0.value = value; + self.vcpu_fd + .set_xcrs(&xcrs) + .map_err(|e| RegisterError::SetXcrs(e.into())) + } + #[cfg(test)] fn set_xsave(&self, xsave: &[u32]) -> std::result::Result<(), RegisterError> { if std::mem::size_of_val(xsave) != XSAVE_BUFFER_SIZE { diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs index a909e3ce3..d0c613e04 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/mod.rs @@ -121,6 +121,10 @@ pub(crate) const XSAVE_MIN_SIZE: usize = 576; #[cfg(all(any(kvm, mshv3), test, not(target_arch = "aarch64")))] pub(crate) const XSAVE_BUFFER_SIZE: usize = 4096; +/// Architectural XCR0 reset value. Only x87 state is enabled. +#[cfg(target_arch = "x86_64")] +pub(crate) const XCR0_RESET: u64 = 1; + // Compiler error if no hypervisor type is available (not applicable on aarch64 yet) #[cfg(not(any(kvm, mshv3, target_os = "windows", target_arch = "aarch64")))] compile_error!( @@ -271,6 +275,15 @@ pub enum RegisterError { GetXsave(HypervisorError), #[error("Failed to set xsave: {0}")] SetXsave(HypervisorError), + #[cfg(target_arch = "x86_64")] + #[error("Failed to get XCRs: {0}")] + GetXcrs(HypervisorError), + #[cfg(target_arch = "x86_64")] + #[error("Failed to set XCRs: {0}")] + SetXcrs(HypervisorError), + #[cfg(target_arch = "x86_64")] + #[error("Hypervisor did not return XCR0")] + MissingXcr0, #[error("Xsave size mismatch: expected {expected} bytes, got {actual}")] XsaveSizeMismatch { /// Expected size in bytes @@ -447,6 +460,11 @@ pub(crate) trait VirtualMachine: Debug + Send { #[cfg(not(target_arch = "aarch64"))] fn set_xsave(&self, xsave: &[u32]) -> std::result::Result<(), RegisterError>; + #[cfg(all(test, target_arch = "x86_64"))] + fn xcr0(&self) -> std::result::Result; + #[cfg(target_arch = "x86_64")] + fn set_xcr0(&self, value: u64) -> std::result::Result<(), RegisterError>; + /// Single-operation vCPU reset #[cfg(target_arch = "aarch64")] fn can_reset_vcpu(&self) -> bool { diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs index 1865f7f0e..5c6232089 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/mshv/x86_64.rs @@ -26,7 +26,7 @@ use mshv_bindings::LapicState; #[cfg(gdb)] use mshv_bindings::{DebugRegisters, hv_message_type_HVMSG_X64_EXCEPTION_INTERCEPT}; use mshv_bindings::{ - FloatingPointUnit, HV_X64_REGISTER_CLASS_IP, SpecialRegisters, StandardRegisters, XSave, + FloatingPointUnit, HV_X64_REGISTER_CLASS_IP, SpecialRegisters, StandardRegisters, XSave, Xcrs, hv_message_type, hv_message_type_HVMSG_GPA_INTERCEPT, hv_message_type_HVMSG_UNMAPPED_GPA, hv_message_type_HVMSG_X64_HALT, hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT, hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES, @@ -612,6 +612,20 @@ impl VirtualMachine for MshvVm { Ok(()) } + #[cfg(test)] + fn xcr0(&self) -> std::result::Result { + self.vcpu_fd + .get_xcrs() + .map(|xcrs| xcrs.xcr0) + .map_err(|e| RegisterError::GetXcrs(e.into())) + } + + fn set_xcr0(&self, value: u64) -> std::result::Result<(), RegisterError> { + self.vcpu_fd + .set_xcrs(&Xcrs { xcr0: value }) + .map_err(|e| RegisterError::SetXcrs(e.into())) + } + #[cfg(test)] fn set_xsave(&self, xsave: &[u32]) -> std::result::Result<(), RegisterError> { if std::mem::size_of_val(xsave) != XSAVE_BUFFER_SIZE { diff --git a/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs b/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs index b60777fcc..22cc967be 100644 --- a/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs +++ b/src/hyperlight_host/src/hypervisor/virtual_machine/whp.rs @@ -1002,6 +1002,23 @@ impl VirtualMachine for WhpVm { Ok(()) } + #[cfg(test)] + fn xcr0(&self) -> std::result::Result { + let mut values = [Align16(WHV_REGISTER_VALUE::default())]; + self.get_registers(&[WHvX64RegisterXCr0], &mut values) + .map_err(|e| RegisterError::GetXcrs(e.into()))?; + // SAFETY: WHP populated the value for the requested 64-bit register. + Ok(unsafe { values[0].0.Reg64 }) + } + + fn set_xcr0(&self, value: u64) -> std::result::Result<(), RegisterError> { + self.set_registers(&[( + WHvX64RegisterXCr0, + Align16(WHV_REGISTER_VALUE { Reg64: value }), + )]) + .map_err(|e| RegisterError::SetXcrs(e.into())) + } + #[cfg(test)] fn set_xsave(&self, xsave: &[u32]) -> std::result::Result<(), RegisterError> { // Get the required buffer size by calling with NULL buffer. diff --git a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs index 623c24667..3ced8ab31 100644 --- a/src/hyperlight_host/src/sandbox/initialized_multi_use.rs +++ b/src/hyperlight_host/src/sandbox/initialized_multi_use.rs @@ -1944,6 +1944,30 @@ mod tests { ); } + #[test] + #[cfg(target_arch = "x86_64")] + fn snapshot_restore_resets_xcr0() { + let mut sandbox: MultiUseSandbox = { + let path = simple_guest_as_pathbuf(); + let u_sbox = UninitializedSandbox::new(GuestBinary::FilePath(path), None).unwrap(); + u_sbox.evolve().unwrap() + }; + + assert_eq!(sandbox.call::("ReadXcr0", ()).unwrap(), 1); + let snapshot = sandbox.snapshot().unwrap(); + + sandbox.call::<()>("WriteXcr0", 3u64).unwrap(); + assert_eq!(sandbox.call::("ReadXcr0", ()).unwrap(), 3); + + sandbox.restore(snapshot).unwrap(); + + assert_eq!( + sandbox.call::("ReadXcr0", ()).unwrap(), + 1, + "restore must reset XCR0" + ); + } + /// Test that stale abort buffer bytes from a previous call don't /// leak into the next call. #[test] diff --git a/src/tests/rust_guests/simpleguest/src/main.rs b/src/tests/rust_guests/simpleguest/src/main.rs index fc0ce416b..17aec9b22 100644 --- a/src/tests/rust_guests/simpleguest/src/main.rs +++ b/src/tests/rust_guests/simpleguest/src/main.rs @@ -900,6 +900,56 @@ fn get_dr0() -> u64 { value } +#[guest_function("ReadXcr0")] +#[cfg(target_arch = "x86_64")] +fn read_xcr0() -> u64 { + let value_low: u32; + let value_high: u32; + // SAFETY: The test guest runs at CPL0. CR4 is restored before returning. + unsafe { + core::arch::asm!( + "mov {original_cr4}, cr4", + "mov {enabled_cr4}, {original_cr4}", + "or {enabled_cr4}, {osxsave}", + "mov cr4, {enabled_cr4}", + "xgetbv", + "mov cr4, {original_cr4}", + original_cr4 = out(reg) _, + enabled_cr4 = out(reg) _, + osxsave = const 1u64 << 18, + in("ecx") 0u32, + out("eax") value_low, + out("edx") value_high, + options(nostack, nomem) + ); + } + ((value_high as u64) << 32) | value_low as u64 +} + +#[guest_function("WriteXcr0")] +#[cfg(target_arch = "x86_64")] +fn write_xcr0(value: u64) { + // SAFETY: The test guest runs at CPL0. The caller supplies a valid XCR0 + // value, and CR4 is restored before returning. + unsafe { + core::arch::asm!( + "mov {original_cr4}, cr4", + "mov {enabled_cr4}, {original_cr4}", + "or {enabled_cr4}, {osxsave}", + "mov cr4, {enabled_cr4}", + "xsetbv", + "mov cr4, {original_cr4}", + original_cr4 = out(reg) _, + enabled_cr4 = out(reg) _, + osxsave = const 1u64 << 18, + in("ecx") 0u32, + in("eax") value as u32, + in("edx") (value >> 32) as u32, + options(nostack, nomem) + ); + } +} + #[guest_function("Add")] fn add(a: i32, b: i32) -> Result { #[host_function("HostAdd")] From d716b5b4f4cad6c2eb30882b1728b38dfbdebbb3 Mon Sep 17 00:00:00 2001 From: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> Date: Tue, 11 Aug 2026 16:50:31 -0700 Subject: [PATCH 2/2] Fix failing test due to increased memory usage Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com> --- src/hyperlight_host/tests/integration_test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyperlight_host/tests/integration_test.rs b/src/hyperlight_host/tests/integration_test.rs index b449ea68d..24db9134a 100644 --- a/src/hyperlight_host/tests/integration_test.rs +++ b/src/hyperlight_host/tests/integration_test.rs @@ -535,7 +535,7 @@ fn guest_malloc_abort() { }); // allocate a vector (on heap) that is bigger than the heap - let heap_size = 0x6000; + let heap_size = 0x8000; let size_to_allocate = 0x10000; assert!( size_to_allocate > heap_size, @@ -616,7 +616,7 @@ fn corrupt_output_back_pointer_rejected() { #[test] fn guest_panic_no_alloc() { - let heap_size = 0x6000; + let heap_size = 0x8000; let mut cfg = SandboxConfiguration::default(); cfg.set_heap_size(heap_size);