From 92b32a46acb7d6ca96ff769e314e4f402205d058 Mon Sep 17 00:00:00 2001 From: Simon Davies <=simongdavies@users.noreply.github.com> Date: Mon, 11 Nov 2024 22:40:59 +0000 Subject: [PATCH 01/25] Adds support for Azure 3 Signed-off-by: Simon Davies --- Cargo.toml | 2 - src/hyperlight_host/Cargo.toml | 11 ++- src/hyperlight_host/build.rs | 8 +- src/hyperlight_host/src/error.rs | 6 ++ .../src/hypervisor/hyperv_linux.rs | 45 +++++++++-- src/hyperlight_host/src/mem/memory_region.rs | 74 ++++++++++++++----- 6 files changed, 115 insertions(+), 31 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9e3d0b3c..52359d14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,8 +33,6 @@ repository = "https://github.com/hyperlight-dev/hyperlight" readme = "README.md" [workspace.dependencies] -mshv-bindings = { version = "=0.2.1" } -mshv-ioctls = { version = "=0.2.1" } hyperlight-common = { path = "src/hyperlight_common", version = "0.1.0", default-features = false } hyperlight-host = { path = "src/hyperlight_host", version = "0.1.0", default-features = false } diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 72fe798e..69568999 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -71,10 +71,12 @@ windows-version = "0.1" [target.'cfg(unix)'.dependencies] seccompiler = { version = "0.4.0", optional = true } -mshv-bindings = { workspace = true, optional = true } -mshv-ioctls = { workspace = true, optional = true } kvm-bindings = { version = "0.10.0", features = ["fam-wrappers"], optional = true } kvm-ioctls = { version = "0.19.1", optional = true } +mshv-bindings2 = { package="mshv-bindings", version = "=0.2.1", optional = true } +mshv-ioctls2 = { package="mshv-ioctls", version = "=0.2.1", optional = true} +mshv-bindings3 = { package="mshv-bindings", version = "0.3.2", optional = true } +mshv-ioctls3 = { package="mshv-ioctls", version = "0.3.2", optional = true} [dev-dependencies] uuid = { version = "1.4.1", features = ["v4"] } @@ -114,7 +116,7 @@ cfg_aliases = "0.2.1" built = { version = "0.7.0", features = ["chrono", "git2"] } [features] -default = ["kvm", "mshv", "seccomp"] +default = ["kvm", "mshv2", "seccomp"] seccomp = ["dep:seccompiler"] function_call_metrics = [] executable_heap = [] @@ -122,7 +124,8 @@ executable_heap = [] print_debug = [] crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds. kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"] -mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"] +mshv2 = ["mshv-bindings2", "mshv-ioctls2"] +mshv3 = ["mshv-bindings3", "mshv-ioctls3"] inprocess = [] [[bench]] diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index 2aa321e7..62f8d1b3 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -18,6 +18,10 @@ use anyhow::Result; use built::write_built_file; fn main() -> Result<()> { + // mshv2 and mshv3 features are mutually exclusive. + #[cfg(all(feature = "mshv2", feature = "mshv3"))] + panic!("mshv2 and mshv3 features are mutually exclusive"); + // re-run the build if this script is changed (or deleted!), // even if the rust code is completely unchanged. println!("cargo:rerun-if-changed=build.rs"); @@ -85,12 +89,12 @@ fn main() -> Result<()> { } // Makes #[cfg(kvm)] == #[cfg(all(feature = "kvm", target_os = "linux"))] - // and #[cfg(mshv)] == #[cfg(all(feature = "mshv", target_os = "linux"))]. + // and #[cfg(mshv)] == #[cfg(all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux"))]. // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")]. // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase. cfg_aliases::cfg_aliases! { kvm: { all(feature = "kvm", target_os = "linux") }, - mshv: { all(feature = "mshv", target_os = "linux") }, + mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") }, // inprocess feature is aliased with debug_assertions to make it only available in debug-builds. // You should never use #[cfg(feature = "inprocess")] in the codebase. Use #[cfg(inprocess)] instead. inprocess: { all(feature = "inprocess", debug_assertions) }, diff --git a/src/hyperlight_host/src/error.rs b/src/hyperlight_host/src/error.rs index 3c0a84d5..d75063d0 100644 --- a/src/hyperlight_host/src/error.rs +++ b/src/hyperlight_host/src/error.rs @@ -14,6 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ +#[cfg(feature = "mshv2")] +extern crate mshv_ioctls2 as mshv_ioctls; + +#[cfg(feature = "mshv3")] +extern crate mshv_ioctls3 as mshv_ioctls; + use std::array::TryFromSliceError; use std::cell::{BorrowError, BorrowMutError}; use std::convert::Infallible; diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 224428eb..545fab38 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -14,16 +14,32 @@ See the License for the specific language governing permissions and limitations under the License. */ +#[cfg(feature = "mshv2")] +extern crate mshv_bindings2 as mshv_bindings; +#[cfg(feature = "mshv2")] +extern crate mshv_ioctls2 as mshv_ioctls; + +#[cfg(feature = "mshv3")] +extern crate mshv_bindings3 as mshv_bindings; +#[cfg(feature = "mshv3")] +extern crate mshv_ioctls3 as mshv_ioctls; + use std::fmt::{Debug, Formatter}; use log::error; +#[cfg(feature = "mshv2")] +use mshv_bindings::hv_message; use mshv_bindings::{ - hv_message, hv_message_type, hv_message_type_HVMSG_GPA_INTERCEPT, - hv_message_type_HVMSG_UNMAPPED_GPA, hv_message_type_HVMSG_X64_HALT, - hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT, hv_register_assoc, + hv_message_type, hv_message_type_HVMSG_GPA_INTERCEPT, hv_message_type_HVMSG_UNMAPPED_GPA, + hv_message_type_HVMSG_X64_HALT, hv_message_type_HVMSG_X64_IO_PORT_INTERCEPT, hv_register_assoc, hv_register_name_HV_X64_REGISTER_RIP, hv_register_value, mshv_user_mem_region, FloatingPointUnit, SegmentRegister, SpecialRegisters, StandardRegisters, }; +#[cfg(feature = "mshv3")] +use mshv_bindings::{ + hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES, + hv_partition_synthetic_processor_features, +}; use mshv_ioctls::{Mshv, VcpuFd, VmFd}; use tracing::{instrument, Span}; @@ -86,7 +102,19 @@ impl HypervLinuxDriver { ) -> Result { let mshv = Mshv::new()?; let pr = Default::default(); + #[cfg(feature = "mshv2")] let vm_fd = mshv.create_vm_with_config(&pr)?; + #[cfg(feature = "mshv3")] + let vm_fd = { + let vm_fd = mshv.create_vm_with_args(&pr)?; + let features: hv_partition_synthetic_processor_features = Default::default(); + vm_fd.hvcall_set_partition_property( + hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES, + unsafe { features.as_uint64[0] }, + )?; + vm_fd + }; + let mut vcpu_fd = vm_fd.create_vcpu(0)?; mem_regions.iter().try_for_each(|region| { @@ -280,8 +308,15 @@ impl Hypervisor for HypervLinuxDriver { const UNMAPPED_GPA_MESSAGE: hv_message_type = hv_message_type_HVMSG_UNMAPPED_GPA; const INVALID_GPA_ACCESS_MESSAGE: hv_message_type = hv_message_type_HVMSG_GPA_INTERCEPT; - let hv_message: hv_message = Default::default(); - let result = match &self.vcpu_fd.run(hv_message) { + #[cfg(feature = "mshv2")] + let run_result = { + let hv_message: hv_message = Default::default(); + &self.vcpu_fd.run(hv_message) + }; + #[cfg(feature = "mshv3")] + let run_result = &self.vcpu_fd.run(); + + let result = match run_result { Ok(m) => match m.header.message_type { HALT_MESSAGE => { crate::debug!("mshv - Halt Details : {:#?}", &self); diff --git a/src/hyperlight_host/src/mem/memory_region.rs b/src/hyperlight_host/src/mem/memory_region.rs index f8911ff6..c2cc210f 100644 --- a/src/hyperlight_host/src/mem/memory_region.rs +++ b/src/hyperlight_host/src/mem/memory_region.rs @@ -14,6 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ +#[cfg(feature = "mshv2")] +extern crate mshv_bindings2 as mshv_bindings; +#[cfg(feature = "mshv2")] +extern crate mshv_ioctls2 as mshv_ioctls; + +#[cfg(feature = "mshv3")] +extern crate mshv_bindings3 as mshv_bindings; +#[cfg(feature = "mshv3")] +extern crate mshv_ioctls3 as mshv_ioctls; + use std::ops::Range; use bitflags::bitflags; @@ -21,9 +31,14 @@ use bitflags::bitflags; use hyperlight_common::mem::PAGE_SHIFT; use hyperlight_common::mem::PAGE_SIZE_USIZE; #[cfg(mshv)] +use mshv_bindings::{hv_x64_memory_intercept_message, mshv_user_mem_region}; +#[cfg(feature = "mshv2")] +use mshv_bindings::{ + HV_MAP_GPA_EXECUTABLE, HV_MAP_GPA_PERMISSIONS_NONE, HV_MAP_GPA_READABLE, HV_MAP_GPA_WRITABLE, +}; +#[cfg(feature = "mshv3")] use mshv_bindings::{ - hv_x64_memory_intercept_message, mshv_user_mem_region, HV_MAP_GPA_EXECUTABLE, - HV_MAP_GPA_PERMISSIONS_NONE, HV_MAP_GPA_READABLE, HV_MAP_GPA_WRITABLE, + MSHV_SET_MEM_BIT_EXECUTABLE, MSHV_SET_MEM_BIT_UNMAP, MSHV_SET_MEM_BIT_WRITABLE, }; #[cfg(target_os = "windows")] use windows::Win32::System::Hypervisor::{self, WHV_MEMORY_ACCESS_TYPE}; @@ -227,22 +242,45 @@ impl From for mshv_user_mem_region { let guest_pfn = region.guest_region.start as u64 >> PAGE_SHIFT; let userspace_addr = region.host_region.start as u64; - let flags = region.flags.iter().fold(0, |acc, flag| { - let flag_value = match flag { - MemoryRegionFlags::NONE => HV_MAP_GPA_PERMISSIONS_NONE, - MemoryRegionFlags::READ => HV_MAP_GPA_READABLE, - MemoryRegionFlags::WRITE => HV_MAP_GPA_WRITABLE, - MemoryRegionFlags::EXECUTE => HV_MAP_GPA_EXECUTABLE, - _ => 0, // ignore any unknown flags - }; - acc | flag_value - }); - - mshv_user_mem_region { - guest_pfn, - size, - userspace_addr, - flags, + #[cfg(feature = "mshv2")] + { + let flags = region.flags.iter().fold(0, |acc, flag| { + let flag_value = match flag { + MemoryRegionFlags::NONE => HV_MAP_GPA_PERMISSIONS_NONE, + MemoryRegionFlags::READ => HV_MAP_GPA_READABLE, + MemoryRegionFlags::WRITE => HV_MAP_GPA_WRITABLE, + MemoryRegionFlags::EXECUTE => HV_MAP_GPA_EXECUTABLE, + _ => 0, // ignore any unknown flags + }; + acc | flag_value + }); + mshv_user_mem_region { + guest_pfn, + size, + userspace_addr, + flags, + } + } + #[cfg(feature = "mshv3")] + { + let flags: u8 = region.flags.iter().fold(0, |acc, flag| { + let flag_value = match flag { + MemoryRegionFlags::NONE => 1 << MSHV_SET_MEM_BIT_UNMAP, + MemoryRegionFlags::READ => 0, + MemoryRegionFlags::WRITE => 1 << MSHV_SET_MEM_BIT_WRITABLE, + MemoryRegionFlags::EXECUTE => 1 << MSHV_SET_MEM_BIT_EXECUTABLE, + _ => 0, // ignore any unknown flags + }; + acc | flag_value + }); + + mshv_user_mem_region { + guest_pfn, + size, + userspace_addr, + flags, + ..Default::default() + } } } } From d3506f525b1dfe2b2c46b2196dc10bbe3bad6b45 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Tue, 12 Nov 2024 17:28:41 +0000 Subject: [PATCH 02/25] Updates to make clippy happy Since clippy runs with --all-features and the mshv2 and mshv3 features are mutually exclusive these changes ensure that clippy is able to run and we can still enforce mutually exclusivity Signed-off-by: Simon Davies --- src/hyperlight_host/build.rs | 13 ++++++++---- src/hyperlight_host/src/error.rs | 4 ++-- .../src/hypervisor/hyperv_linux.rs | 20 +++++++++---------- src/hyperlight_host/src/mem/memory_region.rs | 16 +++++++-------- 4 files changed, 29 insertions(+), 24 deletions(-) diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index 62f8d1b3..4c5766f7 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -18,10 +18,6 @@ use anyhow::Result; use built::write_built_file; fn main() -> Result<()> { - // mshv2 and mshv3 features are mutually exclusive. - #[cfg(all(feature = "mshv2", feature = "mshv3"))] - panic!("mshv2 and mshv3 features are mutually exclusive"); - // re-run the build if this script is changed (or deleted!), // even if the rust code is completely unchanged. println!("cargo:rerun-if-changed=build.rs"); @@ -102,9 +98,18 @@ fn main() -> Result<()> { crashdump: { all(feature = "crashdump", debug_assertions) }, // print_debug feature is aliased with debug_assertions to make it only available in debug-builds. print_debug: { all(feature = "print_debug", debug_assertions) }, + // the following is a bit of a hack to stop clippy failing when run with -all features as it enables both mshv2 and mshv3 at the same time causing errors + mshv2: { all(feature = "mshv2", target_os = "linux", not(clippy)) }, + mshv3: { all(feature = "mshv3", target_os = "linux") }, } write_built_file()?; + // mshv2 and mshv3 features are mutually exclusive + #[cfg(all(feature = "mshv2", feature = "mshv3", not(clippy)))] + Err(anyhow::anyhow!( + "mshv2 and mshv3 features are mutually exclusive" + ))?; + Ok(()) } diff --git a/src/hyperlight_host/src/error.rs b/src/hyperlight_host/src/error.rs index d75063d0..368a126c 100644 --- a/src/hyperlight_host/src/error.rs +++ b/src/hyperlight_host/src/error.rs @@ -14,10 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -#[cfg(feature = "mshv2")] +#[cfg(mshv2)] extern crate mshv_ioctls2 as mshv_ioctls; -#[cfg(feature = "mshv3")] +#[cfg(mshv3)] extern crate mshv_ioctls3 as mshv_ioctls; use std::array::TryFromSliceError; diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 545fab38..d271c025 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -14,20 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -#[cfg(feature = "mshv2")] +#[cfg(mshv2)] extern crate mshv_bindings2 as mshv_bindings; -#[cfg(feature = "mshv2")] +#[cfg(mshv2)] extern crate mshv_ioctls2 as mshv_ioctls; -#[cfg(feature = "mshv3")] +#[cfg(mshv3)] extern crate mshv_bindings3 as mshv_bindings; -#[cfg(feature = "mshv3")] +#[cfg(mshv3)] extern crate mshv_ioctls3 as mshv_ioctls; use std::fmt::{Debug, Formatter}; use log::error; -#[cfg(feature = "mshv2")] +#[cfg(mshv2)] use mshv_bindings::hv_message; use mshv_bindings::{ hv_message_type, hv_message_type_HVMSG_GPA_INTERCEPT, hv_message_type_HVMSG_UNMAPPED_GPA, @@ -35,7 +35,7 @@ use mshv_bindings::{ hv_register_name_HV_X64_REGISTER_RIP, hv_register_value, mshv_user_mem_region, FloatingPointUnit, SegmentRegister, SpecialRegisters, StandardRegisters, }; -#[cfg(feature = "mshv3")] +#[cfg(mshv3)] use mshv_bindings::{ hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES, hv_partition_synthetic_processor_features, @@ -102,9 +102,9 @@ impl HypervLinuxDriver { ) -> Result { let mshv = Mshv::new()?; let pr = Default::default(); - #[cfg(feature = "mshv2")] + #[cfg(mshv2)] let vm_fd = mshv.create_vm_with_config(&pr)?; - #[cfg(feature = "mshv3")] + #[cfg(mshv3)] let vm_fd = { let vm_fd = mshv.create_vm_with_args(&pr)?; let features: hv_partition_synthetic_processor_features = Default::default(); @@ -308,12 +308,12 @@ impl Hypervisor for HypervLinuxDriver { const UNMAPPED_GPA_MESSAGE: hv_message_type = hv_message_type_HVMSG_UNMAPPED_GPA; const INVALID_GPA_ACCESS_MESSAGE: hv_message_type = hv_message_type_HVMSG_GPA_INTERCEPT; - #[cfg(feature = "mshv2")] + #[cfg(mshv2)] let run_result = { let hv_message: hv_message = Default::default(); &self.vcpu_fd.run(hv_message) }; - #[cfg(feature = "mshv3")] + #[cfg(mshv3)] let run_result = &self.vcpu_fd.run(); let result = match run_result { diff --git a/src/hyperlight_host/src/mem/memory_region.rs b/src/hyperlight_host/src/mem/memory_region.rs index c2cc210f..6956d963 100644 --- a/src/hyperlight_host/src/mem/memory_region.rs +++ b/src/hyperlight_host/src/mem/memory_region.rs @@ -14,14 +14,14 @@ See the License for the specific language governing permissions and limitations under the License. */ -#[cfg(feature = "mshv2")] +#[cfg(mshv2)] extern crate mshv_bindings2 as mshv_bindings; -#[cfg(feature = "mshv2")] +#[cfg(mshv2)] extern crate mshv_ioctls2 as mshv_ioctls; -#[cfg(feature = "mshv3")] +#[cfg(mshv3)] extern crate mshv_bindings3 as mshv_bindings; -#[cfg(feature = "mshv3")] +#[cfg(mshv3)] extern crate mshv_ioctls3 as mshv_ioctls; use std::ops::Range; @@ -32,11 +32,11 @@ use hyperlight_common::mem::PAGE_SHIFT; use hyperlight_common::mem::PAGE_SIZE_USIZE; #[cfg(mshv)] use mshv_bindings::{hv_x64_memory_intercept_message, mshv_user_mem_region}; -#[cfg(feature = "mshv2")] +#[cfg(mshv2)] use mshv_bindings::{ HV_MAP_GPA_EXECUTABLE, HV_MAP_GPA_PERMISSIONS_NONE, HV_MAP_GPA_READABLE, HV_MAP_GPA_WRITABLE, }; -#[cfg(feature = "mshv3")] +#[cfg(mshv3)] use mshv_bindings::{ MSHV_SET_MEM_BIT_EXECUTABLE, MSHV_SET_MEM_BIT_UNMAP, MSHV_SET_MEM_BIT_WRITABLE, }; @@ -242,7 +242,7 @@ impl From for mshv_user_mem_region { let guest_pfn = region.guest_region.start as u64 >> PAGE_SHIFT; let userspace_addr = region.host_region.start as u64; - #[cfg(feature = "mshv2")] + #[cfg(mshv2)] { let flags = region.flags.iter().fold(0, |acc, flag| { let flag_value = match flag { @@ -261,7 +261,7 @@ impl From for mshv_user_mem_region { flags, } } - #[cfg(feature = "mshv3")] + #[cfg(mshv3)] { let flags: u8 = region.flags.iter().fold(0, |acc, flag| { let flag_value = match flag { From 308dd4ea240fabd51a55053e3eb1ea1cab38e8f5 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Tue, 12 Nov 2024 18:44:57 +0000 Subject: [PATCH 03/25] update feature name in test Signed-off-by: Simon Davies --- Justfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Justfile b/Justfile index 31904a3c..cb289892 100644 --- a/Justfile +++ b/Justfile @@ -94,7 +94,7 @@ test-rust target=default-target features="": (test-rust-int "rust" target featur test-seccomp target=default-target: # run seccomp test with feature "seccomp" on and off cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib -- --ignored - cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features --features mshv,kvm --lib -- --ignored + cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features --features mshv2,kvm --lib -- --ignored # rust integration tests. guest can either be "rust" or "c" test-rust-int guest target=default-target features="": @@ -185,4 +185,4 @@ fuzz: cd src/hyperlight_host && cargo +nightly fuzz run fuzz_target_1 fuzz-timed: - cd src/hyperlight_host && cargo +nightly fuzz run fuzz_target_1 -- -max_total_time=300 \ No newline at end of file + cd src/hyperlight_host && cargo +nightly fuzz run fuzz_target_1 -- -max_total_time=300 From 10f62240ba520f81a7c9daa2ee3fae1f14a92285 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Tue, 12 Nov 2024 19:43:14 +0000 Subject: [PATCH 04/25] use feature mshv2 not mshv Signed-off-by: Simon Davies --- .github/workflows/dep_rust.yml | 2 +- Cargo.lock | 58 ++++++++++++++++++++++++++++++---- 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index f3180790..056e4a50 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -72,7 +72,7 @@ jobs: just test-rust ${{ matrix.config }} # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess - just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv' || 'kvm' }} + just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv2' || 'kvm' }} # make sure certain cargo features compile cargo check -p hyperlight-host --features crashdump diff --git a/Cargo.lock b/Cargo.lock index ab14586d..7d5cf7cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1102,8 +1102,10 @@ dependencies = [ "lazy_static", "libc", "log", - "mshv-bindings", - "mshv-ioctls", + "mshv-bindings 0.2.1", + "mshv-bindings 0.3.2", + "mshv-ioctls 0.2.1", + "mshv-ioctls 0.3.2", "once_cell", "opentelemetry", "opentelemetry-otlp", @@ -1623,7 +1625,19 @@ dependencies = [ "libc", "num_enum", "vmm-sys-util", - "zerocopy", + "zerocopy 0.7.35", +] + +[[package]] +name = "mshv-bindings" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0cb5031f3243a7459b7c13d960d25420980874eebda816db24ce6077e21d43" +dependencies = [ + "libc", + "num_enum", + "vmm-sys-util", + "zerocopy 0.8.14", ] [[package]] @@ -1633,11 +1647,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d57586da719aacc905042eea71ff2efb52d16c7228a94af155c9ea45fe09c1c7" dependencies = [ "libc", - "mshv-bindings", + "mshv-bindings 0.2.1", "thiserror 1.0.69", "vmm-sys-util", ] +[[package]] +name = "mshv-ioctls" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89abe853221fa6f14ad4066affb9abda241a03d65622887d5794e1422d0bd75a" +dependencies = [ + "libc", + "mshv-bindings 0.3.2", + "thiserror 2.0.7", + "vmm-sys-util", +] + [[package]] name = "nom" version = "7.1.3" @@ -1917,7 +1943,7 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -3442,7 +3468,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a367f292d93d4eab890745e75a778da40909cab4d6ff8173693812f79c4a2468" +dependencies = [ + "zerocopy-derive 0.8.14", ] [[package]] @@ -3456,6 +3491,17 @@ dependencies = [ "syn", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3931cb58c62c13adec22e38686b559c86a30565e16ad6e8510a337cedc611e1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.4" From b3c9fb7eba1d47180a4c431ffd772f475ade0356 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Wed, 13 Nov 2024 10:57:53 +0000 Subject: [PATCH 05/25] Added comment Signed-off-by: Simon Davies --- src/hyperlight_host/src/hypervisor/hyperv_linux.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index d271c025..62f247d2 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -106,6 +106,10 @@ impl HypervLinuxDriver { let vm_fd = mshv.create_vm_with_config(&pr)?; #[cfg(mshv3)] let vm_fd = { + // It's important to avoid create_vm() and explicitly use + // create_vm_with_args() with an empty arguments structure + // here, because otherwise the partition is set up with a SynIC. + let vm_fd = mshv.create_vm_with_args(&pr)?; let features: hv_partition_synthetic_processor_features = Default::default(); vm_fd.hvcall_set_partition_property( From a44ed27b719121545cf6f4021f63dee340294f2b Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Wed, 13 Nov 2024 19:48:19 +0000 Subject: [PATCH 06/25] allow mshv3 to override mshv2 feature Signed-off-by: Simon Davies --- src/hyperlight_host/build.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/hyperlight_host/build.rs b/src/hyperlight_host/build.rs index 4c5766f7..7600f647 100644 --- a/src/hyperlight_host/build.rs +++ b/src/hyperlight_host/build.rs @@ -98,18 +98,14 @@ fn main() -> Result<()> { crashdump: { all(feature = "crashdump", debug_assertions) }, // print_debug feature is aliased with debug_assertions to make it only available in debug-builds. print_debug: { all(feature = "print_debug", debug_assertions) }, - // the following is a bit of a hack to stop clippy failing when run with -all features as it enables both mshv2 and mshv3 at the same time causing errors - mshv2: { all(feature = "mshv2", target_os = "linux", not(clippy)) }, + // the following features are mutually exclusive but rather than enforcing that here we are enabling mshv3 to override mshv2 when both are enabled + // because mshv2 is in the default feature set we want to allow users to enable mshv3 without having to set --no-default-features and the re-enable + // the other features they want. + mshv2: { all(feature = "mshv2", not(feature="mshv3"), target_os = "linux") }, mshv3: { all(feature = "mshv3", target_os = "linux") }, } write_built_file()?; - // mshv2 and mshv3 features are mutually exclusive - #[cfg(all(feature = "mshv2", feature = "mshv3", not(clippy)))] - Err(anyhow::anyhow!( - "mshv2 and mshv3 features are mutually exclusive" - ))?; - Ok(()) } From 9016c6047bcca8e4fa0aa90785e6a3ec93facfa5 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 13 Jan 2025 15:31:37 +0000 Subject: [PATCH 07/25] fix dependencies Signed-off-by: Simon Davies --- src/hyperlight_host/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyperlight_host/Cargo.toml b/src/hyperlight_host/Cargo.toml index 69568999..4a68df86 100644 --- a/src/hyperlight_host/Cargo.toml +++ b/src/hyperlight_host/Cargo.toml @@ -124,8 +124,8 @@ executable_heap = [] print_debug = [] crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds. kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"] -mshv2 = ["mshv-bindings2", "mshv-ioctls2"] -mshv3 = ["mshv-bindings3", "mshv-ioctls3"] +mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"] +mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"] inprocess = [] [[bench]] From 42a65a74942b9ffd8c7e1c9a2f2ed89b69e63b19 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 13 Jan 2025 15:32:05 +0000 Subject: [PATCH 08/25] make sure to call fd.initialize on mshv3 Signed-off-by: Simon Davies --- src/hyperlight_host/src/hypervisor/hyperv_linux.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 62f247d2..dd849d3e 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -111,6 +111,7 @@ impl HypervLinuxDriver { // here, because otherwise the partition is set up with a SynIC. let vm_fd = mshv.create_vm_with_args(&pr)?; + vm_fd.initialise()?; let features: hv_partition_synthetic_processor_features = Default::default(); vm_fd.hvcall_set_partition_property( hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES, From 1391fa8c67a84595c8b2aaaa39bc4ca4677ecd12 Mon Sep 17 00:00:00 2001 From: Simon Davies Date: Mon, 13 Jan 2025 16:24:35 +0000 Subject: [PATCH 09/25] fix typo Signed-off-by: Simon Davies --- src/hyperlight_host/src/hypervisor/hyperv_linux.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index dd849d3e..8e7e443f 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -111,7 +111,7 @@ impl HypervLinuxDriver { // here, because otherwise the partition is set up with a SynIC. let vm_fd = mshv.create_vm_with_args(&pr)?; - vm_fd.initialise()?; + vm_fd.initialize()?; let features: hv_partition_synthetic_processor_features = Default::default(); vm_fd.hvcall_set_partition_property( hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES, From b0e046fcc5db6eac2e3d40eb74ce720ae806c127 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Mon, 13 Jan 2025 09:18:03 -0800 Subject: [PATCH 10/25] Adding azlinux3 testing to dep_rust.yml workflows --- .github/workflows/dep_rust.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 056e4a50..543b9d23 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -26,11 +26,15 @@ jobs: strategy: fail-fast: true matrix: - hypervisor: [hyperv, mshv, kvm] # hyperv is windows, mshv and kvm are linux + hypervisor: [hyperv, mshv, mshv3, kvm] # hyperv is windows, mshv and kvm are linux cpu: [amd, intel] config: [debug, release] - runs-on: ${{ fromJson(format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor, matrix.cpu)) }} + runs-on: ${{ fromJson( + format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', + matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', + matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor == 'mhsv3' && 'azlinux3-mshv' || matrix.hypervisor, + matrix.cpu)) }} steps: - uses: actions/checkout@v4 @@ -72,7 +76,7 @@ jobs: just test-rust ${{ matrix.config }} # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess - just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv2' || 'kvm' }} + just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv2' || matrix.hypervisor== 'mhsv3' && 'mshv3' || 'kvm' }} # make sure certain cargo features compile cargo check -p hyperlight-host --features crashdump @@ -98,7 +102,7 @@ jobs: ### Benchmarks ### - name: Install github-cli (Linux mariner) - if: runner.os == 'Linux' && matrix.hypervisor == 'mshv' + if: runner.os == 'Linux' && (matrix.hypervisor == 'mshv' || matrix.hypervisor == 'mshv3') run: sudo dnf install gh -y - name: Install github-cli (Linux ubuntu) From 696a75135a7c9ff449bb5373c1c7e7e35732f5e8 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Mon, 13 Jan 2025 11:38:07 -0800 Subject: [PATCH 11/25] fixup! Adding azlinux3 testing to dep_rust.yml workflows --- .github/workflows/dep_rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 543b9d23..d9d5152e 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -33,7 +33,7 @@ jobs: runs-on: ${{ fromJson( format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', - matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor == 'mhsv3' && 'azlinux3-mshv' || matrix.hypervisor, + matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor, matrix.cpu)) }} steps: - uses: actions/checkout@v4 From 650c7526bc60083e30c03374d5bec1906ce1ebbf Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 05:15:26 -0800 Subject: [PATCH 12/25] vm_initialize() after setting properties --- src/hyperlight_host/src/hypervisor/hyperv_linux.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs index 8e7e443f..3dcdc2e4 100644 --- a/src/hyperlight_host/src/hypervisor/hyperv_linux.rs +++ b/src/hyperlight_host/src/hypervisor/hyperv_linux.rs @@ -111,12 +111,12 @@ impl HypervLinuxDriver { // here, because otherwise the partition is set up with a SynIC. let vm_fd = mshv.create_vm_with_args(&pr)?; - vm_fd.initialize()?; let features: hv_partition_synthetic_processor_features = Default::default(); vm_fd.hvcall_set_partition_property( hv_partition_property_code_HV_PARTITION_PROPERTY_SYNTHETIC_PROC_FEATURES, unsafe { features.as_uint64[0] }, )?; + vm_fd.initialize()?; vm_fd }; From 6583adab3d6073ca35d6fefbc7eafc90fd8f03c5 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 07:27:28 -0800 Subject: [PATCH 13/25] Updating test targets in Justfile to for mshv3 --- .github/workflows/dep_rust.yml | 2 +- Justfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index d9d5152e..d7c144e8 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -73,7 +73,7 @@ jobs: CARGO_TERM_COLOR: always run: | # with default features - just test-rust ${{ matrix.config }} + just test-rust ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv2' || matrix.hypervisor== 'mhsv3' && 'mshv3' || 'kvm' }} diff --git a/Justfile b/Justfile index cb289892..1a1249bd 100644 --- a/Justfile +++ b/Justfile @@ -101,8 +101,8 @@ test-rust-int guest target=default-target features="": # integration tests # run execute_on_heap test with feature "executable_heap" on and off - {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap --features executable_heap -- --ignored - {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap -- --ignored + {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap {{ if features =="" {'executable_heap'} else {"--features executable_heap," + features} }} -- --ignored + {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap {{ if features =="" {""} else {"--features " + features} }} -- --ignored # run the rest of the integration tests {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test -p hyperlight-host {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --test '*' From 965d09ec62019227cc37bcb718c8e6fe5b36fb27 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 07:42:19 -0800 Subject: [PATCH 14/25] fixup! Updating test targets in Justfile to for mshv3 --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 1a1249bd..8cfb3362 100644 --- a/Justfile +++ b/Justfile @@ -101,7 +101,7 @@ test-rust-int guest target=default-target features="": # integration tests # run execute_on_heap test with feature "executable_heap" on and off - {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap {{ if features =="" {'executable_heap'} else {"--features executable_heap," + features} }} -- --ignored + {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap {{ if features =="" {" --features executable_heap"} else {"--features executable_heap," + features} }} -- --ignored {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap {{ if features =="" {""} else {"--features " + features} }} -- --ignored # run the rest of the integration tests {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test -p hyperlight-host {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --test '*' From bc180f7cf68bfbc36abcadc079cb452c46be8342 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 08:01:47 -0800 Subject: [PATCH 15/25] fixup! Updating test targets in Justfile to for mshv3 --- Justfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Justfile b/Justfile index 8cfb3362..3a3e6953 100644 --- a/Justfile +++ b/Justfile @@ -76,7 +76,7 @@ clean-rust: # Some tests cannot run with other tests, they are marked as ignored so that cargo test works # there may be tests that we really want to ignore so we cant just use --ignored and we have to # Specify the test name of the ignored tests that we want to run -test-rust target=default-target features="": (test-rust-int "rust" target features) (test-rust-int "c" target features) (test-seccomp target) +test-rust target=default-target features="": (test-rust-int "rust" target features) (test-rust-int "c" target features) (test-seccomp target features) # unit tests cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --lib @@ -91,10 +91,10 @@ test-rust target=default-target features="": (test-rust-int "rust" target featur cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} hypervisor::hypervisor_handler::tests::create_1000_sandboxes -p hyperlight-host --lib -- --ignored {{ set-trace-env-vars }} cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --lib sandbox::outb::tests::test_log_outb_log -- --ignored -test-seccomp target=default-target: +test-seccomp target=default-target features="": # run seccomp test with feature "seccomp" on and off - cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib -- --ignored - cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features --features mshv2,kvm --lib -- --ignored + cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib {{ if features =="" {''} else { "--features" + features } }}-- --ignored + cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features {{ if features =~"*mshv3*" {"--features mshv3"} else {"--features mshv2,kvm" } }} --lib -- --ignored # rust integration tests. guest can either be "rust" or "c" test-rust-int guest target=default-target features="": From a2341fe494eae0e48644d6853f950c7e2d2b09de Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 08:16:02 -0800 Subject: [PATCH 16/25] Justfile fixup --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 3a3e6953..81a5e81d 100644 --- a/Justfile +++ b/Justfile @@ -93,7 +93,7 @@ test-rust target=default-target features="": (test-rust-int "rust" target featur test-seccomp target=default-target features="": # run seccomp test with feature "seccomp" on and off - cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib {{ if features =="" {''} else { "--features" + features } }}-- --ignored + cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib {{ if features =="" {''} else { "--features " + features } }}-- --ignored cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features {{ if features =~"*mshv3*" {"--features mshv3"} else {"--features mshv2,kvm" } }} --lib -- --ignored # rust integration tests. guest can either be "rust" or "c" From e9bfca7a96231c49907914278a0672281aa32917 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 08:29:58 -0800 Subject: [PATCH 17/25] Justfile fixup --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 81a5e81d..96cd920b 100644 --- a/Justfile +++ b/Justfile @@ -93,7 +93,7 @@ test-rust target=default-target features="": (test-rust-int "rust" target featur test-seccomp target=default-target features="": # run seccomp test with feature "seccomp" on and off - cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib {{ if features =="" {''} else { "--features " + features } }}-- --ignored + cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib {{ if features =="" {''} else { "--features " + features } }} -- --ignored cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features {{ if features =~"*mshv3*" {"--features mshv3"} else {"--features mshv2,kvm" } }} --lib -- --ignored # rust integration tests. guest can either be "rust" or "c" From 47a943dd7ffe124d08f4f753460336439a61da98 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 08:48:09 -0800 Subject: [PATCH 18/25] Justfile fixup --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 96cd920b..50783af3 100644 --- a/Justfile +++ b/Justfile @@ -94,7 +94,7 @@ test-rust target=default-target features="": (test-rust-int "rust" target featur test-seccomp target=default-target features="": # run seccomp test with feature "seccomp" on and off cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib {{ if features =="" {''} else { "--features " + features } }} -- --ignored - cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features {{ if features =~"*mshv3*" {"--features mshv3"} else {"--features mshv2,kvm" } }} --lib -- --ignored + cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features {{ if features =~"mshv3" {"--features mshv3"} else {"--features mshv2,kvm" } }} --lib -- --ignored # rust integration tests. guest can either be "rust" or "c" test-rust-int guest target=default-target features="": From 632080866821692541dd86a7a397ae3ea3139b91 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 09:10:17 -0800 Subject: [PATCH 19/25] dep_rust.yml fixup --- .github/workflows/dep_rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index d7c144e8..4008c352 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -76,7 +76,7 @@ jobs: just test-rust ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess - just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv2' || matrix.hypervisor== 'mhsv3' && 'mshv3' || 'kvm' }} + just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv2' || matrix.hypervisor== 'msvh3' && 'mshv3' || 'kvm' }} # make sure certain cargo features compile cargo check -p hyperlight-host --features crashdump From e981c34d52b8457bdc21821210add90502ac38f4 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 09:24:06 -0800 Subject: [PATCH 20/25] dep_rust.yml fixup --- .github/workflows/dep_rust.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 4008c352..55fbc2ac 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -76,7 +76,7 @@ jobs: just test-rust ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess - just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv2' || matrix.hypervisor== 'msvh3' && 'mshv3' || 'kvm' }} + just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv2' || matrix.hypervisor == 'mshv3' && 'mshv3' || 'kvm' }} # make sure certain cargo features compile cargo check -p hyperlight-host --features crashdump From f450039a4696bc744bb4bc788d698b92625b76d8 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 09:52:26 -0800 Subject: [PATCH 21/25] dep_rust.yml adding feature passthrough for run-rust-examples to support mshv3 --- Justfile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Justfile b/Justfile index 50783af3..15c30c6c 100644 --- a/Justfile +++ b/Justfile @@ -107,7 +107,7 @@ test-rust-int guest target=default-target features="": {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test -p hyperlight-host {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --test '*' test-rust-feature-compilation-fail target=default-target: - @# the following should fail on linux because either kvm or msh feature must be specified, which is why the exit code is inverted with an !. + @# the following should fail on linux because one of kvm, mshv, or mshv3 feature must be specified, which is why the exit code is inverted with an !. {{ if os() == "linux" { "! cargo check -p hyperlight-host --no-default-features 2> /dev/null"} else { "" } }} test target=default-target: (test-rust target) @@ -149,15 +149,15 @@ gen-all-fbs-rust-code: just fmt-apply # RUST EXAMPLES -run-rust-examples target=default-target: (build-rust target) - cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example metrics - cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example metrics --features "function_call_metrics" +run-rust-examples target=default-target features="": (build-rust target) + cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example metrics {{ if features =="" {''} else { "--features " + features } }} + cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example metrics {{ if features =="" {"--features function_call_metrics"} else {"--features function_call_metrics," + features} }} {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example logging # The two tracing examples are flaky on windows so we run them on linux only for now, need to figure out why as they run fine locally on windows -run-rust-examples-linux target=default-target: (build-rust target) (run-rust-examples target) - {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example tracing - {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example tracing --features "function_call_metrics" +run-rust-examples-linux target=default-target: (build-rust target) (run-rust-examples target features) + {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example tracing {{ if features =="" {''} else { "--features " + features } }} + {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example tracing {{ if features =="" {"--features function_call_metrics" } else {"--features function_call_metrics," + features} }} # BENCHMARKING From b0c3f088df807e29f041b875487c87c1238f7236 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Tue, 14 Jan 2025 09:59:55 -0800 Subject: [PATCH 22/25] dep_rust.yml fixup --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 15c30c6c..79a86f5f 100644 --- a/Justfile +++ b/Justfile @@ -155,7 +155,7 @@ run-rust-examples target=default-target features="": (build-rust target) {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example logging # The two tracing examples are flaky on windows so we run them on linux only for now, need to figure out why as they run fine locally on windows -run-rust-examples-linux target=default-target: (build-rust target) (run-rust-examples target features) +run-rust-examples-linux target=default-target features="": (build-rust target) (run-rust-examples target features) {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example tracing {{ if features =="" {''} else { "--features " + features } }} {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example tracing {{ if features =="" {"--features function_call_metrics" } else {"--features function_call_metrics," + features} }} From c3bb8240e92db9982b33e6227c76ca8065b19177 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Wed, 15 Jan 2025 02:16:24 -0800 Subject: [PATCH 23/25] dep_rust.yml fixup --- .github/workflows/dep_rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 55fbc2ac..70ecc252 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -98,11 +98,11 @@ jobs: env: CARGO_TERM_COLOR: always RUST_LOG: debug - run: just run-rust-examples-linux ${{ matrix.config }} + run: just run-rust-examples-linux ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} ### Benchmarks ### - name: Install github-cli (Linux mariner) - if: runner.os == 'Linux' && (matrix.hypervisor == 'mshv' || matrix.hypervisor == 'mshv3') + if: runner.os == 'Linux' && matrix.hypervisor == 'mshv' run: sudo dnf install gh -y - name: Install github-cli (Linux ubuntu) From cfc7860d7804ec0805a04d1062a5c8f72402f306 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Wed, 15 Jan 2025 02:31:57 -0800 Subject: [PATCH 24/25] Justfile - add features to logging example --- Justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Justfile b/Justfile index 79a86f5f..40c5f721 100644 --- a/Justfile +++ b/Justfile @@ -152,7 +152,7 @@ gen-all-fbs-rust-code: run-rust-examples target=default-target features="": (build-rust target) cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example metrics {{ if features =="" {''} else { "--features " + features } }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example metrics {{ if features =="" {"--features function_call_metrics"} else {"--features function_call_metrics," + features} }} - {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example logging + {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example logging {{ if features =="" {''} else { "--features " + features } }} # The two tracing examples are flaky on windows so we run them on linux only for now, need to figure out why as they run fine locally on windows run-rust-examples-linux target=default-target features="": (build-rust target) (run-rust-examples target features) From b9d4b79822ddb4ffe4df8609f560c597a9d0f4e3 Mon Sep 17 00:00:00 2001 From: Mark Rossetti Date: Wed, 15 Jan 2025 02:59:52 -0800 Subject: [PATCH 25/25] Updating benchmark testing to support mshv3 features gates and driver Signed-off-by: Mark Rossetti --- .github/workflows/Benchmarks.yml | 6 +++--- .github/workflows/dep_rust.yml | 2 +- Justfile | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/Benchmarks.yml b/.github/workflows/Benchmarks.yml index c2665370..302a06d4 100644 --- a/.github/workflows/Benchmarks.yml +++ b/.github/workflows/Benchmarks.yml @@ -17,11 +17,11 @@ jobs: strategy: fail-fast: true matrix: - hypervisor: [hyperv, mshv, kvm] # hyperv is windows, mshv and kvm are linux + hypervisor: [hyperv, mshv, mshv3, kvm] # hyperv is windows, mshv and kvm are linux cpu: [amd, intel] config: [release] # don't want to benchmark debug-builds - runs-on: ${{ fromJson(format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor, matrix.cpu)) }} + runs-on: ${{ fromJson(format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor, matrix.cpu)) }} steps: ### Setup ### @@ -67,7 +67,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run Benchmarks - run: just bench-ci main release + run: just bench-ci main release ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 70ecc252..4930aa90 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -118,5 +118,5 @@ jobs: - name: Run benchmarks run: | - just bench-ci main ${{ matrix.config }} + just bench-ci main ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}} if: ${{ matrix.config == 'release' }} diff --git a/Justfile b/Justfile index 40c5f721..5e896621 100644 --- a/Justfile +++ b/Justfile @@ -174,11 +174,11 @@ bench-download os hypervisor cpu tag="": tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1 # Warning: compares to and then OVERWRITES the given baseline -bench-ci baseline target=default-target: - cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} -- --verbose --save-baseline {{ baseline }} +bench-ci baseline target=default-target features="": + cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }} -bench target=default-target: - cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} -- --verbose +bench target=default-target features="": + cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose # FUZZING fuzz: