Adds support for Azure Linux 3

.github/workflows/Benchmarks.yml

@@ -17,11 +17,11 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        hypervisor: [hyperv, mshv, kvm] # hyperv is windows, mshv and kvm are linux
+        hypervisor: [hyperv, mshv, mshv3, kvm] # hyperv is windows, mshv and kvm are linux
         cpu: [amd, intel]
         config: [release] # don't want to benchmark debug-builds
 
-    runs-on: ${{ fromJson(format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor, matrix.cpu)) }} 
+    runs-on: ${{ fromJson(format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor, matrix.cpu)) }} 
 
     steps:
       ### Setup ###
@@ -67,7 +67,7 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run Benchmarks
-        run: just bench-ci main release
+        run: just bench-ci main release ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
 
       - uses: actions/upload-artifact@v4
         with:

.github/workflows/dep_rust.yml

@@ -26,11 +26,15 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        hypervisor: [hyperv, mshv, kvm] # hyperv is windows, mshv and kvm are linux
+        hypervisor: [hyperv, mshv, mshv3, kvm] # hyperv is windows, mshv and kvm are linux
         cpu: [amd, intel]
         config: [debug, release]
 
-    runs-on: ${{ fromJson(format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor, matrix.cpu)) }} 
+    runs-on: ${{ fromJson(
+        format('["self-hosted", "{0}", "X64", "1ES.Pool=hld-{1}-{2}"]', 
+          matrix.hypervisor == 'hyperv' && 'Windows' || 'Linux', 
+          matrix.hypervisor == 'hyperv' && 'win2022' || matrix.hypervisor == 'mshv3' && 'azlinux3-mshv' || matrix.hypervisor, 
+          matrix.cpu)) }} 
     steps:
       - uses: actions/checkout@v4
 
@@ -75,10 +79,10 @@ jobs:
           CARGO_TERM_COLOR: always
         run: |
           # with default features
-          just test-rust ${{ matrix.config }}
+          just test-rust ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
 
           # with only one driver enabled (driver mshv/kvm feature is ignored on windows) + seccomp + inprocess
-          just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv' || 'kvm' }} 
+          just test-rust ${{ matrix.config }} inprocess,seccomp,${{ matrix.hypervisor == 'mshv' && 'mshv2' || matrix.hypervisor == 'mshv3' && 'mshv3' || 'kvm' }} 
 
           # make sure certain cargo features compile
           cargo check -p hyperlight-host --features crashdump
@@ -100,7 +104,7 @@ jobs:
         env:
           CARGO_TERM_COLOR: always
           RUST_LOG: debug
-        run: just run-rust-examples-linux ${{ matrix.config }}
+        run: just run-rust-examples-linux ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
 
       ### Benchmarks ###
       - name: Install github-cli (Linux mariner)
@@ -120,5 +124,5 @@ jobs:
 
       - name: Run benchmarks
         run: |
-          just bench-ci main ${{ matrix.config }}
+          just bench-ci main ${{ matrix.config }} ${{ matrix.hypervisor == 'mshv3' && 'mshv3' || ''}}
         if: ${{ matrix.config == 'release' }}

Cargo.toml

@@ -33,8 +33,6 @@ repository = "https://github.com/hyperlight-dev/hyperlight"
 readme = "README.md"
 
 [workspace.dependencies]
-mshv-bindings = { version = "=0.2.1" }
-mshv-ioctls = { version = "=0.2.1" }
 
 hyperlight-common = { path = "src/hyperlight_common", version = "0.1.0", default-features = false }
 hyperlight-host = { path = "src/hyperlight_host", version = "0.1.0", default-features = false }

Justfile

@@ -76,7 +76,7 @@ clean-rust:
 # Some tests cannot run with other tests, they are marked as ignored so that cargo test works
 # there may be tests that we really want to ignore so we cant just use --ignored and we have to
 # Specify the test name of the ignored tests that we want to run
-test-rust target=default-target features="": (test-rust-int "rust" target features) (test-rust-int "c" target features) (test-seccomp target)
+test-rust target=default-target features="": (test-rust-int "rust" target features) (test-rust-int "c" target features) (test-seccomp target features)
     # unit tests
     cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }}  --lib
 
@@ -91,23 +91,23 @@ test-rust target=default-target features="": (test-rust-int "rust" target featur
     cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} hypervisor::hypervisor_handler::tests::create_1000_sandboxes -p hyperlight-host --lib -- --ignored
     {{ set-trace-env-vars }} cargo test {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --lib sandbox::outb::tests::test_log_outb_log -- --ignored
 
-test-seccomp target=default-target:
+test-seccomp target=default-target features="":
     # run seccomp test with feature "seccomp" on and off
-    cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib -- --ignored
-    cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features --features mshv,kvm --lib -- --ignored
+    cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --lib {{ if features =="" {''} else { "--features " + features } }} -- --ignored
+    cargo test --profile={{ if target == "debug" { "dev" } else { target } }} -p hyperlight-host test_violate_seccomp_filters --no-default-features {{ if features =~"mshv3" {"--features mshv3"} else {"--features mshv2,kvm" } }} --lib -- --ignored
 
 # rust integration tests. guest can either be "rust" or "c"
 test-rust-int guest target=default-target features="":
     # integration tests
 
     # run execute_on_heap test with feature "executable_heap" on and off
-    {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap --features executable_heap -- --ignored
-    {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap -- --ignored
+    {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap {{ if features =="" {" --features executable_heap"} else {"--features executable_heap," + features} }} -- --ignored
+    {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test --profile={{ if target == "debug" { "dev" } else { target } }} --test integration_test execute_on_heap {{ if features =="" {""} else {"--features " + features} }} -- --ignored
     # run the rest of the integration tests
     {{if os() == "windows" { "$env:" } else { "" } }}GUEST="{{guest}}"{{if os() == "windows" { ";" } else { "" } }} cargo test -p hyperlight-host {{ if features =="" {''} else if features=="no-default-features" {"--no-default-features" } else {"--no-default-features -F " + features } }} --profile={{ if target == "debug" { "dev" } else { target } }} --test '*'
 
 test-rust-feature-compilation-fail target=default-target:
-    @# the following should fail on linux because either kvm or msh feature must be specified, which is why the exit code is inverted with an !.
+    @# the following should fail on linux because one of kvm, mshv, or mshv3 feature must be specified, which is why the exit code is inverted with an !.
     {{ if os() == "linux" { "! cargo check -p hyperlight-host --no-default-features 2> /dev/null"} else { "" } }}
 
 test target=default-target: (test-rust target)
@@ -149,15 +149,15 @@ gen-all-fbs-rust-code:
     just fmt-apply
 
 # RUST EXAMPLES
-run-rust-examples target=default-target: (build-rust target)
-    cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example metrics
-    cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example metrics --features "function_call_metrics"
-    {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example logging
+run-rust-examples target=default-target features="": (build-rust target)
+    cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example metrics {{ if features =="" {''} else { "--features " + features } }}
+    cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example metrics {{ if features =="" {"--features function_call_metrics"} else {"--features function_call_metrics," + features} }}
+    {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example logging {{ if features =="" {''} else { "--features " + features } }}
 
 # The two tracing examples are flaky on windows so we run them on linux only for now, need to figure out why as they run fine locally on windows
-run-rust-examples-linux target=default-target: (build-rust target) (run-rust-examples target)
-    {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example tracing
-    {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example tracing --features "function_call_metrics"
+run-rust-examples-linux target=default-target features="": (build-rust target) (run-rust-examples target features)
+    {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example tracing {{ if features =="" {''} else { "--features " + features } }}
+    {{ set-trace-env-vars }} cargo run --profile={{ if target == "debug" { "dev" } else { target } }} --example tracing {{ if features =="" {"--features function_call_metrics" } else {"--features function_call_metrics," + features} }}
 
 # BENCHMARKING
 
@@ -174,15 +174,15 @@ bench-download os hypervisor cpu tag="":
     tar -zxvf target/benchmarks_{{ os }}_{{ hypervisor }}_{{ cpu }}.tar.gz -C target/criterion/ --strip-components=1
 
 # Warning: compares to and then OVERWRITES the given baseline
-bench-ci baseline target=default-target:
-    cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} -- --verbose --save-baseline {{ baseline }}
+bench-ci baseline target=default-target features="":
+    cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose --save-baseline {{ baseline }}
 
-bench target=default-target:
-    cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} -- --verbose
+bench target=default-target features="":
+    cargo bench --profile={{ if target == "debug" { "dev" } else { target } }} {{ if features =="" {''} else { "--features " + features } }} -- --verbose
 
 # FUZZING
 fuzz:
     cd src/hyperlight_host && cargo +nightly fuzz run fuzz_target_1
 
 fuzz-timed:
-    cd src/hyperlight_host && cargo +nightly fuzz run fuzz_target_1 -- -max_total_time=300
+    cd src/hyperlight_host && cargo +nightly fuzz run fuzz_target_1 -- -max_total_time=300

src/hyperlight_host/Cargo.toml

@@ -71,10 +71,12 @@ windows-version = "0.1"
 
 [target.'cfg(unix)'.dependencies]
 seccompiler = { version = "0.4.0", optional = true }
-mshv-bindings = { workspace = true, optional = true }
-mshv-ioctls = { workspace = true, optional = true }
 kvm-bindings = { version = "0.10.0", features = ["fam-wrappers"], optional = true }
 kvm-ioctls = { version = "0.19.1", optional = true }
+mshv-bindings2 = { package="mshv-bindings", version = "=0.2.1", optional = true }
+mshv-ioctls2 = { package="mshv-ioctls",  version = "=0.2.1", optional = true}
+mshv-bindings3 = { package="mshv-bindings", version = "0.3.2", optional = true }
+mshv-ioctls3 = { package="mshv-ioctls",  version = "0.3.2", optional = true}
 
 [dev-dependencies]
 uuid = { version = "1.4.1", features = ["v4"] }
@@ -114,15 +116,16 @@ cfg_aliases = "0.2.1"
 built = { version = "0.7.0", features = ["chrono", "git2"] }
 
 [features]
-default = ["kvm", "mshv", "seccomp"]
+default = ["kvm", "mshv2", "seccomp"]
 seccomp = ["dep:seccompiler"]
 function_call_metrics = []
 executable_heap = []
 # This feature enables printing of debug information to stdout in debug builds
 print_debug = []
 crashdump = ["dep:tempfile"] # Dumps the VM state to a file on unexpected errors or crashes. The path of the file will be printed on stdout and logged. This feature can only be used in debug builds.
 kvm = ["dep:kvm-bindings", "dep:kvm-ioctls"]
-mshv = ["dep:mshv-bindings", "dep:mshv-ioctls"]
+mshv2 = ["dep:mshv-bindings2", "dep:mshv-ioctls2"]
+mshv3 = ["dep:mshv-bindings3", "dep:mshv-ioctls3"]
 inprocess = []
 
 [[bench]]

src/hyperlight_host/build.rs

@@ -85,19 +85,24 @@ fn main() -> Result<()> {
     }
 
     // Makes #[cfg(kvm)] == #[cfg(all(feature = "kvm", target_os = "linux"))]
-    // and #[cfg(mshv)] == #[cfg(all(feature = "mshv", target_os = "linux"))].
+    // and #[cfg(mshv)] == #[cfg(all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux"))].
     // Essentially the kvm and mshv features are ignored on windows as long as you use #[cfg(kvm)] and not #[cfg(feature = "kvm")].
     // You should never use #[cfg(feature = "kvm")] or #[cfg(feature = "mshv")] in the codebase.
     cfg_aliases::cfg_aliases! {
         kvm: { all(feature = "kvm", target_os = "linux") },
-        mshv: { all(feature = "mshv", target_os = "linux") },
+        mshv: { all(any(feature = "mshv2", feature = "mshv3"), target_os = "linux") },
         // inprocess feature is aliased with debug_assertions to make it only available in debug-builds.
         // You should never use #[cfg(feature = "inprocess")] in the codebase. Use #[cfg(inprocess)] instead.
         inprocess: { all(feature = "inprocess", debug_assertions) },
         // crashdump feature is aliased with debug_assertions to make it only available in debug-builds.
         crashdump: { all(feature = "crashdump", debug_assertions) },
         // print_debug feature is aliased with debug_assertions to make it only available in debug-builds.
         print_debug: { all(feature = "print_debug", debug_assertions) },
+        // the following features are mutually exclusive but rather than enforcing that here we are enabling mshv3 to override mshv2 when both are enabled
+        // because mshv2 is in the default feature set we want to allow users to enable mshv3 without having to set --no-default-features and the re-enable
+        // the other features they want.
+        mshv2: { all(feature = "mshv2", not(feature="mshv3"), target_os = "linux") },
+        mshv3: { all(feature = "mshv3", target_os = "linux") },
     }
 
     write_built_file()?;

src/hyperlight_host/src/error.rs

@@ -14,6 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+#[cfg(mshv2)]
+extern crate mshv_ioctls2 as mshv_ioctls;
+
+#[cfg(mshv3)]
+extern crate mshv_ioctls3 as mshv_ioctls;
+
 use std::array::TryFromSliceError;
 use std::cell::{BorrowError, BorrowMutError};
 use std::convert::Infallible;