raise the soft limit for file descriptors

If we don't do this, we quickly run into the limit when trying to
create memfds.

host/mushroom/Cargo.toml

@@ -24,7 +24,7 @@ constants = { workspace = true }
 loader = { workspace = true }
 log-types = { workspace = true, features = ["std"] }
 mushroom-verify = { workspace = true, optional = true }
-nix = { version = "0.29.0", features = ["fs", "ioctl", "mman", "pthread", "signal"] }
+nix = { version = "0.29.0", features = ["fs", "ioctl", "mman", "pthread", "resource", "signal"] }
 profiler-types = { workspace = true }
 qgs-client = { workspace = true, optional = true }
 rand = "0.8.5"

host/mushroom/src/lib.rs

@@ -3,7 +3,10 @@ use std::{collections::HashMap, num::NonZeroU32, sync::Once};
 use anyhow::{Context, Result};
 use bit_field::BitField;
 use kvm::KvmCap;
-use nix::sys::signal::{sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal};
+use nix::sys::{
+    resource::{getrlimit, setrlimit, Resource},
+    signal::{sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal},
+};
 use slot::Slot;
 use x86_64::structures::paging::PhysFrame;
 
@@ -94,3 +97,13 @@ fn install_signal_handler() {
         };
     });
 }
+
+fn raise_file_no_limit() {
+    static RAISE_NO_LIMIT: Once = Once::new();
+    RAISE_NO_LIMIT.call_once(|| {
+        // Set the soft limit to the hard limit. We need this because we
+        // allocate a lot of memfds.
+        let (_soft, hard) = getrlimit(Resource::RLIMIT_NOFILE).unwrap();
+        setrlimit(Resource::RLIMIT_NOFILE, hard, hard).unwrap();
+    });
+}

host/mushroom/src/snp.rs

@@ -35,6 +35,7 @@ use crate::{
     },
     logging::start_log_collection,
     profiler::{start_profile_collection, ProfileFolder},
+    raise_file_no_limit,
     slot::Slot,
     MushroomResult, SIG_KICK,
 };
@@ -229,6 +230,7 @@ impl VmContext {
         }
 
         install_signal_handler();
+        raise_file_no_limit();
 
         Ok(Self {
             vm,

host/mushroom/src/tdx.rs

@@ -34,6 +34,7 @@ use crate::{
     },
     logging::start_log_collection,
     profiler::{start_profile_collection, ProfileFolder},
+    raise_file_no_limit,
     slot::Slot,
     MushroomResult, SIG_KICK, TSC_MHZ,
 };
@@ -255,6 +256,7 @@ impl VmContext {
         }
 
         install_signal_handler();
+        raise_file_no_limit();
 
         Ok((
             Self {