Configure NVMe keep-alive ability from host (#487)

Inform OpenHCL that Host is compatible with NVMe keep-alive feature by
adding a new device tree entry
`/openhcl/vf-keepalive=nvme`
where vf-keepalive is a list of strings which may be expanded in future
to include other device types such as `vf-keepalive=nvme,mana`

During servicing operation the host must now explicitly indicate that
NVMe data should be saved (if VM supports it), this change flips the
definition of keepalive bit in capabilities_flags bitmask (it used to be
an emergency shut-off, not it is an "allow" indication). This flipping
resolves compatibility issues with older host OSes after Live Migration.

openhcl/host_fdt_parser/src/lib.rs

@@ -234,6 +234,8 @@ pub struct ParsedDeviceTree<
     /// This is used to allocate a persistent VTL2 pool on non-isolated guests,
     /// to allow devices to stay alive during a servicing operation.
     pub device_dma_page_count: Option<u64>,
+    /// Indicates that Host does support NVMe keep-alive.
+    pub nvme_keepalive: bool,
 }
 
 /// The memory allocation mode provided by the host. This determines how OpenHCL
@@ -313,6 +315,7 @@ impl<
             memory_allocation_mode: MemoryAllocationMode::Host,
             entropy: None,
             device_dma_page_count: None,
+            nvme_keepalive: false,
         }
     }
 
@@ -513,9 +516,18 @@ impl<
                                 storage.entropy = Some(entropy);
                             }
                             // These parameters may not be present so it is not an error if they are missing.
-                            "servicing" => {
+                            "keep-alive" => {
+                                storage.nvme_keepalive = openhcl_child
+                                    .find_property("device-types")
+                                    .ok()
+                                    .flatten()
+                                    .and_then(|p| p.read_str().ok())
+                                    == Some("nvme");
+                            }
+                            "device-dma" => {
+                                // DMA reserved page count hint.
                                 storage.device_dma_page_count = openhcl_child
-                                    .find_property("dma-preserve-pages")
+                                    .find_property("total-pages")
                                     .ok()
                                     .flatten()
                                     .and_then(|p| p.read_u64(0).ok());
@@ -527,6 +539,7 @@ impl<
                         }
                     }
                 }
+
                 _ if child.name.starts_with("memory@") => {
                     let igvm_type = if let Some(igvm_type) = child
                         .find_property(igvm_defs::dt::IGVM_DT_IGVM_TYPE_PROPERTY)
@@ -713,6 +726,7 @@ impl<
             memory_allocation_mode: _,
             entropy: _,
             device_dma_page_count: _,
+            nvme_keepalive: _,
         } = storage;
 
         *device_tree_size = parser.total_size;
@@ -1358,7 +1372,7 @@ mod tests {
         let p_memory_allocation_mode = root.add_string("memory-allocation-mode").unwrap();
         let p_memory_allocation_size = root.add_string("memory-size").unwrap();
         let p_mmio_allocation_size = root.add_string("mmio-size").unwrap();
-        let p_device_dma_page_count = root.add_string("dma-preserve-pages").unwrap();
+        let p_device_dma_page_count = root.add_string("total-pages").unwrap();
         let mut openhcl = root.start_node("openhcl").unwrap();
 
         let memory_alloc_str = match context.memory_allocation_mode {
@@ -1387,7 +1401,7 @@ mod tests {
         // add device_dma_page_count
         if let Some(device_dma_page_count) = context.device_dma_page_count {
             openhcl = openhcl
-                .start_node("servicing")
+                .start_node("device-dma")
                 .unwrap()
                 .add_u64(p_device_dma_page_count, device_dma_page_count)
                 .unwrap()

openhcl/openhcl_boot/src/host_params/dt.rs

@@ -520,6 +520,7 @@ impl PartitionInfo {
             memory_allocation_mode: _,
             entropy,
             vtl0_alias_map: _,
+            nvme_keepalive,
         } = storage;
 
         assert!(!vtl2_used_ranges.is_empty());
@@ -541,6 +542,7 @@ impl PartitionInfo {
         *com3_serial = parsed.com3_serial;
         *gic = parsed.gic.clone();
         *entropy = parsed.entropy.clone();
+        *nvme_keepalive = parsed.nvme_keepalive;
 
         Ok(Some(storage))
     }

openhcl/openhcl_boot/src/host_params/mod.rs

@@ -92,6 +92,8 @@ pub struct PartitionInfo {
     pub entropy: Option<ArrayVec<u8, MAX_ENTROPY_SIZE>>,
     /// The VTL0 alias map physical address.
     pub vtl0_alias_map: Option<u64>,
+    /// Host is compatible with DMA preservation / NVMe keep-alive.
+    pub nvme_keepalive: bool,
 }
 
 impl PartitionInfo {
@@ -122,6 +124,7 @@ impl PartitionInfo {
             memory_allocation_mode: MemoryAllocationMode::Host,
             entropy: None,
             vtl0_alias_map: None,
+            nvme_keepalive: false,
         }
     }
 

openhcl/openhcl_boot/src/main.rs

@@ -225,6 +225,12 @@ fn build_kernel_command_line(
         )?;
     }
 
+    // Only when explicitly supported by Host.
+    // TODO: Move from command line to device tree when stabilized.
+    if partition_info.nvme_keepalive && !partition_info.vtl2_pool_memory.is_empty() {
+        write!(cmdline, "OPENHCL_NVME_KEEP_ALIVE=1 ")?;
+    }
+
     if let Some(sidecar) = sidecar {
         write!(cmdline, "{} ", sidecar.kernel_command_line())?;
     }
@@ -910,6 +916,7 @@ mod test {
             memory_allocation_mode: host_fdt_parser::MemoryAllocationMode::Host,
             entropy: None,
             vtl0_alias_map: None,
+            nvme_keepalive: false,
         }
     }
 

openhcl/underhill_core/src/dispatch/mod.rs

@@ -469,15 +469,15 @@ impl LoadedVm {
         &mut self,
         correlation_id: Guid,
         deadline: std::time::Instant,
-        _capabilities_flags: SaveGuestVtl2StateFlags,
+        capabilities_flags: SaveGuestVtl2StateFlags,
     ) -> anyhow::Result<ServicingState> {
         if self.isolation.is_isolated() {
             anyhow::bail!("Servicing is not yet supported for isolated VMs");
         }
 
         // NOTE: This is set via the corresponding env arg, as this feature is
         // experimental.
-        let nvme_keepalive = self.nvme_keep_alive;
+        let nvme_keepalive = self.nvme_keep_alive && capabilities_flags.enable_nvme_keepalive();
 
         // Do everything before the log flush under a span.
         let mut state = async {

openhcl/underhill_core/src/worker.rs

@@ -1870,7 +1870,7 @@ async fn new_underhill_vm(
 
         let private_pool_spanwer = private_pool.as_ref().map(|p| p.allocator_spawner());
 
-        let save_restore_supported = env_cfg.nvme_keep_alive;
+        let save_restore_supported = env_cfg.nvme_keep_alive && private_pool_spanwer.is_some();
         let vfio_dma_buffer_spawner = Box::new(
             move |device_id: String| -> anyhow::Result<Arc<dyn VfioDmaBuffer>> {
                 shared_vis_pool_spawner

vm/devices/get/get_protocol/src/lib.rs

@@ -1179,8 +1179,11 @@ impl UpdateGenerationId {
 #[bitfield(u64)]
 #[derive(AsBytes, FromBytes, FromZeroes)]
 pub struct SaveGuestVtl2StateFlags {
+    /// Explicitly allow nvme_keepalive feature when servicing.
+    #[bits(1)]
+    pub enable_nvme_keepalive: bool,
     /// Reserved, must be zero.
-    #[bits(64)]
+    #[bits(63)]
     _rsvd1: u64,
 }