vmgenid: add support for ARM systems

Expose the VMGenID device via Device Tree bindings. We add a node in the
Device Tree that passes to the guest the address of the generation ID
and the interrupt we are using to notify it about generation changes,
aka snapshot events.

Signed-off-by: Sudan Landge <[email protected]>
Signed-off-by: Babis Chalios <[email protected]>

src/vmm/src/arch/aarch64/fdt.rs

@@ -16,6 +16,7 @@ use super::super::{DeviceType, InitrdConfig};
 use super::cache_info::{read_cache_config, CacheEntry};
 use super::get_fdt_addr;
 use super::gic::GICDevice;
+use crate::devices::acpi::vmgenid::VmGenId;
 use crate::vstate::memory::{Address, Bytes, GuestAddress, GuestMemory, GuestMemoryMmap};
 
 // This is a value for uniquely identifying the FDT node declaring the interrupt controller.
@@ -70,6 +71,7 @@ pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug, S: std::hash::BuildHasher
     cmdline: CString,
     device_info: &HashMap<(DeviceType, String), T, S>,
     gic_device: &GICDevice,
+    vmgenid: &Option<VmGenId>,
     initrd: &Option<InitrdConfig>,
 ) -> Result<Vec<u8>, FdtError> {
     // Allocate stuff necessary for storing the blob.
@@ -97,6 +99,7 @@ pub fn create_fdt<T: DeviceInfoForFDT + Clone + Debug, S: std::hash::BuildHasher
     create_clock_node(&mut fdt_writer)?;
     create_psci_node(&mut fdt_writer)?;
     create_devices_node(&mut fdt_writer, device_info)?;
+    create_vmgenid_node(&mut fdt_writer, vmgenid)?;
 
     // End Header node.
     fdt_writer.end_node(root)?;
@@ -219,12 +222,34 @@ fn create_cpu_nodes(fdt: &mut FdtWriter, vcpu_mpidr: &[u64]) -> Result<(), FdtEr
 }
 
 fn create_memory_node(fdt: &mut FdtWriter, guest_mem: &GuestMemoryMmap) -> Result<(), FdtError> {
-    let mem_size = guest_mem.last_addr().raw_value() - super::layout::DRAM_MEM_START + 1;
+    let mem_size = guest_mem.last_addr().raw_value()
+        - super::layout::DRAM_MEM_START
+        - super::layout::SYSTEM_MEM_SIZE
+        + 1;
     // See https://github.com/torvalds/linux/blob/master/Documentation/devicetree/booting-without-of.txt#L960
     // for an explanation of this.
-    let mem_reg_prop = &[super::layout::DRAM_MEM_START, mem_size];
 
-    let mem = fdt.begin_node("memory")?;
+    // On ARM we reserve ACPI memory so that it can be utilized for devices like VMGenID
+    // to send data to kernel modules.
+    // Linux does not allow remapping of system memory however,
+    // without remap, kernel module cannot get virtual addresses to read data from ACPI region.
+    // Reserving a memory region allows Linux kernel modules to remap
+    // and thus read this region.
+    let mem = fdt.begin_node("memory@acpi")?;
+    fdt.property_array_u64(
+        "reg",
+        &[
+            super::layout::DRAM_MEM_START,
+            super::layout::SYSTEM_MEM_SIZE,
+        ],
+    )?;
+    fdt.end_node(mem)?;
+
+    let mem_reg_prop = &[
+        super::layout::DRAM_MEM_START + super::layout::SYSTEM_MEM_SIZE,
+        mem_size,
+    ];
+    let mem = fdt.begin_node("memory@ram")?;
     fdt.property_string("device_type", "memory")?;
     fdt.property_array_u64("reg", mem_reg_prop)?;
     fdt.end_node(mem)?;
@@ -258,6 +283,22 @@ fn create_chosen_node(
     Ok(())
 }
 
+fn create_vmgenid_node(fdt: &mut FdtWriter, vmgenid: &Option<VmGenId>) -> Result<(), FdtError> {
+    if let Some(vmgenid_info) = vmgenid {
+        let compatible = "microsoft,vmgenid";
+
+        let vmgenid = fdt.begin_node("vmgenid")?;
+        fdt.property_string("compatible", compatible)?;
+        fdt.property_array_u64("reg", &[vmgenid_info.guest_address.0, 4096])?;
+        fdt.property_array_u32(
+            "interrupts",
+            &[GIC_FDT_IRQ_TYPE_SPI, vmgenid_info.gsi, IRQ_TYPE_EDGE_RISING],
+        )?;
+        fdt.end_node(vmgenid)?;
+    }
+    Ok(())
+}
+
 fn create_gic_node(fdt: &mut FdtWriter, gic_device: &GICDevice) -> Result<(), FdtError> {
     let interrupt = fdt.begin_node("intc")?;
     fdt.property_string("compatible", gic_device.fdt_compatibility())?;
@@ -492,6 +533,7 @@ mod tests {
             &dev_info,
             &gic,
             &None,
+            &None,
         )
         .unwrap();
     }
@@ -516,6 +558,7 @@ mod tests {
             &HashMap::<(DeviceType, std::string::String), MMIODeviceInfo>::new(),
             &gic,
             &None,
+            &None,
         )
         .unwrap();
 
@@ -576,6 +619,7 @@ mod tests {
             CString::new("console=tty0").unwrap(),
             &HashMap::<(DeviceType, std::string::String), MMIODeviceInfo>::new(),
             &gic,
+            &None,
             &Some(initrd),
         )
         .unwrap();

src/vmm/src/arch/aarch64/layout.rs

@@ -54,6 +54,12 @@ pub const DRAM_MEM_START: u64 = 0x8000_0000; // 2 GB.
 /// The maximum RAM size.
 pub const DRAM_MEM_MAX_SIZE: usize = 0x00FF_8000_0000; // 1024 - 2 = 1022G.
 
+/// Start of RAM on 64 bit ARM.
+pub const SYSTEM_MEM_START: u64 = DRAM_MEM_START;
+
+/// This is used by ACPI device manager for acpi tables or devices like vmgenid
+pub const SYSTEM_MEM_SIZE: u64 = 0x20_0000;
+
 /// Kernel command line maximum size.
 /// As per `arch/arm64/include/uapi/asm/setup.h`.
 pub const CMDLINE_MAX_SIZE: usize = 2048;

src/vmm/src/arch/aarch64/mod.rs

@@ -20,6 +20,7 @@ use std::fmt::Debug;
 pub use self::fdt::DeviceInfoForFDT;
 use self::gic::GICDevice;
 use crate::arch::DeviceType;
+use crate::devices::acpi::vmgenid::VmGenId;
 use crate::vstate::memory::{Address, GuestAddress, GuestMemory, GuestMemoryMmap};
 
 /// Errors thrown while configuring aarch64 system.
@@ -60,6 +61,7 @@ pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug, S: std::hash::Build
     vcpu_mpidr: Vec<u64>,
     device_info: &HashMap<(DeviceType, String), T, S>,
     gic_device: &GICDevice,
+    vmgenid: &Option<VmGenId>,
     initrd: &Option<super::InitrdConfig>,
 ) -> Result<(), ConfigurationError> {
     fdt::create_fdt(
@@ -68,14 +70,15 @@ pub fn configure_system<T: DeviceInfoForFDT + Clone + Debug, S: std::hash::Build
         cmdline_cstring,
         device_info,
         gic_device,
+        vmgenid,
         initrd,
     )?;
     Ok(())
 }
 
 /// Returns the memory address where the kernel could be loaded.
 pub fn get_kernel_start() -> u64 {
-    layout::DRAM_MEM_START
+    layout::SYSTEM_MEM_START + layout::SYSTEM_MEM_SIZE
 }
 
 /// Returns the memory address where the initrd could be loaded.

src/vmm/src/arch/mod.rs

@@ -12,8 +12,8 @@ pub mod aarch64;
 #[cfg(target_arch = "aarch64")]
 pub use aarch64::{
     arch_memory_regions, configure_system, get_kernel_start, initrd_load_addr,
-    layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE, layout::IRQ_MAX, ConfigurationError, MMIO_MEM_SIZE,
-    MMIO_MEM_START,
+    layout::CMDLINE_MAX_SIZE, layout::IRQ_BASE, layout::IRQ_MAX, layout::SYSTEM_MEM_SIZE,
+    layout::SYSTEM_MEM_START, ConfigurationError, MMIO_MEM_SIZE, MMIO_MEM_START,
 };
 
 /// Module for x86_64 related functionality.

src/vmm/src/builder.rs

@@ -36,18 +36,14 @@ use crate::cpu_config::templates::{
     CpuConfiguration, CustomCpuTemplate, GetCpuTemplate, GetCpuTemplateError, GuestConfigError,
     KvmCapability,
 };
-#[cfg(target_arch = "x86_64")]
 use crate::device_manager::acpi::ACPIDeviceManager;
 #[cfg(target_arch = "x86_64")]
 use crate::device_manager::legacy::PortIODeviceManager;
 use crate::device_manager::mmio::MMIODeviceManager;
-use crate::device_manager::persist::MMIODevManagerConstructorArgs;
-#[cfg(target_arch = "x86_64")]
 use crate::device_manager::persist::{
-    ACPIDeviceManagerConstructorArgs, ACPIDeviceManagerRestoreError,
+    ACPIDeviceManagerConstructorArgs, ACPIDeviceManagerRestoreError, MMIODevManagerConstructorArgs,
 };
 use crate::device_manager::resources::ResourceAllocator;
-#[cfg(target_arch = "x86_64")]
 use crate::devices::acpi::vmgenid::{VmGenId, VmGenIdError};
 use crate::devices::legacy::serial::SerialOut;
 #[cfg(target_arch = "aarch64")]
@@ -79,7 +75,6 @@ pub enum StartMicrovmError {
     /// Unable to attach block device to Vmm: {0}
     AttachBlockDevice(io::Error),
     /// Unable to attach the VMGenID device: {0}
-    #[cfg(target_arch = "x86_64")]
     AttachVmgenidDevice(kvm_ioctls::Error),
     /// System configuration error: {0}
     ConfigureSystem(crate::arch::ConfigurationError),
@@ -93,7 +88,6 @@ pub enum StartMicrovmError {
     #[cfg(target_arch = "x86_64")]
     CreateLegacyDevice(device_manager::legacy::LegacyDeviceError),
     /// Error creating VMGenID device: {0}
-    #[cfg(target_arch = "x86_64")]
     CreateVMGenID(VmGenIdError),
     /// Invalid Memory Configuration: {0}
     GuestMemory(crate::vstate::memory::MemoryError),
@@ -175,7 +169,6 @@ fn create_vmm_and_vcpus(
     let mmio_device_manager = MMIODeviceManager::new();
 
     // Instantiate ACPI device manager.
-    #[cfg(target_arch = "x86_64")]
     let acpi_device_manager = ACPIDeviceManager::new();
 
     // For x86_64 we need to create the interrupt controller before calling `KVM_CREATE_VCPUS`
@@ -233,7 +226,6 @@ fn create_vmm_and_vcpus(
         mmio_device_manager,
         #[cfg(target_arch = "x86_64")]
         pio_device_manager,
-        #[cfg(target_arch = "x86_64")]
         acpi_device_manager,
     };
 
@@ -347,7 +339,6 @@ pub fn build_microvm_for_boot(
     #[cfg(target_arch = "aarch64")]
     attach_legacy_devices_aarch64(event_manager, &mut vmm, &mut boot_cmdline).map_err(Internal)?;
 
-    #[cfg(target_arch = "x86_64")]
     attach_vmgenid_device(&mut vmm)?;
 
     configure_system_for_boot(
@@ -449,7 +440,6 @@ pub enum BuildMicrovmFromSnapshotError {
     /// Failed to apply VMM secccomp filter: {0}
     SeccompFiltersInternal(#[from] seccompiler::InstallationError),
     /// Failed to restore ACPI device manager: {0}
-    #[cfg(target_arch = "x86_64")]
     ACPIDeviManager(#[from] ACPIDeviceManagerRestoreError),
     /// VMGenID update failed: {0}
     VMGenIDUpdate(std::io::Error),
@@ -532,7 +522,6 @@ pub fn build_microvm_from_snapshot(
             .map_err(MicrovmStateError::RestoreDevices)?;
     vmm.emulate_serial_init()?;
 
-    #[cfg(target_arch = "x86_64")]
     {
         let acpi_ctor_args = ACPIDeviceManagerConstructorArgs {
             mem: &guest_memory,
@@ -859,6 +848,7 @@ pub fn configure_system_for_boot(
             vcpu_mpidr,
             vmm.mmio_device_manager.get_device_info(),
             vmm.vm.get_irqchip(),
+            &vmm.acpi_device_manager.vmgenid,
             initrd,
         )
         .map_err(ConfigureSystem)?;
@@ -908,7 +898,6 @@ pub(crate) fn attach_boot_timer_device(
     Ok(())
 }
 
-#[cfg(target_arch = "x86_64")]
 fn attach_vmgenid_device(vmm: &mut Vmm) -> Result<(), StartMicrovmError> {
     let vmgenid = VmGenId::new(&vmm.guest_memory, &mut vmm.resource_allocator)
         .map_err(StartMicrovmError::CreateVMGenID)?;
@@ -1117,7 +1106,6 @@ pub mod tests {
         let mut vm = Vm::new(vec![]).unwrap();
         vm.memory_init(&guest_memory, false).unwrap();
         let mmio_device_manager = MMIODeviceManager::new();
-        #[cfg(target_arch = "x86_64")]
         let acpi_device_manager = ACPIDeviceManager::new();
         #[cfg(target_arch = "x86_64")]
         let pio_device_manager = PortIODeviceManager::new(
@@ -1158,7 +1146,6 @@ pub mod tests {
             mmio_device_manager,
             #[cfg(target_arch = "x86_64")]
             pio_device_manager,
-            #[cfg(target_arch = "x86_64")]
             acpi_device_manager,
         }
     }

src/vmm/src/device_manager/mod.rs

@@ -6,7 +6,6 @@
 // found in the THIRD-PARTY file.
 
 /// ACPI device manager.
-#[cfg(target_arch = "x86_64")]
 pub mod acpi;
 /// Legacy Device Manager.
 pub mod legacy;

src/vmm/src/device_manager/persist.rs

@@ -12,13 +12,11 @@ use log::{error, warn};
 use serde::{Deserialize, Serialize};
 use vm_allocator::AllocPolicy;
 
-#[cfg(target_arch = "x86_64")]
 use super::acpi::ACPIDeviceManager;
 use super::mmio::*;
 use super::resources::ResourceAllocator;
 #[cfg(target_arch = "aarch64")]
 use crate::arch::DeviceType;
-#[cfg(target_arch = "x86_64")]
 use crate::devices::acpi::vmgenid::{VMGenIDState, VMGenIdConstructorArgs, VmGenId, VmGenIdError};
 use crate::devices::virtio::balloon::persist::{BalloonConstructorArgs, BalloonState};
 use crate::devices::virtio::balloon::{Balloon, BalloonError};
@@ -230,20 +228,17 @@ impl fmt::Debug for MMIODevManagerConstructorArgs<'_> {
     }
 }
 
-#[cfg(target_arch = "x86_64")]
 #[derive(Default, Debug, Clone, Serialize, Deserialize)]
 pub struct ACPIDeviceManagerState {
     vmgenid: Option<VMGenIDState>,
 }
 
-#[cfg(target_arch = "x86_64")]
 pub struct ACPIDeviceManagerConstructorArgs<'a> {
     pub mem: &'a GuestMemoryMmap,
     pub resource_allocator: &'a mut ResourceAllocator,
     pub vm: &'a VmFd,
 }
 
-#[cfg(target_arch = "x86_64")]
 #[derive(Debug, thiserror::Error, displaydoc::Display)]
 pub enum ACPIDeviceManagerRestoreError {
     /// Could not register device: {0}
@@ -252,7 +247,6 @@ pub enum ACPIDeviceManagerRestoreError {
     VMGenID(#[from] VmGenIdError),
 }
 
-#[cfg(target_arch = "x86_64")]
 impl<'a> Persist<'a> for ACPIDeviceManager {
     type State = ACPIDeviceManagerState;
     type ConstructorArgs = ACPIDeviceManagerConstructorArgs<'a>;

src/vmm/src/device_manager/resources.rs

@@ -20,7 +20,6 @@ pub struct ResourceAllocator {
     // Allocator for memory in the MMIO address space
     mmio_memory: AddressAllocator,
     // Memory allocator for system data
-    #[cfg(target_arch = "x86_64")]
     system_memory: AddressAllocator,
 }
 
@@ -30,7 +29,6 @@ impl ResourceAllocator {
         Ok(Self {
             gsi_allocator: IdAllocator::new(arch::IRQ_BASE, arch::IRQ_MAX)?,
             mmio_memory: AddressAllocator::new(arch::MMIO_MEM_START, arch::MMIO_MEM_SIZE)?,
-            #[cfg(target_arch = "x86_64")]
             system_memory: AddressAllocator::new(arch::SYSTEM_MEM_START, arch::SYSTEM_MEM_SIZE)?,
         })
     }
@@ -86,7 +84,6 @@ impl ResourceAllocator {
     /// * `size` - The size in bytes of the memory to allocate
     /// * `alignment` - The alignment of the address of the first byte
     /// * `policy` - A [`vm_allocator::AllocPolicy`] variant for determining the allocation policy
-    #[cfg(target_arch = "x86_64")]
     pub fn allocate_system_memory(
         &mut self,
         size: u64,

src/vmm/src/devices/mod.rs

@@ -9,7 +9,6 @@
 
 use std::io;
 
-#[cfg(target_arch = "x86_64")]
 pub mod acpi;
 pub mod bus;
 pub mod legacy;