From 565359b0d41b75cf02290d03f428fb0ed83207f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Rodr=C3=ADguez?= Date: Sat, 25 May 2024 20:40:06 +0100 Subject: [PATCH 1/2] style: format files using clang-format-19 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Rodríguez --- src/arch/armv8/armv8-r/mpu.c | 2 +- src/core/inc/config.h | 22 +++++++++++++--------- src/core/inc/cpu.h | 6 +++--- src/core/inc/types.h | 8 ++++---- src/lib/inc/bit.h | 8 ++++---- src/lib/inc/util.h | 16 ++++++++-------- 6 files changed, 33 insertions(+), 29 deletions(-) diff --git a/src/arch/armv8/armv8-r/mpu.c b/src/arch/armv8/armv8-r/mpu.c index 8acf13fe2..cd715a2e6 100644 --- a/src/arch/armv8/armv8-r/mpu.c +++ b/src/arch/armv8/armv8-r/mpu.c @@ -116,7 +116,7 @@ static inline void mpu_entry_set_perms(struct mp_region* mpr, struct mpu_perms m bool el1_priv = mpu_perms.el1 != PERM_NONE; perms_t perms = mpu_perms.el1 | mpu_perms.el2; - mpr->mem_flags.prbar &= (uint16_t) ~(PRBAR_PERMS_FLAGS_MSK); + mpr->mem_flags.prbar &= (uint16_t)~(PRBAR_PERMS_FLAGS_MSK); if (perms & PERM_W) { mpr->mem_flags.prbar |= PRBAR_AP_RW_EL2; } else { diff --git a/src/core/inc/config.h b/src/core/inc/config.h index 7154e7e2d..7cdefc48b 100644 --- a/src/core/inc/config.h +++ b/src/core/inc/config.h @@ -30,23 +30,27 @@ // clang-format on #define VM_IMAGE_OFFSET(img_name) ((paddr_t) & _##img_name##_vm_beg) -#define VM_IMAGE_SIZE(img_name) ((size_t) & _##img_name##_vm_size) +#define VM_IMAGE_SIZE(img_name) ((size_t)&_##img_name##_vm_size) #else #define VM_IMAGE(img_name, img_path) #define VM_IMAGE_OFFSET(img_name) ((paddr_t)0) #define VM_IMAGE_SIZE(img_name) ((size_t)0) #endif -#define VM_IMAGE_BUILTIN(img_name, image_base_addr) \ - { \ - .base_addr = image_base_addr, .load_addr = VM_IMAGE_OFFSET(img_name), \ - .size = VM_IMAGE_SIZE(img_name), .separately_loaded = false, \ +#define VM_IMAGE_BUILTIN(img_name, image_base_addr) \ + { \ + .base_addr = image_base_addr, \ + .load_addr = VM_IMAGE_OFFSET(img_name), \ + .size = VM_IMAGE_SIZE(img_name), \ + .separately_loaded = false, \ } -#define VM_IMAGE_LOADED(image_base_addr, image_load_addr, image_size) \ - { \ - .base_addr = image_base_addr, .load_addr = image_load_addr, .size = image_size, \ - .separately_loaded = true, \ +#define VM_IMAGE_LOADED(image_base_addr, image_load_addr, image_size) \ + { \ + .base_addr = image_base_addr, \ + .load_addr = image_load_addr, \ + .size = image_size, \ + .separately_loaded = true, \ } /* CONFIG_HEADER is just defined for compatibility with older configs */ diff --git a/src/core/inc/cpu.h b/src/core/inc/cpu.h index d14fac387..70c915336 100644 --- a/src/core/inc/cpu.h +++ b/src/core/inc/cpu.h @@ -48,9 +48,9 @@ void cpu_send_msg(cpuid_t cpu, struct cpu_msg* msg); typedef void (*cpu_msg_handler_t)(uint32_t event, uint64_t data); -#define CPU_MSG_HANDLER(handler, handler_id) \ - __attribute__((section(".ipi_cpumsg_handlers"), used)) \ - cpu_msg_handler_t __cpumsg_handler_##handler = handler; \ +#define CPU_MSG_HANDLER(handler, handler_id) \ + __attribute__((section(".ipi_cpumsg_handlers"), \ + used)) cpu_msg_handler_t __cpumsg_handler_##handler = handler; \ __attribute__((section(".ipi_cpumsg_handlers_id"), used)) volatile const size_t handler_id; struct cpu_synctoken { diff --git a/src/core/inc/types.h b/src/core/inc/types.h index 3b7ca4fec..0bd35ef88 100644 --- a/src/core/inc/types.h +++ b/src/core/inc/types.h @@ -25,23 +25,23 @@ typedef signed long ssize_t; typedef unsigned long asid_t; typedef unsigned long vmid_t; -#define INVALID_VMID ((vmid_t)-1) +#define INVALID_VMID ((vmid_t) - 1) typedef uintptr_t paddr_t; typedef uintptr_t regaddr_t; typedef uintptr_t vaddr_t; -#define MAX_VA ((vaddr_t)-1) +#define MAX_VA ((vaddr_t) - 1) #define INVALID_VA MAX_VA typedef size_t mpid_t; -#define INVALID_MPID ((mpid_t)-1) +#define INVALID_MPID ((mpid_t) - 1) typedef unsigned long colormap_t; typedef unsigned long cpuid_t; typedef unsigned long vcpuid_t; typedef unsigned long cpumap_t; -#define INVALID_CPUID ((cpuid_t)-1) +#define INVALID_CPUID ((cpuid_t) - 1) typedef unsigned irqid_t; diff --git a/src/lib/inc/bit.h b/src/lib/inc/bit.h index 168b6061d..8512fdadb 100644 --- a/src/lib/inc/bit.h +++ b/src/lib/inc/bit.h @@ -14,9 +14,9 @@ * word length masks with the cost of an extra shift instruction. For static masks, there should be * no extra costs. */ -#define BIT32_MASK(OFF, LEN) ((((UINT32_C(1) << ((LEN)-1)) << 1) - 1) << (OFF)) -#define BIT64_MASK(OFF, LEN) ((((UINT64_C(1) << ((LEN)-1)) << 1) - 1) << (OFF)) -#define BIT_MASK(OFF, LEN) (((((1UL) << ((LEN)-1)) << 1) - 1) << (OFF)) +#define BIT32_MASK(OFF, LEN) ((((UINT32_C(1) << ((LEN) - 1)) << 1) - 1) << (OFF)) +#define BIT64_MASK(OFF, LEN) ((((UINT64_C(1) << ((LEN) - 1)) << 1) - 1) << (OFF)) +#define BIT_MASK(OFF, LEN) (((((1UL) << ((LEN) - 1)) << 1) - 1) << (OFF)) #ifndef __ASSEMBLER__ @@ -52,7 +52,7 @@ mask <<= 1U; \ pos++; \ } \ - return (mask != 0U) ? pos : (ssize_t)-1; \ + return (mask != 0U) ? pos : (ssize_t) - 1; \ } \ static inline size_t PRE##_count(TYPE word) \ { \ diff --git a/src/lib/inc/util.h b/src/lib/inc/util.h index dd9066d39..2cbc6de69 100644 --- a/src/lib/inc/util.h +++ b/src/lib/inc/util.h @@ -9,20 +9,20 @@ /* UTILITY MACROS */ /* align VAL to TO which must be power a two */ -#define ALIGN(VAL, TO) ((((VAL) + (TO)-1) / (TO)) * TO) +#define ALIGN(VAL, TO) ((((VAL) + (TO) - 1) / (TO)) * TO) #define IS_ALIGNED(VAL, TO) (!((VAL) % (TO))) -#define ALIGN_FLOOR(VAL, TO) ((VAL) & ~((TO)-1)) +#define ALIGN_FLOOR(VAL, TO) ((VAL) & ~((TO) - 1)) #define NUM_PAGES(SZ) (ALIGN(SZ, PAGE_SIZE) / PAGE_SIZE) -#define PAGE_OFFSET_MASK ((PAGE_SIZE)-1) +#define PAGE_OFFSET_MASK ((PAGE_SIZE) - 1) #define PAGE_FRAME_MASK (~(PAGE_OFFSET_MASK)) #define SR_OR(VAL, SHIFT) (((VAL) >> (SHIFT)) | VAL) /* Next Power Of Two */ -#define NPOT(VAL) \ - ((SR_OR(((VAL)-1), 1) | SR_OR(SR_OR(((VAL)-1), 1), 2) | \ - SR_OR(SR_OR(SR_OR(((VAL)-1), 1), 2), 4) | \ - SR_OR(SR_OR(SR_OR(SR_OR(((VAL)-1), 1), 2), 4), 8) | \ - SR_OR(SR_OR(SR_OR(SR_OR(SR_OR(((VAL)-1), 1), 2), 4), 8), 16)) + \ +#define NPOT(VAL) \ + ((SR_OR(((VAL) - 1), 1) | SR_OR(SR_OR(((VAL) - 1), 1), 2) | \ + SR_OR(SR_OR(SR_OR(((VAL) - 1), 1), 2), 4) | \ + SR_OR(SR_OR(SR_OR(SR_OR(((VAL) - 1), 1), 2), 4), 8) | \ + SR_OR(SR_OR(SR_OR(SR_OR(SR_OR(((VAL) - 1), 1), 2), 4), 8), 16)) + \ 1) /* Previous Power Of Two */ From 00f78a11e1e2744b45793ec7935f78626f8a9846 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20Rodr=C3=ADguez?= Date: Sat, 25 May 2024 20:37:38 +0100 Subject: [PATCH 2/2] update(riscv/iommu): add multi-lvl DDT support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Manuel Rodríguez --- src/arch/riscv/inc/arch/iommu.h | 11 + src/arch/riscv/inc/arch/platform.h | 1 - src/arch/riscv/iommu.c | 526 ++++++++++++++++++++++++----- 3 files changed, 461 insertions(+), 77 deletions(-) diff --git a/src/arch/riscv/inc/arch/iommu.h b/src/arch/riscv/inc/arch/iommu.h index 63398004d..d6605bd87 100644 --- a/src/arch/riscv/inc/arch/iommu.h +++ b/src/arch/riscv/inc/arch/iommu.h @@ -13,4 +13,15 @@ struct iommu_vm_arch { EMPTY_STRUCT_FIELDS }; +struct vm; +void rv_iommu_fq_irq_handler(irqid_t irq_id); +void alloc_2lvl_vptrs(void); +void up_1lvl_to_2lvl(void); +void alloc_3lvl_vptrs(void); +void up_2lvl_to_3lvl(void); +void ddt_init(void); +void rv_iommu_init(void); +bool rv_iommu_alloc_did(deviceid_t dev_id); +void rv_iommu_write_ddt(deviceid_t dev_id, struct vm* vm, paddr_t root_pt); + #endif /* __IOMMU_ARCH_H__ */ diff --git a/src/arch/riscv/inc/arch/platform.h b/src/arch/riscv/inc/arch/platform.h index c7560c234..c1c24cd3e 100644 --- a/src/arch/riscv/inc/arch/platform.h +++ b/src/arch/riscv/inc/arch/platform.h @@ -23,7 +23,6 @@ struct arch_platform { struct { paddr_t base; // Base address of the IOMMU mmapped IF - unsigned mode; // Overall IOMMU mode (Off, Bypass, DDT-lvl) irqid_t fq_irq_id; // Fault Queue IRQ ID (wired) } iommu; diff --git a/src/arch/riscv/iommu.c b/src/arch/riscv/iommu.c index 65b62f600..e1474e9bb 100644 --- a/src/arch/riscv/iommu.c +++ b/src/arch/riscv/iommu.c @@ -8,11 +8,6 @@ #include #include #include -#include - -// We initially use a 1-LVL DDT with DC in extended format -// N entries = 4kiB / 64 B p/ entry = 64 Entries -#define DDT_N_ENTRIES (64) #define FQ_N_ENTRIES (64) #define FQ_LOG2SZ_1 (5ULL) @@ -34,6 +29,12 @@ #define RV_IOMMU_FCTL_WSI_BIT (0x1UL << 1) #define RV_IOMMU_FCTL_DEFAULT (RV_IOMMU_FCTL_WSI_BIT) +// Device Directory Table Pointer register +#define RV_IOMMU_DDTP_MODE_BARE (1ULL) +#define RV_IOMMU_DDTP_MODE_1LVL (2ULL) +#define RV_IOMMU_DDTP_MODE_2LVL (3ULL) +#define RV_IOMMU_DDTP_MODE_3LVL (4ULL) + #define RV_IOMMU_DDTP_PPN_OFF (10) #define RV_IOMMU_DDTP_PPN_LEN (44) #define RV_IOMMU_DDTP_PPN_MASK BIT64_MASK(RV_IOMMU_DDTP_PPN_OFF, RV_IOMMU_DDTP_PPN_LEN) @@ -108,6 +109,28 @@ struct riscv_iommu_regmap { } __attribute__((__packed__, __aligned__(PAGE_SIZE))); // # RISC-V IOMMU Device Directory Table + +// device_id masks +#define DDI_2_BASE_OFF (16) +#define DDI_1_BASE_OFF (7) +#define DDI_0_BASE_OFF (0) + +#define DDI_2_BASE_LEN (8) +#define DDI_1_BASE_LEN (9) +#define DDI_0_BASE_LEN (7) + +#define DDI_2_MASK_BASE BIT32_MASK(DDI_2_BASE_OFF, DDI_2_BASE_LEN) +#define DDI_1_MASK_BASE BIT32_MASK(DDI_1_BASE_OFF, DDI_1_BASE_LEN) +#define DDI_0_MASK_BASE BIT32_MASK(DDI_0_BASE_OFF, DDI_0_BASE_LEN) + +// DC fields + +// Non-leaf +#define RV_IOMMU_DDT_NL_PPN_OFF (10) +#define RV_IOMMU_DDT_NL_PPN_LEN (44) +#define RV_IOMMU_DDT_NL_PPN_MASK BIT64_MASK(RV_IOMMU_DDT_NL_PPN_OFF, RV_IOMMU_DDT_NL_PPN_LEN) + +// Leaf #define RV_IOMMU_DC_VALID_BIT (1ULL << 0) #define RV_IOMMU_DC_IOHGATP_PPN_OFF (0) @@ -121,18 +144,25 @@ struct riscv_iommu_regmap { #define RV_IOMMU_DC_IOHGATP_MODE_OFF (60) #define RV_IOMMU_IOHGATP_SV39X4 (8ULL << RV_IOMMU_DC_IOHGATP_MODE_OFF) +typedef uint64_t* ddt_bitmap_t; + +#define DDT_BITMAP_GET(BITMAP, DEV_ID) (BITMAP[DEV_ID / 64] & (1ULL << (DEV_ID % 64))) +#define DDT_BITMAP_SET(BITMAP, DEV_ID) (BITMAP[DEV_ID / 64] |= (1ULL << (DEV_ID % 64))) + +// Abstract Device Context Struct +// Each entry may represent 1 or 2 entries, depending on MSI translation support struct ddt_entry { - uint64_t tc; - uint64_t iohgatp; - uint64_t ta; - uint64_t fsc; - uint64_t msiptp; - uint64_t msi_addr_mask; - uint64_t msi_addr_pattern; - uint64_t __rsv; + uint64_t dw_1; + uint64_t dw_2; + uint64_t dw_3; + uint64_t dw_4; + uint64_t dw_5; + uint64_t dw_6; + uint64_t dw_7; + uint64_t dw_8; } __attribute__((__packed__)); -// # Fault Queue Record +// # Fault Queue #define RV_IOMMU_FQ_CAUSE_OFF (0) #define RV_IOMMU_FQ_CAUSE_LEN (12) #define RV_IOMMU_FQ_DID_OFF (40) @@ -149,15 +179,25 @@ struct fq_entry { // TODO: Add CQ struct riscv_iommu_hw { volatile struct riscv_iommu_regmap* reg_ptr; - volatile struct ddt_entry* ddt; + volatile uint64_t* vddt_lvl0; + volatile uint64_t** vddt_lvl1; + volatile uint64_t*** vddt_lvl2; volatile struct fq_entry* fq; }; +// Private data struct riscv_iommu_priv { - struct riscv_iommu_hw hw; + struct riscv_iommu_hw hw; // HW ptrs + // DDT spinlock_t ddt_lock; - BITMAP_ALLOC(ddt_bitmap, DDT_N_ENTRIES); + ddt_bitmap_t ddt_bitmap; + + // Misc + unsigned long long caps; // caps register + uint8_t supported_modes; // supported IOMMU modes + unsigned long long iommu_mode; // current IOMMU mode + bool msi_support; // MSI translation support flag }; struct riscv_iommu_priv rv_iommu; @@ -165,36 +205,32 @@ struct riscv_iommu_priv rv_iommu; /**************** RISC-V IOMMU specific functions ****************/ /** - * Read and check IOMMU caps + * Read and check IOMMU caps */ static void rv_iommu_check_features(void) { - uint64_t caps = rv_iommu.hw.reg_ptr->caps; - uint64_t version = bit64_extract(caps, RV_IOMMU_CAPS_VERSION_OFF, RV_IOMMU_CAPS_VERSION_LEN); + uint64_t version = + bit64_extract(rv_iommu.caps, RV_IOMMU_CAPS_VERSION_OFF, RV_IOMMU_CAPS_VERSION_LEN); if (version != RV_IOMMU_SUPPORTED_VERSION) { - ERROR("RISC-V IOMMU unsupported version: %d", version); + ERROR("RV_IOMMU: Unsupported version: %d", version); } - if (!(caps & RV_IOMMU_CAPS_SV39X4_BIT)) { - ERROR("RISC-V IOMMU HW does not support Sv39x4"); + if (!(rv_iommu.caps & RV_IOMMU_CAPS_SV39X4_BIT)) { + ERROR("RV IOMMU: Sv39x4 not supported"); } - if (!(caps & RV_IOMMU_CAPS_MSI_FLAT_BIT)) { - WARNING("RISC-V IOMMU HW does not support MSI Address Translation " - "(basic-translate mode)"); - } - - uint64_t igs = bit64_extract(caps, RV_IOMMU_CAPS_IGS_OFF, RV_IOMMU_CAPS_IGS_LEN); + uint64_t igs = bit64_extract(rv_iommu.caps, RV_IOMMU_CAPS_IGS_OFF, RV_IOMMU_CAPS_IGS_LEN); if (!igs) { - ERROR("RISC-V IOMMU HW does not support WSI generation"); + WARNING("RV IOMMU: WSI generation not supported. MSI generation is currently not supported " + "by Bao"); } } /** - * RISC-V IOMMU Fault Queue IRQ handler. + * RISC-V IOMMU Fault Queue IRQ handler. */ -static void rv_iommu_fq_irq_handler(irqid_t irq_id) +void rv_iommu_fq_irq_handler(irqid_t irq_id) { UNUSED_ARG(irq_id); @@ -245,9 +281,198 @@ static void rv_iommu_fq_irq_handler(irqid_t irq_id) } /** - * Init and enable RISC-V IOMMU. + * Allocate one page to store first-level table virtual ptrs + * Indexed with DDI[1] + */ +void alloc_2lvl_vptrs(void) +{ + // 2^9 * 8 bytes per non-leaf entry = 4096 bytes (1 page) + volatile uint64_t** lvl1_vaddr = (volatile uint64_t**)mem_alloc_page(1, SEC_HYP_GLOBAL, true); + memset((void*)lvl1_vaddr, 0, 0x1000); + rv_iommu.hw.vddt_lvl1 = lvl1_vaddr; +} + +/** + * Upgrade the IOMMU mode from 1LVL to 2LVL. + * Rearrange DDT and update ddtp + */ +void up_1lvl_to_2lvl(void) +{ + // As the IOMMU is in 1LVL mode, rv_iommu.hw.vddt_lvl0 (and thus the ddtp) register points to a + // table that may have leaf entries in it (DCs). + + // 1. Allocate a new page to work as the root table + uint64_t* new_root_vaddr = (uint64_t*)mem_alloc_page(1, SEC_HYP_GLOBAL, true); + memset((void*)new_root_vaddr, 0, 0x1000); + + // 2. Configure its first entry to point to the physical address of the table pointed by + // rv_iommu.hw.vddt_lvl0 + paddr_t nl_paddr; + mem_translate(&cpu()->as, (vaddr_t)rv_iommu.hw.vddt_lvl0, &nl_paddr); + new_root_vaddr[0] = ((nl_paddr >> 2) & RV_IOMMU_DDT_NL_PPN_MASK) | RV_IOMMU_DC_VALID_BIT; + + // 3. Point rv_iommu.hw.vddt_lvl1[0] to the table pointed by rv_iommu.hw.vddt_lvl0 with the DCs + // (now lvl1 table) + rv_iommu.hw.vddt_lvl1[0] = rv_iommu.hw.vddt_lvl0; + + // 4. Point rv_iommu.hw.vddt_lvl0 to the new root table + rv_iommu.hw.vddt_lvl0 = new_root_vaddr; + + // 5. Update ddtp with the base physical address of the new table pointed by + // rv_iommu.hw.vddt_lvl0 + rv_iommu.iommu_mode = RV_IOMMU_DDTP_MODE_2LVL; + + paddr_t ddt_paddr; + mem_translate(&cpu()->as, (vaddr_t)new_root_vaddr, &ddt_paddr); + rv_iommu.hw.reg_ptr->ddtp = rv_iommu.iommu_mode | ((ddt_paddr >> 2) & RV_IOMMU_DDTP_PPN_MASK); +} + +/* + * Allocate 512 pages to store second-level table virtual ptrs + * Indexed with DDI[2] DDI[1] + */ +void alloc_3lvl_vptrs(void) +{ + // 2^9 * 8 bytes per non-leaf entry = 4096 bytes (1 page) + volatile uint64_t*** lvl2_vaddr = (volatile uint64_t***)mem_alloc_page(1, SEC_HYP_GLOBAL, true); + memset((void*)lvl2_vaddr, 0, 0x1000); + + // Allocate one page for each second-level table + for (size_t i = 0; i < 512; i++) { + volatile uint64_t** lvl2_vaddr_i = + (volatile uint64_t**)mem_alloc_page(1, SEC_HYP_GLOBAL, true); + memset((void*)lvl2_vaddr_i, 0, 0x1000); + lvl2_vaddr[i] = lvl2_vaddr_i; + } + + rv_iommu.hw.vddt_lvl2 = lvl2_vaddr; +} + +/** + * Upgrade the IOMMU mode from 2LVL to 3LVL. + * Rearrange DDT and update ddtp + */ +void up_2lvl_to_3lvl(void) +{ + // As the IOMMU is in 2LVL mode, rv_iommu.hw.vddt_lvl0 (and thus the ddtp) register points to + // 512 tables that may have leaf entries in it (DCs). + + // 1. Allocate a new page to work as the root table + uint64_t* new_root_vaddr = (uint64_t*)mem_alloc_page(1, SEC_HYP_GLOBAL, true); + memset((void*)new_root_vaddr, 0, 0x1000); + + // 2. Configure its first entry to point to the physical address of the table pointed by + // rv_iommu.hw.vddt_lvl0 + paddr_t nl_paddr; + mem_translate(&cpu()->as, (vaddr_t)rv_iommu.hw.vddt_lvl0, &nl_paddr); + new_root_vaddr[0] = ((nl_paddr >> 2) & RV_IOMMU_DDT_NL_PPN_MASK) | RV_IOMMU_DC_VALID_BIT; + + // 3. Point rv_iommu.hw.vddt_lvl2[0][0..512-1] to the tables pointed by + // rv_iommu.hw.vddt_lvl1[0..512-1] with the DCs (now lvl2 tables) + for (size_t i = 0; i < (0x1000 / 8); i++) { + rv_iommu.hw.vddt_lvl2[0][i] = rv_iommu.hw.vddt_lvl1[i]; + } + + // 4. Point rv_iommu.hw.vddt_lvl1[0] to the table pointed by rv_iommu.hw.vddt_lvl0 with the + // non-leaf entries (now lvl1 table) + rv_iommu.hw.vddt_lvl1[0] = rv_iommu.hw.vddt_lvl0; + + // 5. Point rv_iommu.hw.vddt_lvl0 to the new root table + rv_iommu.hw.vddt_lvl0 = new_root_vaddr; + + // 6. Update ddtp with the base physical address of the new table pointed by + // rv_iommu.hw.vddt_lvl0 + rv_iommu.iommu_mode = RV_IOMMU_DDTP_MODE_3LVL; + + paddr_t ddt_paddr; + mem_translate(&cpu()->as, (vaddr_t)new_root_vaddr, &ddt_paddr); + rv_iommu.hw.reg_ptr->ddtp = rv_iommu.iommu_mode | ((ddt_paddr >> 2) & RV_IOMMU_DDTP_PPN_MASK); +} + +/** + * Determine supported IOMMU modes (number of DDT levels) + * Allocate memory for leaf and non-leaf DDT tables + * Program the IOMMU with the simplest supported mode + * Enable IOMMU + */ +void ddt_init(void) +{ + // Lock DDT + rv_iommu.ddt_lock = SPINLOCK_INITVAL; + + // Determine supported modes (3LVL, 2LVL, 1LVL, BARE) + uint64_t ddtp_mode = RV_IOMMU_DDTP_MODE_1LVL; + uint64_t simplest_mode = 0; + bool first = true; + rv_iommu.supported_modes = 0; + + for (size_t i = 0; i < RV_IOMMU_DDTP_MODE_3LVL; i++) { + // Probe ddtp register + rv_iommu.hw.reg_ptr->ddtp = ddtp_mode; + uint64_t ddtp_readback = rv_iommu.hw.reg_ptr->ddtp; + + if (ddtp_readback == ddtp_mode) { + rv_iommu.supported_modes |= (uint8_t)(1 << (i + 2)); + if (first) { + first = false; + simplest_mode = ddtp_mode; // save simplest mode + } + } + + (ddtp_mode)++; + } + + // 1LVL, 2LVL and 3LVL not supported + if (!simplest_mode) { + // Probe ddtp register with Bare mode + rv_iommu.hw.reg_ptr->ddtp = RV_IOMMU_DDTP_MODE_BARE; + uint64_t ddtp_readback = rv_iommu.hw.reg_ptr->ddtp; + + if (ddtp_readback == RV_IOMMU_DDTP_MODE_BARE) { + // No need to populate the DDT if IOMMU is in Bare mode + WARNING("RV_IOMMU: Only Bare mode supported"); + rv_iommu.iommu_mode = RV_IOMMU_DDTP_MODE_BARE; + return; + } else { + ERROR("RV_IOMMU: No valid IOMMU mode supported"); + } + } + + // Save IOMMU mode + rv_iommu.iommu_mode = simplest_mode; + + // Allocate memory for the DDT bitmap + rv_iommu.ddt_bitmap = (ddt_bitmap_t)mem_alloc_page(512, SEC_HYP_GLOBAL, true); + memset((void*)rv_iommu.ddt_bitmap, 0, 512 * 0x1000); + + // Simplest supported mode is 3LVL + if (rv_iommu.iommu_mode >= RV_IOMMU_DDTP_MODE_3LVL) { + alloc_3lvl_vptrs(); + } + + // Simplest supported mode is 2LVL + if (rv_iommu.iommu_mode >= RV_IOMMU_DDTP_MODE_2LVL) { + alloc_2lvl_vptrs(); + } + + vaddr_t ddt_vaddr = 0; + + // Allocate one page for the root table of the DDT + ddt_vaddr = (vaddr_t)mem_alloc_page(1, SEC_HYP_GLOBAL, true); + memset((void*)ddt_vaddr, 0, 0x1000); + rv_iommu.hw.vddt_lvl0 = (uint64_t*)ddt_vaddr; + + // Get physical address of the root table and configure ddtp + paddr_t ddt_paddr; + mem_translate(&cpu()->as, ddt_vaddr, &ddt_paddr); + rv_iommu.hw.reg_ptr->ddtp = rv_iommu.iommu_mode | ((ddt_paddr >> 2) & RV_IOMMU_DDTP_PPN_MASK); + // TODO: poll ddtp.busy +} + +/** + * Initialize IOMMU hardware */ -static void rv_iommu_init(void) +void rv_iommu_init(void) { // Map register IF (4k) vaddr_t reg_ptr = mem_alloc_map_dev(&cpu()->as, SEC_HYP_GLOBAL, INVALID_VA, @@ -255,9 +480,15 @@ static void rv_iommu_init(void) rv_iommu.hw.reg_ptr = (struct riscv_iommu_regmap*)reg_ptr; - // Read and check caps + // Read caps register + rv_iommu.caps = rv_iommu.hw.reg_ptr->caps; + + // Check caps rv_iommu_check_features(); + // Determine whether the IOMMU supports MSI translation + rv_iommu.msi_support = ((rv_iommu.caps & RV_IOMMU_CAPS_MSI_FLAT_BIT) != 0); + // Set fctl.WSI We will be first using WSI as IOMMU interrupt mechanism. Then MSIs will be // included rv_iommu.hw.reg_ptr->fctl = RV_IOMMU_FCTL_DEFAULT; @@ -295,23 +526,8 @@ static void rv_iommu_init(void) rv_iommu.hw.reg_ptr->fqcsr = RV_IOMMU_FQCSR_DEFAULT; // TODO: poll fqcsr.busy - // Init DDT bitmap - rv_iommu.ddt_lock = SPINLOCK_INITVAL; - bitmap_clear_consecutive(rv_iommu.ddt_bitmap, 0, DDT_N_ENTRIES); - - // Allocate a page of memory (aligned) for the DDT - vaddr_t ddt_vaddr = (vaddr_t)mem_alloc_page(NUM_PAGES(sizeof(struct ddt_entry) * DDT_N_ENTRIES), - SEC_HYP_GLOBAL, true); - // Clear entries - memset((void*)ddt_vaddr, 0, sizeof(struct ddt_entry) * DDT_N_ENTRIES); - rv_iommu.hw.ddt = (struct ddt_entry*)ddt_vaddr; - - // Configure ddtp with DDT base address and IOMMU mode - paddr_t ddt_paddr; - mem_translate(&cpu()->as, ddt_vaddr, &ddt_paddr); - rv_iommu.hw.reg_ptr->ddtp = - (unsigned long long)platform.arch.iommu.mode | ((ddt_paddr >> 2) & RV_IOMMU_DDTP_PPN_MASK); - // TODO: poll ddtp.busy + // Init DDT + ddt_init(); } /** @@ -321,14 +537,14 @@ static void rv_iommu_init(void) * * @returns true on success, false on error */ -static bool rv_iommu_alloc_did(deviceid_t dev_id) +bool rv_iommu_alloc_did(deviceid_t dev_id) { bool allocated; spin_lock(&rv_iommu.ddt_lock); // Check if DC already exists - if (!bitmap_get(rv_iommu.ddt_bitmap, dev_id)) { - bitmap_set(rv_iommu.ddt_bitmap, dev_id); + if (!(DDT_BITMAP_GET(rv_iommu.ddt_bitmap, dev_id))) { + DDT_BITMAP_SET(rv_iommu.ddt_bitmap, dev_id); allocated = true; } else { allocated = false; // device_id already exists @@ -339,33 +555,191 @@ static bool rv_iommu_alloc_did(deviceid_t dev_id) } /** - * Program DDT entry with base address of the root PT, VMID and translation configuration. Enable - * DC. + * Program DDT entry with base address of the root PT, VMID and translation configuration. Enable + * DC. * - * @dev_id: device_id to index DDT - * @vm: VM to which the device is being assigned - * @root_pt: Base physical address of the root second-stage PT + * @dev_id: device_id to index DDT + * @vm: VM to which the device is being assigned + * @root_pt: Base physical address of the root second-stage PT */ -static void rv_iommu_write_ddt(deviceid_t dev_id, struct vm* vm, paddr_t root_pt) +void rv_iommu_write_ddt(deviceid_t dev_id, struct vm* vm, paddr_t root_pt) { spin_lock(&rv_iommu.ddt_lock); - if (!bitmap_get(rv_iommu.ddt_bitmap, dev_id)) { + if (!(DDT_BITMAP_GET(rv_iommu.ddt_bitmap, dev_id))) { ERROR("IOMMU DC %d is not allocated", dev_id); - } else { - // Configure DC - uint64_t tc = 0; - tc |= RV_IOMMU_DC_VALID_BIT; - rv_iommu.hw.ddt[dev_id].tc = tc; - - uint64_t iohgatp = 0; - iohgatp |= ((root_pt >> 12) & RV_IOMMU_DC_IOHGATP_PPN_MASK); - iohgatp |= ((vm->id << RV_IOMMU_DC_IOHGATP_GSCID_OFF) & RV_IOMMU_DC_IOHGATP_GSCID_MASK); - iohgatp |= RV_IOMMU_IOHGATP_SV39X4; - rv_iommu.hw.ddt[dev_id].iohgatp = iohgatp; - - // TODO: Configure first-stage translation. Second-stage only by now Configure MSI - // translation } + + else { + // Get DID indexes + unsigned ddi_2 = + (dev_id & (DDI_2_MASK_BASE | ((unsigned int)rv_iommu.msi_support << 15))) >> + (DDI_2_BASE_OFF - (int)rv_iommu.msi_support); + unsigned ddi_1 = (dev_id & (DDI_1_MASK_BASE >> (unsigned int)rv_iommu.msi_support)) >> + (DDI_1_BASE_OFF - (int)rv_iommu.msi_support); + unsigned ddi_0 = (dev_id & (DDI_0_MASK_BASE >> (unsigned int)rv_iommu.msi_support)); + + // In bare mode there is no need to populate the DDT + if (rv_iommu.iommu_mode == RV_IOMMU_DDTP_MODE_BARE) { + return; + } + + /*** Check whether we need to scale to a greater mode ***/ + + // If the IOMMU is not in 3LVL mode and DDI[2] != 0, we need to scale to 3LVL + if (ddi_2 && (rv_iommu.iommu_mode < RV_IOMMU_DDTP_MODE_3LVL)) { + // Check if 3LVL is supported. Otherwise, raise error + if (rv_iommu.supported_modes & (1 << RV_IOMMU_DDTP_MODE_3LVL)) { + if (rv_iommu.iommu_mode == RV_IOMMU_DDTP_MODE_1LVL) { + // Scale from 1LVL to 2LVL + alloc_2lvl_vptrs(); + up_1lvl_to_2lvl(); + } + + // Scale from 2LVL to 3LVL + alloc_3lvl_vptrs(); + up_2lvl_to_3lvl(); + } + + else { + ERROR("RV_IOMMU: Unsupported device_id width"); + } + } + + // If the IOMMU is in 1LVL mode and DDI[1] != 0, we need to scale to 2LVL + if (ddi_1 && (rv_iommu.iommu_mode < RV_IOMMU_DDTP_MODE_2LVL)) { + // Check if 2LVL is supported. Otherwise, raise error + if (rv_iommu.supported_modes & (1 << RV_IOMMU_DDTP_MODE_2LVL)) { + // Scale from 1LVL to 2LVL + alloc_2lvl_vptrs(); + up_1lvl_to_2lvl(); + } + + else { + ERROR("RV_IOMMU: Unsupported device_id width"); + } + } + + /*** Write non-leaf entries if needed ***/ + + if (rv_iommu.iommu_mode >= RV_IOMMU_DDTP_MODE_2LVL) { + // Get root index according to the IOMMU mode + unsigned ddi_root = + (rv_iommu.iommu_mode == RV_IOMMU_DDTP_MODE_3LVL) ? (ddi_2) : (ddi_1); + + // Allocate first-level table if it does not exist + if (!rv_iommu.hw.vddt_lvl1[ddi_root]) { + // Allocate a page of memory to use as the first-level table + vaddr_t lvl1_vaddr = (vaddr_t)mem_alloc_page(1, SEC_HYP_GLOBAL, true); + memset((void*)lvl1_vaddr, 0, 0x1000); + // Save base virtual address + rv_iommu.hw.vddt_lvl1[ddi_root] = (uint64_t*)lvl1_vaddr; + + // Get physical address of the lvl1 table + paddr_t lvl1_paddr; + mem_translate(&cpu()->as, lvl1_vaddr, &lvl1_paddr); + + // Program root table entry with base physical address of the lvl1 table + rv_iommu.hw.vddt_lvl0[ddi_root] = + ((lvl1_paddr >> 2) & RV_IOMMU_DDT_NL_PPN_MASK) | RV_IOMMU_DC_VALID_BIT; + } + + if (rv_iommu.iommu_mode == RV_IOMMU_DDTP_MODE_3LVL) { + // Allocate second-level table if it does not exist + if (!rv_iommu.hw.vddt_lvl2[ddi_2][ddi_1]) { + // Allocate a page of memory to use as the second-level table for for DDI[2] and + // DDI[1] + vaddr_t lvl2_vaddr = (vaddr_t)mem_alloc_page(1, SEC_HYP_GLOBAL, true); + memset((void*)lvl2_vaddr, 0, 0x1000); + // Save base virtual address + rv_iommu.hw.vddt_lvl2[ddi_2][ddi_1] = (uint64_t*)lvl2_vaddr; + + // Get physical address of the lvl2 table + paddr_t lvl2_paddr; + mem_translate(&cpu()->as, lvl2_vaddr, &lvl2_paddr); + + // Program root table entry with base physical address of the lvl1 table + rv_iommu.hw.vddt_lvl1[ddi_2][ddi_1] = + ((lvl2_paddr >> 2) & RV_IOMMU_DDT_NL_PPN_MASK) | RV_IOMMU_DC_VALID_BIT; + } + } + } + + /*** Write DC ***/ + + // Get DC pointer + volatile struct ddt_entry* dc_ptr = NULL; + switch (rv_iommu.iommu_mode) { + case RV_IOMMU_DDTP_MODE_3LVL: + dc_ptr = (volatile struct ddt_entry*)(rv_iommu.hw.vddt_lvl2[ddi_2][ddi_1]); + break; + case RV_IOMMU_DDTP_MODE_2LVL: + dc_ptr = (volatile struct ddt_entry*)(rv_iommu.hw.vddt_lvl1[ddi_1]); + break; + case RV_IOMMU_DDTP_MODE_1LVL: + dc_ptr = (volatile struct ddt_entry*)rv_iommu.hw.vddt_lvl0; + break; + + default: + break; + } + + // MSI translation supported. DC in extended format + if (rv_iommu.msi_support) { + // tc + uint64_t tc = 0; + tc |= RV_IOMMU_DC_VALID_BIT; + dc_ptr[ddi_0].dw_1 = tc; + + // iohgatp + uint64_t iohgatp = 0; + iohgatp |= ((root_pt >> 12) & RV_IOMMU_DC_IOHGATP_PPN_MASK); + iohgatp |= ((vm->id << RV_IOMMU_DC_IOHGATP_GSCID_OFF) & RV_IOMMU_DC_IOHGATP_GSCID_MASK); + iohgatp |= RV_IOMMU_IOHGATP_SV39X4; + dc_ptr[ddi_0].dw_2 = iohgatp; + + // Future work: ta, fsc, msiptp, msi_addr_mask, msi_addr_pattern + } + + // MSI translation not supported: DC in base format + else { + // Even DCs occupy the first 32 bytes of each entry + if (ddi_0 % 2 == 0) { + // tc + uint64_t tc = 0; + tc |= RV_IOMMU_DC_VALID_BIT; + dc_ptr[ddi_0 / 2].dw_1 = tc; + + // iohgatp + uint64_t iohgatp = 0; + iohgatp |= ((root_pt >> 12) & RV_IOMMU_DC_IOHGATP_PPN_MASK); + iohgatp |= + ((vm->id << RV_IOMMU_DC_IOHGATP_GSCID_OFF) & RV_IOMMU_DC_IOHGATP_GSCID_MASK); + iohgatp |= RV_IOMMU_IOHGATP_SV39X4; + dc_ptr[ddi_0 / 2].dw_2 = iohgatp; + + // Future work: ta, fsc + } + + // Odd DCs occupy the second 32 bytes of each entry + else { + // tc + uint64_t tc = 0; + tc |= RV_IOMMU_DC_VALID_BIT; + dc_ptr[ddi_0 / 2].dw_5 = tc; + + // iohgatp + uint64_t iohgatp = 0; + iohgatp |= ((root_pt >> 12) & RV_IOMMU_DC_IOHGATP_PPN_MASK); + iohgatp |= + ((vm->id << RV_IOMMU_DC_IOHGATP_GSCID_OFF) & RV_IOMMU_DC_IOHGATP_GSCID_MASK); + iohgatp |= RV_IOMMU_IOHGATP_SV39X4; + dc_ptr[ddi_0 / 2].dw_6 = iohgatp; + + // Future work: ta, fsc + } + } + } + spin_unlock(&rv_iommu.ddt_lock); }