Add shstk access type for special permission of shadow stack page

riscv/insns/c_sspopchk_x5.h

@@ -22,9 +22,9 @@ if (!xSSE) {
   uint64_t ss_addr;
 
   if (xlen == 32)
-    ss_addr = MMU.load<uint32_t>(ssp);
+    ss_addr = MMU.ss_load<uint32_t>(ssp);
   else
-    ss_addr = MMU.load<uint64_t>(ssp);
+    ss_addr = MMU.ss_load<uint64_t>(ssp);
 
   software_check(T0 == ss_addr, SHADOW_STACK_FAULT);
 

riscv/insns/c_sspush_x1.h

@@ -21,9 +21,9 @@ if (!xSSE) {
   const auto new_sp = ssp - xlen / 8;
 
   if (xlen == 32)
-    MMU.store<uint32_t>(new_sp, RA);
+    MMU.ss_store<uint32_t>(new_sp, RA);
   else
-    MMU.store<uint64_t>(new_sp, RA);
+    MMU.ss_store<uint64_t>(new_sp, RA);
 
   STATE.ssp->write(new_sp);
 }

riscv/insns/ssamoswap_d.h

@@ -15,4 +15,4 @@ else if (prv == PRV_S && STATE.v && !get_field(STATE.henvcfg->read(), HENVCFG_SS
 else if (prv == PRV_U && STATE.v && !get_field(STATE.senvcfg->read(), SENVCFG_SSE))
   throw trap_virtual_instruction(insn.bits());
 else
-  WRITE_RD(MMU.amo<uint64_t>(RS1, [&](uint64_t UNUSED lhs) { return RS2; }));
+  WRITE_RD(MMU.ss_amo<uint64_t>(RS1, [&](uint64_t UNUSED lhs) { return RS2; }));

riscv/insns/sspopchk_x1.h

@@ -24,9 +24,9 @@ if (!xSSE) {
   uint64_t ss_addr;
 
   if (xlen == 32)
-    ss_addr = MMU.load<uint32_t>(ssp);
+    ss_addr = MMU.ss_load<uint32_t>(ssp);
   else
-    ss_addr = MMU.load<uint64_t>(ssp);
+    ss_addr = MMU.ss_load<uint64_t>(ssp);
 
   software_check(RA == ss_addr, SHADOW_STACK_FAULT);
 

riscv/insns/sspopchk_x5.h

@@ -24,9 +24,9 @@ if (!xSSE) {
   uint64_t ss_addr;
 
   if (xlen == 32)
-    ss_addr = MMU.load<uint32_t>(ssp);
+    ss_addr = MMU.ss_load<uint32_t>(ssp);
   else
-    ss_addr = MMU.load<uint64_t>(ssp);
+    ss_addr = MMU.ss_load<uint64_t>(ssp);
 
   software_check(T0 == ss_addr, SHADOW_STACK_FAULT);
 

riscv/insns/sspush_x1.h

@@ -23,9 +23,9 @@ if (!xSSE) {
   const auto new_sp = ssp - xlen / 8;
 
   if (xlen == 32)
-    MMU.store<uint32_t>(new_sp, RA);
+    MMU.ss_store<uint32_t>(new_sp, RA);
   else
-    MMU.store<uint64_t>(new_sp, RA);
+    MMU.ss_store<uint64_t>(new_sp, RA);
 
   STATE.ssp->write(new_sp);
 }

riscv/insns/sspush_x5.h

@@ -23,9 +23,9 @@ if (!xSSE) {
   const auto new_sp = ssp - xlen / 8;
 
   if (xlen == 32)
-    MMU.store<uint32_t>(new_sp, T0);
+    MMU.ss_store<uint32_t>(new_sp, T0);
   else
-    MMU.store<uint64_t>(new_sp, T0);
+    MMU.ss_store<uint64_t>(new_sp, T0);
 
   STATE.ssp->write(new_sp);
 }

riscv/mmu.cc

@@ -491,8 +491,15 @@ reg_t mmu_t::walk(mem_access_info_t access_info)
   reg_t page_mask = (reg_t(1) << PGSHIFT) - 1;
   reg_t satp = proc->get_state()->satp->readvirt(virt);
   vm_info vm = decode_vm_info(proc->get_const_xlen(), false, mode, satp);
-  if (vm.levels == 0)
+
+  // use access flags(shstk) and zicfiss extension as conditions for checking shadow stack
+  bool check_ss = proc->extension_enabled(EXT_ZICFISS) && access_info.flags.shstk;
+
+  if (vm.levels == 0) {
+    if (check_ss)
+      trap_store_access_fault(virt, addr, 0, 0);
     return s2xlate(addr, addr & ((reg_t(2) << (proc->xlen-1))-1), type, type, virt, hlvx, false) & ~page_mask; // zero-extend from xlen
+  }
 
   bool s_mode = mode == PRV_S;
   bool sum = proc->state.sstatus->readvirt(virt) & MSTATUS_SUM;
@@ -506,6 +513,9 @@ reg_t mmu_t::walk(mem_access_info_t access_info)
     vm.levels = 0;
 
   reg_t base = vm.ptbase;
+  reg_t vw = PTE_V | PTE_W;
+  reg_t invalid_pte;
+
   for (int i = vm.levels - 1; i >= 0; i--) {
     int ptshift = i * vm.idxbits;
     reg_t idx = (addr >> (PGSHIFT + ptshift)) & ((1 << vm.idxbits) - 1);
@@ -517,6 +527,8 @@ reg_t mmu_t::walk(mem_access_info_t access_info)
     bool pbmte = virt ? (proc->get_state()->henvcfg->read() & HENVCFG_PBMTE) : (proc->get_state()->menvcfg->read() & MENVCFG_PBMTE);
     bool hade = virt ? (proc->get_state()->henvcfg->read() & HENVCFG_ADUE) : (proc->get_state()->menvcfg->read() & MENVCFG_ADUE);
 
+    invalid_pte = pte;
+
     if (pte & PTE_RSVD) {
       break;
     } else if (!proc->extension_enabled(EXT_SVNAPOT) && (pte & PTE_N)) {
@@ -531,11 +543,23 @@ reg_t mmu_t::walk(mem_access_info_t access_info)
       base = ppn << PGSHIFT;
     } else if ((pte & PTE_U) ? s_mode && (type == FETCH || !sum) : !s_mode) {
       break;
-    } else if (!(pte & PTE_V) || (!(pte & PTE_R) && (pte & PTE_W))) {
+    } else if (!(pte & PTE_V)) {
+      break;
+    } else if (!(pte & PTE_R) && (pte & PTE_W) && !check_ss) {
+      // allow shadow stack page with R=0, W=1, X=0 permission
+      break;
+    } else if ((type == FETCH || hlvx) && !(pte & PTE_X)) {
       break;
-    } else if (type == FETCH || hlvx ? !(pte & PTE_X) :
-               type == LOAD          ? !(pte & PTE_R) && !(mxr && (pte & PTE_X)) :
-                                       !((pte & PTE_R) && (pte & PTE_W))) {
+    } else if ((type == LOAD) && !(pte & PTE_R) && !(mxr && (pte & PTE_X)) &&
+               !check_ss) {
+      // allow ss load page with R=0, W=1, X=0 page
+      break;
+    } else if ((type == STORE) && !((pte & PTE_R) && (pte & PTE_W)) &&
+               !check_ss) {
+      // allow ss store page with R=0, W=1, X=0
+      break;
+    } else if (check_ss && ((pte & vw) != vw)) {
+      // ss load/store page with R=0, W=1, X=0 and valid
       break;
     } else if ((ppn & ((reg_t(1) << ptshift) - 1)) != 0) {
       break;
@@ -570,10 +594,27 @@ reg_t mmu_t::walk(mem_access_info_t access_info)
     }
   }
 
+  if (check_ss) {
+    if (!(invalid_pte & PTE_V))
+      throw trap_store_page_fault(virt, addr, 0, 0);
+    else if (invalid_pte & PTE_W || invalid_pte & PTE_X)
+      throw trap_store_access_fault(virt, addr, 0, 0);
+    else
+      throw trap_store_page_fault(virt, addr, 0, 0);
+  }
+
   switch (type) {
-    case FETCH: throw trap_instruction_page_fault(virt, addr, 0, 0);
+    case FETCH: {
+      if (proc->extension_enabled(EXT_ZICFISS) && ((invalid_pte & vw) == vw))
+        throw trap_instruction_access_fault(virt, addr, 0, 0);
+      throw trap_instruction_page_fault(virt, addr, 0, 0);
+    }
     case LOAD: throw trap_load_page_fault(virt, addr, 0, 0);
-    case STORE: throw trap_store_page_fault(virt, addr, 0, 0);
+    case STORE: {
+      if (proc->extension_enabled(EXT_ZICFISS) && ((invalid_pte & vw) == vw))
+        throw trap_store_access_fault(virt, addr, 0, 0);
+      throw trap_store_page_fault(virt, addr, 0, 0);
+    }
     default: abort();
   }
 }

riscv/mmu.h

@@ -42,9 +42,10 @@ struct xlate_flags_t {
   const bool forced_virt : 1;
   const bool hlvx : 1;
   const bool lr : 1;
+  const bool shstk : 1; //shadow stack page permission is R=0, W=1, X=0
 
   bool is_special_access() const {
-    return forced_virt || hlvx || lr;
+    return forced_virt || hlvx || lr || shstk;
   }
 };
 
@@ -136,6 +137,11 @@ class mmu_t
     return load<T>(addr, {forced_virt, hlvx, lr});
   }
 
+  template<typename T>
+  T ss_load(reg_t addr) {
+    return load<T>(addr, {false, false, false, /*shstk=*/ true});
+  }
+
   template<typename T>
   void ALWAYS_INLINE store(reg_t addr, T val, xlate_flags_t xlate_flags = {false, false, false}) {
     reg_t vpn = addr >> PGSHIFT;
@@ -161,6 +167,12 @@ class mmu_t
     store(addr, val, {forced_virt, hlvx, lr});
   }
 
+  // shadow stack store
+  template<typename T>
+  void ss_store(reg_t addr, T val) {
+    store<T>(addr, val, {false, false, false, /*shstk=*/ true});
+  }
+
   // AMO/Zicbom faults should be reported as store faults
   #define convert_load_traps_to_store_traps(BODY) \
     try { \
@@ -187,6 +199,17 @@ class mmu_t
     })
   }
 
+  // for shadow stack amo swap
+  template<typename T, typename op>
+  T ss_amo(reg_t addr, op f) {
+    convert_load_traps_to_store_traps({
+      store_slow_path(addr, sizeof(T), nullptr, {false, false, false, /*shstk*/true}, false, true);
+      auto lhs = ss_load<T>(addr);
+      ss_store<T>(addr, f(lhs));
+      return lhs;
+    })
+  }
+
   template<typename T>
   T amo_compare_and_swap(reg_t addr, T comp, T swap) {
     convert_load_traps_to_store_traps({