From 41076b9353de70311a829621f4061101ec541917 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 2 Nov 2024 08:00:58 +0000 Subject: [PATCH 1/8] move userspace entering code into global_asm! block Regular asm! blocks are tricky because there's no guarantee that the compiler will emit them exactly once into the binary. This guarantee doesn't technically exist for global_asm! either, but the compiler is much more restricted in the optimizations (e.g. inlining) it can perform. --- .../src/user/process/syscall/cpu_state.rs | 511 +++++++++--------- 1 file changed, 258 insertions(+), 253 deletions(-) diff --git a/tee/kernel/src/user/process/syscall/cpu_state.rs b/tee/kernel/src/user/process/syscall/cpu_state.rs index c2a2803d..af8353fa 100644 --- a/tee/kernel/src/user/process/syscall/cpu_state.rs +++ b/tee/kernel/src/user/process/syscall/cpu_state.rs @@ -1,6 +1,6 @@ use core::{ arch::{ - asm, + global_asm, x86_64::{__cpuid_count, _xrstor64, _xsaveopt64}, }, ffi::c_void, @@ -79,17 +79,6 @@ impl CpuState { } pub fn run_user(&mut self, virtual_memory: &VirtualMemory) -> Result { - macro_rules! kernel_reg_offset { - ($ident:ident) => {{ - offset_of!(PerCpu, kernel_registers) + offset_of!(KernelRegisters, $ident) - }}; - } - macro_rules! userspace_reg_offset { - ($ident:ident) => {{ - offset_of!(PerCpu, new_userspace_registers) + offset_of!(Registers, $ident) - }}; - } - let per_cpu = PerCpu::get(); per_cpu.new_userspace_registers.set(self.registers); @@ -110,247 +99,7 @@ impl CpuState { per_cpu.exit_with_sysret.set(self.last_exit_was_syscall); - virtual_memory.run_with(|| { - unsafe { - asm!( - // Save kernel state. - // Save the kernel registers. - "mov gs:[{K_RAX_OFFSET}], rax", - "mov gs:[{K_RBX_OFFSET}], rbx", - "mov gs:[{K_RCX_OFFSET}], rcx", - "mov gs:[{K_RDX_OFFSET}], rdx", - "mov gs:[{K_RSI_OFFSET}], rsi", - "mov gs:[{K_RDI_OFFSET}], rdi", - "mov gs:[{K_RSP_OFFSET}], rsp", - "mov gs:[{K_RBP_OFFSET}], rbp", - "mov gs:[{K_R8_OFFSET}], r8", - "mov gs:[{K_R9_OFFSET}], r9", - "mov gs:[{K_R10_OFFSET}], r10", - "mov gs:[{K_R11_OFFSET}], r11", - "mov gs:[{K_R12_OFFSET}], r12", - "mov gs:[{K_R13_OFFSET}], r13", - "mov gs:[{K_R14_OFFSET}], r14", - "mov gs:[{K_R15_OFFSET}], r15", - // Save RFLAGS. - "pushfq", - "pop rax", - "mov gs:[{K_RFLAGS_OFFSET}], rax", - - // Prepare exit points. - // Set exception/interrupt handler exit point. - "lea rax, [rip+66f]", - "mov gs:[{EXCEPTION_HANDLER_EXIT_POINT_OFFSET}], rax", - // Set syscall instruction exit point. - "lea rax, [rip+67f]", - "mov rdx, rax", - "shr rdx, 32", - "mov ecx, 0xC0000082", - "wrmsr", - - // Restore user state. - // Restore segment registers. - "xor rax, rax", - "mov ax, gs:[{U_DS_OFFSET}]", - "mov ds, ax", - "mov ax, gs:[{U_ES_OFFSET}]", - "mov es, ax", - "mov ax, gs:[{U_FS_OFFSET}]", - "mov fs, ax", - "mov ax, gs:[{U_GS_OFFSET}]", - "swapgs", - "mov gs, ax", - "swapgs", - // Restore FS base. - "mov rax, gs:[{U_FS_BASE_OFFSET}]", - "wrfsbase rax", - // Restore userspace registers. - "mov rax, gs:[{U_RAX_OFFSET}]", - "mov rbx, gs:[{U_RBX_OFFSET}]", - "mov rcx, gs:[{U_RCX_OFFSET}]", - "mov rdx, gs:[{U_RDX_OFFSET}]", - "mov rsi, gs:[{U_RSI_OFFSET}]", - "mov rdi, gs:[{U_RDI_OFFSET}]", - "mov rbp, gs:[{U_RBP_OFFSET}]", - "mov r8, gs:[{U_R8_OFFSET}]", - "mov r9, gs:[{U_R9_OFFSET}]", - "mov r10, gs:[{U_R10_OFFSET}]", - "mov r11, gs:[{U_R11_OFFSET}]", - "mov r12, gs:[{U_R12_OFFSET}]", - "mov r13, gs:[{U_R13_OFFSET}]", - "mov r14, gs:[{U_R14_OFFSET}]", - "mov r15, gs:[{U_R15_OFFSET}]", - - // Check if we should use the `sysretq` instruction to enter - // userspace. - "cmp byte ptr gs:[{EXIT_WITH_SYSRET_OFFSET}], 0", - "je 65f", - - // Enter userspace using sysretq. - "mov rcx, gs:[{U_RIP_OFFSET}]", - "mov r11, gs:[{U_RFLAGS_OFFSET}]", - "mov rsp, gs:[{U_RSP_OFFSET}]", - // Swap in userspace GS. - "swapgs", - // Enter usermode. - "sysretq", - - // Enter userspace using iretq. - "65:", - // Setup stack frame. - // SS - "mov qword ptr [rsp - 8], 0", - "sub rsp, 6", - "push word ptr gs:[{U_SS_OFFSET}]", - // RSP - "push gs:[{U_RSP_OFFSET}]", - // RFLAGS - "push gs:[{U_RFLAGS_OFFSET}]", - // CS - "mov qword ptr [rsp - 8], 0", - "sub rsp, 6", - "push word ptr gs:[{U_CS_OFFSET}]", - // RIP - "push gs:[{U_RIP_OFFSET}]", - // Swap in userspace GS. - "swapgs", - // Enter usermode. - "iretq", - - // Exit point for an exception/interrupt. - // Note that `swapgs` was already executed by the exception/interrupt handler. - "66:", - // Record the exit reason. - "mov byte ptr gs:[{EXIT_OFFSET}], {EXIT_EXCP}", - // Save values from stack frame. - "mov gs:[{U_RAX_OFFSET}], rax", - "pop qword ptr gs:[{U_RIP_OFFSET}]", // pop RIP - "pop rax", // pop CS, - "mov gs:[{U_CS_OFFSET}], ax", - "pop qword ptr gs:[{U_RFLAGS_OFFSET}]", // pop RFLAGS - "pop qword ptr gs:[{U_RSP_OFFSET}]", // pop RSP - "pop rax", // pop SS - "mov gs:[{U_SS_OFFSET}], ax", - // Jump to the common save state code. - "jmp 68f", - - // Exit point for the `syscall` instruction - "67:", - // Swap in kernel GS. - "swapgs", - // Record the exit reason. - "mov byte ptr gs:[{EXIT_OFFSET}], {EXIT_SYSCALL}", - // Save userspace registers. - "mov gs:[{U_RAX_OFFSET}], rax", - "mov gs:[{U_RSP_OFFSET}], rsp", - "mov gs:[{U_RIP_OFFSET}], rcx", - "mov gs:[{U_RFLAGS_OFFSET}], r11", - // Fall through to 68f - - // Common user save state code. - "68:", - // Save segment registers. - "mov ax, ds", - "mov gs:[{U_DS_OFFSET}], ax", - "mov ax, es", - "mov gs:[{U_ES_OFFSET}], ax", - "mov ax, fs", - "mov gs:[{U_FS_OFFSET}], ax", - "mov ax, gs", - "mov gs:[{U_GS_OFFSET}], ax", - // Save FS base. - "rdfsbase rax", - "mov gs:[{U_FS_BASE_OFFSET}], rax", - // Save registers. - "mov gs:[{U_RBX_OFFSET}], rbx", - "mov gs:[{U_RCX_OFFSET}], rcx", - "mov gs:[{U_RDX_OFFSET}], rdx", - "mov gs:[{U_RSI_OFFSET}], rsi", - "mov gs:[{U_RDI_OFFSET}], rdi", - "mov gs:[{U_RBP_OFFSET}], rbp", - "mov gs:[{U_R8_OFFSET}], r8", - "mov gs:[{U_R9_OFFSET}], r9", - "mov gs:[{U_R10_OFFSET}], r10", - "mov gs:[{U_R11_OFFSET}], r11", - "mov gs:[{U_R12_OFFSET}], r12", - "mov gs:[{U_R13_OFFSET}], r13", - "mov gs:[{U_R14_OFFSET}], r14", - "mov gs:[{U_R15_OFFSET}], r15", - - // Restore kernel state. - // Restore the kernel registers. - "mov rbx, gs:[{K_RBX_OFFSET}]", - "mov rcx, gs:[{K_RCX_OFFSET}]", - "mov rdx, gs:[{K_RDX_OFFSET}]", - "mov rsi, gs:[{K_RSI_OFFSET}]", - "mov rdi, gs:[{K_RDI_OFFSET}]", - "mov rsp, gs:[{K_RSP_OFFSET}]", - "mov rbp, gs:[{K_RBP_OFFSET}]", - "mov r8, gs:[{K_R8_OFFSET}]", - "mov r9, gs:[{K_R9_OFFSET}]", - "mov r10, gs:[{K_R10_OFFSET}]", - "mov r11, gs:[{K_R11_OFFSET}]", - "mov r12, gs:[{K_R12_OFFSET}]", - "mov r13, gs:[{K_R13_OFFSET}]", - "mov r14, gs:[{K_R14_OFFSET}]", - "mov r15, gs:[{K_R15_OFFSET}]", - // Restore RFLAGS. - "mov rax, gs:[{K_RFLAGS_OFFSET}]", - "push rax", - "popfq", - // Restore rax - "mov rax, gs:[{K_RAX_OFFSET}]", - - EXIT_WITH_SYSRET_OFFSET = const offset_of!(PerCpu, exit_with_sysret), - EXCEPTION_HANDLER_EXIT_POINT_OFFSET = const offset_of!(PerCpu, userspace_exception_exit_point), - EXIT_OFFSET = const offset_of!(PerCpu, exit), - EXIT_SYSCALL = const RawExit::Syscall as u8, - EXIT_EXCP = const RawExit::Exception as u8, - K_RAX_OFFSET = const kernel_reg_offset!(rax), - K_RBX_OFFSET = const kernel_reg_offset!(rbx), - K_RCX_OFFSET = const kernel_reg_offset!(rcx), - K_RDX_OFFSET = const kernel_reg_offset!(rdx), - K_RSI_OFFSET = const kernel_reg_offset!(rsi), - K_RDI_OFFSET = const kernel_reg_offset!(rdi), - K_RSP_OFFSET = const kernel_reg_offset!(rsp), - K_RBP_OFFSET = const kernel_reg_offset!(rbp), - K_R8_OFFSET = const kernel_reg_offset!(r8), - K_R9_OFFSET = const kernel_reg_offset!(r9), - K_R10_OFFSET = const kernel_reg_offset!(r10), - K_R11_OFFSET = const kernel_reg_offset!(r11), - K_R12_OFFSET = const kernel_reg_offset!(r12), - K_R13_OFFSET = const kernel_reg_offset!(r13), - K_R14_OFFSET = const kernel_reg_offset!(r14), - K_R15_OFFSET = const kernel_reg_offset!(r15), - K_RFLAGS_OFFSET = const kernel_reg_offset!(rflags), - U_RAX_OFFSET = const userspace_reg_offset!(rax), - U_RBX_OFFSET = const userspace_reg_offset!(rbx), - U_RCX_OFFSET = const userspace_reg_offset!(rcx), - U_RDX_OFFSET = const userspace_reg_offset!(rdx), - U_RSI_OFFSET = const userspace_reg_offset!(rsi), - U_RDI_OFFSET = const userspace_reg_offset!(rdi), - U_RSP_OFFSET = const userspace_reg_offset!(rsp), - U_RBP_OFFSET = const userspace_reg_offset!(rbp), - U_R8_OFFSET = const userspace_reg_offset!(r8), - U_R9_OFFSET = const userspace_reg_offset!(r9), - U_R10_OFFSET = const userspace_reg_offset!(r10), - U_R11_OFFSET = const userspace_reg_offset!(r11), - U_R12_OFFSET = const userspace_reg_offset!(r12), - U_R13_OFFSET = const userspace_reg_offset!(r13), - U_R14_OFFSET = const userspace_reg_offset!(r14), - U_R15_OFFSET = const userspace_reg_offset!(r15), - U_RIP_OFFSET = const userspace_reg_offset!(rip), - U_RFLAGS_OFFSET = const userspace_reg_offset!(rflags), - U_CS_OFFSET = const userspace_reg_offset!(cs), - U_DS_OFFSET = const userspace_reg_offset!(ds), - U_ES_OFFSET = const userspace_reg_offset!(es), - U_FS_OFFSET = const userspace_reg_offset!(fs), - U_FS_BASE_OFFSET = const userspace_reg_offset!(fs_base), - U_GS_OFFSET = const userspace_reg_offset!(gs), - U_SS_OFFSET = const userspace_reg_offset!(ss), - options(preserves_flags), - ); - } - }); + virtual_memory.run_with(|| unsafe { enter_userspace() }); // Save x87, SSE and AVX state. self.xsave_area.save(); @@ -1022,3 +771,259 @@ struct FpxSwBytes { xstate_size: u32, padding: [u32; 7], } + +unsafe extern "C" { + fn enter_userspace(); +} + +macro_rules! kernel_reg_offset { + ($ident:ident) => {{ + offset_of!(PerCpu, kernel_registers) + offset_of!(KernelRegisters, $ident) + }}; +} +macro_rules! userspace_reg_offset { + ($ident:ident) => {{ + offset_of!(PerCpu, new_userspace_registers) + offset_of!(Registers, $ident) + }}; +} + +global_asm!( + ".global enter_userspace", + "enter_userspace:", + + // Save kernel state. + // Save the kernel registers. + "mov gs:[{K_RAX_OFFSET}], rax", + "mov gs:[{K_RBX_OFFSET}], rbx", + "mov gs:[{K_RCX_OFFSET}], rcx", + "mov gs:[{K_RDX_OFFSET}], rdx", + "mov gs:[{K_RSI_OFFSET}], rsi", + "mov gs:[{K_RDI_OFFSET}], rdi", + "mov gs:[{K_RSP_OFFSET}], rsp", + "mov gs:[{K_RBP_OFFSET}], rbp", + "mov gs:[{K_R8_OFFSET}], r8", + "mov gs:[{K_R9_OFFSET}], r9", + "mov gs:[{K_R10_OFFSET}], r10", + "mov gs:[{K_R11_OFFSET}], r11", + "mov gs:[{K_R12_OFFSET}], r12", + "mov gs:[{K_R13_OFFSET}], r13", + "mov gs:[{K_R14_OFFSET}], r14", + "mov gs:[{K_R15_OFFSET}], r15", + // Save RFLAGS. + "pushfq", + "pop rax", + "mov gs:[{K_RFLAGS_OFFSET}], rax", + + // Prepare exit points. + // Set exception/interrupt handler exit point. + "lea rax, [rip+66f]", + "mov gs:[{EXCEPTION_HANDLER_EXIT_POINT_OFFSET}], rax", + // Set syscall instruction exit point. + "lea rax, [rip+67f]", + "mov rdx, rax", + "shr rdx, 32", + "mov ecx, 0xC0000082", + "wrmsr", + + // Restore user state. + // Restore segment registers. + "xor rax, rax", + "mov ax, gs:[{U_DS_OFFSET}]", + "mov ds, ax", + "mov ax, gs:[{U_ES_OFFSET}]", + "mov es, ax", + "mov ax, gs:[{U_FS_OFFSET}]", + "mov fs, ax", + "mov ax, gs:[{U_GS_OFFSET}]", + "swapgs", + "mov gs, ax", + "swapgs", + // Restore FS base. + "mov rax, gs:[{U_FS_BASE_OFFSET}]", + "wrfsbase rax", + // Restore userspace registers. + "mov rax, gs:[{U_RAX_OFFSET}]", + "mov rbx, gs:[{U_RBX_OFFSET}]", + "mov rcx, gs:[{U_RCX_OFFSET}]", + "mov rdx, gs:[{U_RDX_OFFSET}]", + "mov rsi, gs:[{U_RSI_OFFSET}]", + "mov rdi, gs:[{U_RDI_OFFSET}]", + "mov rbp, gs:[{U_RBP_OFFSET}]", + "mov r8, gs:[{U_R8_OFFSET}]", + "mov r9, gs:[{U_R9_OFFSET}]", + "mov r10, gs:[{U_R10_OFFSET}]", + "mov r11, gs:[{U_R11_OFFSET}]", + "mov r12, gs:[{U_R12_OFFSET}]", + "mov r13, gs:[{U_R13_OFFSET}]", + "mov r14, gs:[{U_R14_OFFSET}]", + "mov r15, gs:[{U_R15_OFFSET}]", + + // Check if we should use the `sysretq` instruction to enter + // userspace. + "cmp byte ptr gs:[{EXIT_WITH_SYSRET_OFFSET}], 0", + "je 65f", + + // Enter userspace using sysretq. + "mov rcx, gs:[{U_RIP_OFFSET}]", + "mov r11, gs:[{U_RFLAGS_OFFSET}]", + "mov rsp, gs:[{U_RSP_OFFSET}]", + // Swap in userspace GS. + "swapgs", + // Enter usermode. + "sysretq", + + // Enter userspace using iretq. + "65:", + // Setup stack frame. + // SS + "mov qword ptr [rsp - 8], 0", + "sub rsp, 6", + "push word ptr gs:[{U_SS_OFFSET}]", + // RSP + "push gs:[{U_RSP_OFFSET}]", + // RFLAGS + "push gs:[{U_RFLAGS_OFFSET}]", + // CS + "mov qword ptr [rsp - 8], 0", + "sub rsp, 6", + "push word ptr gs:[{U_CS_OFFSET}]", + // RIP + "push gs:[{U_RIP_OFFSET}]", + // Swap in userspace GS. + "swapgs", + // Enter usermode. + "iretq", + + // Exit point for an exception/interrupt. + // Note that `swapgs` was already executed by the exception/interrupt handler. + "66:", + // Record the exit reason. + "mov byte ptr gs:[{EXIT_OFFSET}], {EXIT_EXCP}", + // Save values from stack frame. + "mov gs:[{U_RAX_OFFSET}], rax", + "pop qword ptr gs:[{U_RIP_OFFSET}]", // pop RIP + "pop rax", // pop CS, + "mov gs:[{U_CS_OFFSET}], ax", + "pop qword ptr gs:[{U_RFLAGS_OFFSET}]", // pop RFLAGS + "pop qword ptr gs:[{U_RSP_OFFSET}]", // pop RSP + "pop rax", // pop SS + "mov gs:[{U_SS_OFFSET}], ax", + // Jump to the common save state code. + "jmp 68f", + + // Exit point for the `syscall` instruction + "67:", + // Swap in kernel GS. + "swapgs", + // Record the exit reason. + "mov byte ptr gs:[{EXIT_OFFSET}], {EXIT_SYSCALL}", + // Save userspace registers. + "mov gs:[{U_RAX_OFFSET}], rax", + "mov gs:[{U_RSP_OFFSET}], rsp", + "mov gs:[{U_RIP_OFFSET}], rcx", + "mov gs:[{U_RFLAGS_OFFSET}], r11", + // Fall through to 68f + + // Common user save state code. + "68:", + // Save segment registers. + "mov ax, ds", + "mov gs:[{U_DS_OFFSET}], ax", + "mov ax, es", + "mov gs:[{U_ES_OFFSET}], ax", + "mov ax, fs", + "mov gs:[{U_FS_OFFSET}], ax", + "mov ax, gs", + "mov gs:[{U_GS_OFFSET}], ax", + // Save FS base. + "rdfsbase rax", + "mov gs:[{U_FS_BASE_OFFSET}], rax", + // Save registers. + "mov gs:[{U_RBX_OFFSET}], rbx", + "mov gs:[{U_RCX_OFFSET}], rcx", + "mov gs:[{U_RDX_OFFSET}], rdx", + "mov gs:[{U_RSI_OFFSET}], rsi", + "mov gs:[{U_RDI_OFFSET}], rdi", + "mov gs:[{U_RBP_OFFSET}], rbp", + "mov gs:[{U_R8_OFFSET}], r8", + "mov gs:[{U_R9_OFFSET}], r9", + "mov gs:[{U_R10_OFFSET}], r10", + "mov gs:[{U_R11_OFFSET}], r11", + "mov gs:[{U_R12_OFFSET}], r12", + "mov gs:[{U_R13_OFFSET}], r13", + "mov gs:[{U_R14_OFFSET}], r14", + "mov gs:[{U_R15_OFFSET}], r15", + + // Restore kernel state. + // Restore the kernel registers. + "mov rbx, gs:[{K_RBX_OFFSET}]", + "mov rcx, gs:[{K_RCX_OFFSET}]", + "mov rdx, gs:[{K_RDX_OFFSET}]", + "mov rsi, gs:[{K_RSI_OFFSET}]", + "mov rdi, gs:[{K_RDI_OFFSET}]", + "mov rsp, gs:[{K_RSP_OFFSET}]", + "mov rbp, gs:[{K_RBP_OFFSET}]", + "mov r8, gs:[{K_R8_OFFSET}]", + "mov r9, gs:[{K_R9_OFFSET}]", + "mov r10, gs:[{K_R10_OFFSET}]", + "mov r11, gs:[{K_R11_OFFSET}]", + "mov r12, gs:[{K_R12_OFFSET}]", + "mov r13, gs:[{K_R13_OFFSET}]", + "mov r14, gs:[{K_R14_OFFSET}]", + "mov r15, gs:[{K_R15_OFFSET}]", + // Restore RFLAGS. + "mov rax, gs:[{K_RFLAGS_OFFSET}]", + "push rax", + "popfq", + // Restore rax + "mov rax, gs:[{K_RAX_OFFSET}]", + "ret", + + EXIT_WITH_SYSRET_OFFSET = const offset_of!(PerCpu, exit_with_sysret), + EXCEPTION_HANDLER_EXIT_POINT_OFFSET = const offset_of!(PerCpu, userspace_exception_exit_point), + EXIT_OFFSET = const offset_of!(PerCpu, exit), + EXIT_SYSCALL = const RawExit::Syscall as u8, + EXIT_EXCP = const RawExit::Exception as u8, + K_RAX_OFFSET = const kernel_reg_offset!(rax), + K_RBX_OFFSET = const kernel_reg_offset!(rbx), + K_RCX_OFFSET = const kernel_reg_offset!(rcx), + K_RDX_OFFSET = const kernel_reg_offset!(rdx), + K_RSI_OFFSET = const kernel_reg_offset!(rsi), + K_RDI_OFFSET = const kernel_reg_offset!(rdi), + K_RSP_OFFSET = const kernel_reg_offset!(rsp), + K_RBP_OFFSET = const kernel_reg_offset!(rbp), + K_R8_OFFSET = const kernel_reg_offset!(r8), + K_R9_OFFSET = const kernel_reg_offset!(r9), + K_R10_OFFSET = const kernel_reg_offset!(r10), + K_R11_OFFSET = const kernel_reg_offset!(r11), + K_R12_OFFSET = const kernel_reg_offset!(r12), + K_R13_OFFSET = const kernel_reg_offset!(r13), + K_R14_OFFSET = const kernel_reg_offset!(r14), + K_R15_OFFSET = const kernel_reg_offset!(r15), + K_RFLAGS_OFFSET = const kernel_reg_offset!(rflags), + U_RAX_OFFSET = const userspace_reg_offset!(rax), + U_RBX_OFFSET = const userspace_reg_offset!(rbx), + U_RCX_OFFSET = const userspace_reg_offset!(rcx), + U_RDX_OFFSET = const userspace_reg_offset!(rdx), + U_RSI_OFFSET = const userspace_reg_offset!(rsi), + U_RDI_OFFSET = const userspace_reg_offset!(rdi), + U_RSP_OFFSET = const userspace_reg_offset!(rsp), + U_RBP_OFFSET = const userspace_reg_offset!(rbp), + U_R8_OFFSET = const userspace_reg_offset!(r8), + U_R9_OFFSET = const userspace_reg_offset!(r9), + U_R10_OFFSET = const userspace_reg_offset!(r10), + U_R11_OFFSET = const userspace_reg_offset!(r11), + U_R12_OFFSET = const userspace_reg_offset!(r12), + U_R13_OFFSET = const userspace_reg_offset!(r13), + U_R14_OFFSET = const userspace_reg_offset!(r14), + U_R15_OFFSET = const userspace_reg_offset!(r15), + U_RIP_OFFSET = const userspace_reg_offset!(rip), + U_RFLAGS_OFFSET = const userspace_reg_offset!(rflags), + U_CS_OFFSET = const userspace_reg_offset!(cs), + U_DS_OFFSET = const userspace_reg_offset!(ds), + U_ES_OFFSET = const userspace_reg_offset!(es), + U_FS_OFFSET = const userspace_reg_offset!(fs), + U_FS_BASE_OFFSET = const userspace_reg_offset!(fs_base), + U_GS_OFFSET = const userspace_reg_offset!(gs), + U_SS_OFFSET = const userspace_reg_offset!(ss), +); From d97c42a009f074fb10338d3cd7f2221e952f71b1 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 2 Nov 2024 08:10:21 +0000 Subject: [PATCH 2/8] save and restore less kernel registers There's no good reason to save and restore caller-saved registers. --- .../src/user/process/syscall/cpu_state.rs | 66 ++++--------------- 1 file changed, 11 insertions(+), 55 deletions(-) diff --git a/tee/kernel/src/user/process/syscall/cpu_state.rs b/tee/kernel/src/user/process/syscall/cpu_state.rs index af8353fa..a07a5aa6 100644 --- a/tee/kernel/src/user/process/syscall/cpu_state.rs +++ b/tee/kernel/src/user/process/syscall/cpu_state.rs @@ -615,41 +615,25 @@ impl Registers { }; } +/// The set of kernel registers saved when entering userspace. These correspond +/// to the callee-saved registers of the System V AMD64 ABI. #[derive(Clone, Copy)] pub struct KernelRegisters { - pub rax: u64, - pub rbx: u64, - pub rcx: u64, - pub rdx: u64, - pub rsi: u64, - pub rdi: u64, - pub rsp: u64, - pub rbp: u64, - pub r8: u64, - pub r9: u64, - pub r10: u64, - pub r11: u64, - pub r12: u64, - pub r13: u64, - pub r14: u64, - pub r15: u64, - pub rflags: u64, + rbx: u64, + rsp: u64, + rbp: u64, + r12: u64, + r13: u64, + r14: u64, + r15: u64, + rflags: u64, } impl KernelRegisters { pub const ZERO: Self = Self { - rax: 0, rbx: 0, - rcx: 0, - rdx: 0, - rsi: 0, - rdi: 0, rsp: 0, rbp: 0, - r8: 0, - r9: 0, - r10: 0, - r11: 0, r12: 0, r13: 0, r14: 0, @@ -772,7 +756,7 @@ struct FpxSwBytes { padding: [u32; 7], } -unsafe extern "C" { +unsafe extern "sysv64" { fn enter_userspace(); } @@ -793,18 +777,9 @@ global_asm!( // Save kernel state. // Save the kernel registers. - "mov gs:[{K_RAX_OFFSET}], rax", "mov gs:[{K_RBX_OFFSET}], rbx", - "mov gs:[{K_RCX_OFFSET}], rcx", - "mov gs:[{K_RDX_OFFSET}], rdx", - "mov gs:[{K_RSI_OFFSET}], rsi", - "mov gs:[{K_RDI_OFFSET}], rdi", "mov gs:[{K_RSP_OFFSET}], rsp", "mov gs:[{K_RBP_OFFSET}], rbp", - "mov gs:[{K_R8_OFFSET}], r8", - "mov gs:[{K_R9_OFFSET}], r9", - "mov gs:[{K_R10_OFFSET}], r10", - "mov gs:[{K_R11_OFFSET}], r11", "mov gs:[{K_R12_OFFSET}], r12", "mov gs:[{K_R13_OFFSET}], r13", "mov gs:[{K_R14_OFFSET}], r14", @@ -957,16 +932,8 @@ global_asm!( // Restore kernel state. // Restore the kernel registers. "mov rbx, gs:[{K_RBX_OFFSET}]", - "mov rcx, gs:[{K_RCX_OFFSET}]", - "mov rdx, gs:[{K_RDX_OFFSET}]", - "mov rsi, gs:[{K_RSI_OFFSET}]", - "mov rdi, gs:[{K_RDI_OFFSET}]", "mov rsp, gs:[{K_RSP_OFFSET}]", "mov rbp, gs:[{K_RBP_OFFSET}]", - "mov r8, gs:[{K_R8_OFFSET}]", - "mov r9, gs:[{K_R9_OFFSET}]", - "mov r10, gs:[{K_R10_OFFSET}]", - "mov r11, gs:[{K_R11_OFFSET}]", "mov r12, gs:[{K_R12_OFFSET}]", "mov r13, gs:[{K_R13_OFFSET}]", "mov r14, gs:[{K_R14_OFFSET}]", @@ -975,8 +942,6 @@ global_asm!( "mov rax, gs:[{K_RFLAGS_OFFSET}]", "push rax", "popfq", - // Restore rax - "mov rax, gs:[{K_RAX_OFFSET}]", "ret", EXIT_WITH_SYSRET_OFFSET = const offset_of!(PerCpu, exit_with_sysret), @@ -984,18 +949,9 @@ global_asm!( EXIT_OFFSET = const offset_of!(PerCpu, exit), EXIT_SYSCALL = const RawExit::Syscall as u8, EXIT_EXCP = const RawExit::Exception as u8, - K_RAX_OFFSET = const kernel_reg_offset!(rax), K_RBX_OFFSET = const kernel_reg_offset!(rbx), - K_RCX_OFFSET = const kernel_reg_offset!(rcx), - K_RDX_OFFSET = const kernel_reg_offset!(rdx), - K_RSI_OFFSET = const kernel_reg_offset!(rsi), - K_RDI_OFFSET = const kernel_reg_offset!(rdi), K_RSP_OFFSET = const kernel_reg_offset!(rsp), K_RBP_OFFSET = const kernel_reg_offset!(rbp), - K_R8_OFFSET = const kernel_reg_offset!(r8), - K_R9_OFFSET = const kernel_reg_offset!(r9), - K_R10_OFFSET = const kernel_reg_offset!(r10), - K_R11_OFFSET = const kernel_reg_offset!(r11), K_R12_OFFSET = const kernel_reg_offset!(r12), K_R13_OFFSET = const kernel_reg_offset!(r13), K_R14_OFFSET = const kernel_reg_offset!(r14), From c05471ba23a3b76efbb261a7bc4a231f75c10d96 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 2 Nov 2024 08:16:25 +0000 Subject: [PATCH 3/8] initialize LSTAR only once Previously we have to initialize it everytime because the compiler was allowed to instantiate the inline assembly more than once. This is technically still true, but doesn't matter much because there's no surrounding relevant context (e.g. code in a function the asm block was inlined into that could be present for normal inline assembling block). --- tee/kernel/src/user.rs | 4 ++++ tee/kernel/src/user/process/syscall.rs | 2 ++ tee/kernel/src/user/process/syscall/cpu_state.rs | 16 ++++++++-------- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tee/kernel/src/user.rs b/tee/kernel/src/user.rs index f2f86432..75221ade 100644 --- a/tee/kernel/src/user.rs +++ b/tee/kernel/src/user.rs @@ -1,8 +1,12 @@ +use process::syscall; + use crate::{memory::frame, rt::poll, supervisor::halt, time::advance_time}; pub mod process; pub fn run() -> ! { + syscall::init(); + loop { while poll() {} diff --git a/tee/kernel/src/user/process/syscall.rs b/tee/kernel/src/user/process/syscall.rs index 4391dc37..cabe3474 100644 --- a/tee/kernel/src/user/process/syscall.rs +++ b/tee/kernel/src/user/process/syscall.rs @@ -62,6 +62,8 @@ pub mod args; pub mod cpu_state; pub mod traits; +pub use cpu_state::init; + impl Thread { /// Returns true if the thread should continue to run. pub async fn execute_syscall(self: Arc, args: SyscallArgs) { diff --git a/tee/kernel/src/user/process/syscall/cpu_state.rs b/tee/kernel/src/user/process/syscall/cpu_state.rs index a07a5aa6..be52b8ff 100644 --- a/tee/kernel/src/user/process/syscall/cpu_state.rs +++ b/tee/kernel/src/user/process/syscall/cpu_state.rs @@ -14,7 +14,7 @@ use usize_conversions::{usize_from, FromUsize}; use x86_64::{ align_down, instructions::tables::{lgdt, sgdt}, - registers::{control::Cr2, xcontrol::XCr0Flags}, + registers::{control::Cr2, model_specific::LStar, xcontrol::XCr0Flags}, structures::{gdt::Entry, idt::PageFaultErrorCode, DescriptorTablePointer}, VirtAddr, }; @@ -756,8 +756,13 @@ struct FpxSwBytes { padding: [u32; 7], } +pub fn init() { + LStar::write(VirtAddr::new(syscall_entry as usize as u64)); +} + unsafe extern "sysv64" { fn enter_userspace(); + fn syscall_entry(); } macro_rules! kernel_reg_offset { @@ -793,12 +798,6 @@ global_asm!( // Set exception/interrupt handler exit point. "lea rax, [rip+66f]", "mov gs:[{EXCEPTION_HANDLER_EXIT_POINT_OFFSET}], rax", - // Set syscall instruction exit point. - "lea rax, [rip+67f]", - "mov rdx, rax", - "shr rdx, 32", - "mov ecx, 0xC0000082", - "wrmsr", // Restore user state. // Restore segment registers. @@ -887,7 +886,8 @@ global_asm!( "jmp 68f", // Exit point for the `syscall` instruction - "67:", + ".global syscall_entry", + "syscall_entry:", // Swap in kernel GS. "swapgs", // Record the exit reason. From 47965fe795cef2d259325d1f91f5883a7308898d Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 2 Nov 2024 08:24:33 +0000 Subject: [PATCH 4/8] don't user indirection for userspace exception exit points Much like with the syscall entrypoint, now that we use global_asm! we can rely on a single address for the label. --- tee/kernel/src/exception.rs | 21 ++++++++++--------- tee/kernel/src/per_cpu.rs | 2 -- .../src/user/process/syscall/cpu_state.rs | 10 +++------ 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/tee/kernel/src/exception.rs b/tee/kernel/src/exception.rs index bfae96ca..be939f42 100644 --- a/tee/kernel/src/exception.rs +++ b/tee/kernel/src/exception.rs @@ -8,6 +8,7 @@ use core::{ }; use crate::spin::lazy::Lazy; +use crate::user::process::syscall::cpu_state::exception_entry; use alloc::alloc::alloc; use log::{debug, error, trace}; use snp_types::intercept::VMEXIT_CPUID; @@ -179,11 +180,11 @@ extern "x86-interrupt" fn divide_error_handler(frame: InterruptStackFrame) { // Store the error code. "mov byte ptr gs:[{VECTOR_OFFSET}], 0x0", // Jump to the userspace exit point. - "jmp gs:[{HANDLER_OFFSET}]", + "jmp {exception_entry}", kernel_divide_error_handler = sym kernel_divide_error_handler, VECTOR_OFFSET = const offset_of!(PerCpu, vector), - HANDLER_OFFSET = const offset_of!(PerCpu, userspace_exception_exit_point), + exception_entry = sym exception_entry, ); } } @@ -209,7 +210,7 @@ extern "x86-interrupt" fn page_fault_handler( "mov byte ptr gs:[{VECTOR_OFFSET}], 0xe", "pop qword ptr gs:[{ERROR_CODE_OFFSET}]", // Jump to the userspace exit point. - "jmp gs:[{HANDLER_OFFSET}]", + "jmp {exception_entry}", // Kernel code path: "66:", @@ -258,7 +259,7 @@ extern "x86-interrupt" fn page_fault_handler( kernel_page_fault_handler = sym kernel_page_fault_handler, VECTOR_OFFSET = const offset_of!(PerCpu, vector), ERROR_CODE_OFFSET = const offset_of!(PerCpu, error_code), - HANDLER_OFFSET = const offset_of!(PerCpu, userspace_exception_exit_point), + exception_entry = sym exception_entry, ); } } @@ -313,12 +314,12 @@ extern "x86-interrupt" fn general_protection_fault_handler( "mov byte ptr gs:[{VECTOR_OFFSET}], 0xd", "pop qword ptr gs:[{ERROR_CODE_OFFSET}]", // Jump to the userspace exit point. - "jmp gs:[{HANDLER_OFFSET}]", + "jmp {exception_entry}", kernel_general_protection_fault_handler = sym kernel_general_protection_fault_handler, VECTOR_OFFSET = const offset_of!(PerCpu, vector), ERROR_CODE_OFFSET = const offset_of!(PerCpu, error_code), - HANDLER_OFFSET = const offset_of!(PerCpu, userspace_exception_exit_point), + exception_entry = sym exception_entry, ); } } @@ -348,12 +349,12 @@ extern "x86-interrupt" fn vc_handler(frame: InterruptStackFrame, error_code: u64 "mov byte ptr gs:[{VECTOR_OFFSET}], 0x1d", "pop qword ptr gs:[{ERROR_CODE_OFFSET}]", // Jump to the userspace exit point. - "jmp gs:[{HANDLER_OFFSET}]", + "jmp {exception_entry}", kernel_vc_handler = sym kernel_vc_handler, VECTOR_OFFSET = const offset_of!(PerCpu, vector), ERROR_CODE_OFFSET = const offset_of!(PerCpu, error_code), - HANDLER_OFFSET = const offset_of!(PerCpu, userspace_exception_exit_point), + exception_entry = sym exception_entry, ); } } @@ -466,9 +467,9 @@ extern "x86-interrupt" fn int0x80_handler(frame: InterruptStackFrame) { naked_asm!( "swapgs", "mov byte ptr gs:[{VECTOR_OFFSET}], 0x80", - "jmp gs:[{HANDLER_OFFSET}]", + "jmp {exception_entry}", VECTOR_OFFSET = const offset_of!(PerCpu, vector), - HANDLER_OFFSET = const offset_of!(PerCpu, userspace_exception_exit_point), + exception_entry = sym exception_entry, ); } } diff --git a/tee/kernel/src/per_cpu.rs b/tee/kernel/src/per_cpu.rs index 085533c7..459f5625 100644 --- a/tee/kernel/src/per_cpu.rs +++ b/tee/kernel/src/per_cpu.rs @@ -33,7 +33,6 @@ pub struct PerCpu { pub gdt: OnceCell, pub int0x80_handler: Cell, pub exit_with_sysret: Cell, - pub userspace_exception_exit_point: Cell, pub exit: Cell, pub vector: Cell, pub error_code: Cell, @@ -53,7 +52,6 @@ impl PerCpu { gdt: OnceCell::new(), int0x80_handler: Cell::new(0), exit_with_sysret: Cell::new(false), - userspace_exception_exit_point: Cell::new(0), exit: Cell::new(RawExit::Syscall), vector: Cell::new(0), error_code: Cell::new(0), diff --git a/tee/kernel/src/user/process/syscall/cpu_state.rs b/tee/kernel/src/user/process/syscall/cpu_state.rs index be52b8ff..72b4d741 100644 --- a/tee/kernel/src/user/process/syscall/cpu_state.rs +++ b/tee/kernel/src/user/process/syscall/cpu_state.rs @@ -762,6 +762,7 @@ pub fn init() { unsafe extern "sysv64" { fn enter_userspace(); + pub fn exception_entry(); fn syscall_entry(); } @@ -794,11 +795,6 @@ global_asm!( "pop rax", "mov gs:[{K_RFLAGS_OFFSET}], rax", - // Prepare exit points. - // Set exception/interrupt handler exit point. - "lea rax, [rip+66f]", - "mov gs:[{EXCEPTION_HANDLER_EXIT_POINT_OFFSET}], rax", - // Restore user state. // Restore segment registers. "xor rax, rax", @@ -870,7 +866,8 @@ global_asm!( // Exit point for an exception/interrupt. // Note that `swapgs` was already executed by the exception/interrupt handler. - "66:", + ".global exception_entry", + "exception_entry:", // Record the exit reason. "mov byte ptr gs:[{EXIT_OFFSET}], {EXIT_EXCP}", // Save values from stack frame. @@ -945,7 +942,6 @@ global_asm!( "ret", EXIT_WITH_SYSRET_OFFSET = const offset_of!(PerCpu, exit_with_sysret), - EXCEPTION_HANDLER_EXIT_POINT_OFFSET = const offset_of!(PerCpu, userspace_exception_exit_point), EXIT_OFFSET = const offset_of!(PerCpu, exit), EXIT_SYSCALL = const RawExit::Syscall as u8, EXIT_EXCP = const RawExit::Exception as u8, From cf76a2d6710864b20e6e760585c1fbb9137dfc62 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 2 Nov 2024 08:28:46 +0000 Subject: [PATCH 5/8] clear direction flag on syscall The Rust compiler assumes that the direction flag is unset. --- common/tdx-types/src/tdcall.rs | 3 +++ tee/kernel/src/user/process/syscall/cpu_state.rs | 8 +++++++- tee/supervisor-tdx/src/vcpu.rs | 1 + 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/common/tdx-types/src/tdcall.rs b/common/tdx-types/src/tdcall.rs index 1619593d..1d4b8765 100644 --- a/common/tdx-types/src/tdcall.rs +++ b/common/tdx-types/src/tdcall.rs @@ -52,6 +52,9 @@ impl MdFieldId { pub const LSTAR_WRITE: Self = Self::msr_bitmaps1(0xC000_0082, true); pub const LSTAR_WRITE_MASK: u64 = Self::msr_bitmaps_mask(0xC000_0082); + pub const SFMASK_WRITE: Self = Self::msr_bitmaps1(0xC000_0084, true); + pub const SFMASK_WRITE_MASK: u64 = Self::msr_bitmaps_mask(0xC000_0084); + pub const TDVPS_L2_CTLS1: Self = Self::new( 81, ElementSizeCode::SixtyFour, diff --git a/tee/kernel/src/user/process/syscall/cpu_state.rs b/tee/kernel/src/user/process/syscall/cpu_state.rs index 72b4d741..57f003e7 100644 --- a/tee/kernel/src/user/process/syscall/cpu_state.rs +++ b/tee/kernel/src/user/process/syscall/cpu_state.rs @@ -14,7 +14,12 @@ use usize_conversions::{usize_from, FromUsize}; use x86_64::{ align_down, instructions::tables::{lgdt, sgdt}, - registers::{control::Cr2, model_specific::LStar, xcontrol::XCr0Flags}, + registers::{ + control::Cr2, + model_specific::{LStar, SFMask}, + rflags::RFlags, + xcontrol::XCr0Flags, + }, structures::{gdt::Entry, idt::PageFaultErrorCode, DescriptorTablePointer}, VirtAddr, }; @@ -758,6 +763,7 @@ struct FpxSwBytes { pub fn init() { LStar::write(VirtAddr::new(syscall_entry as usize as u64)); + SFMask::write(RFlags::DIRECTION_FLAG); } unsafe extern "sysv64" { diff --git a/tee/supervisor-tdx/src/vcpu.rs b/tee/supervisor-tdx/src/vcpu.rs index ab978c84..62022778 100644 --- a/tee/supervisor-tdx/src/vcpu.rs +++ b/tee/supervisor-tdx/src/vcpu.rs @@ -131,6 +131,7 @@ pub unsafe fn init_vcpu(apic: &mut Apic) { Tdcall::vp_wr(MdFieldId::STAR_WRITE, 0, MdFieldId::STAR_WRITE_MASK); Tdcall::vp_wr(MdFieldId::LSTAR_WRITE, 0, MdFieldId::LSTAR_WRITE_MASK); + Tdcall::vp_wr(MdFieldId::SFMASK_WRITE, 0, MdFieldId::SFMASK_WRITE_MASK); } pub fn run_vcpu() -> ! { From 8f192c51754ab0663f21b79f30ef573eaf8a6189 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 2 Nov 2024 08:31:34 +0000 Subject: [PATCH 6/8] clear the direction flag in exception handlers --- tee/kernel/src/exception.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tee/kernel/src/exception.rs b/tee/kernel/src/exception.rs index be939f42..511801b1 100644 --- a/tee/kernel/src/exception.rs +++ b/tee/kernel/src/exception.rs @@ -171,6 +171,7 @@ pub fn load_idt() { extern "x86-interrupt" fn divide_error_handler(frame: InterruptStackFrame) { unsafe { naked_asm!( + "cld", // Check whether the exception happened in userspace. "test word ptr [rsp+16], 3", "je {kernel_divide_error_handler}", @@ -200,6 +201,7 @@ extern "x86-interrupt" fn page_fault_handler( ) { unsafe { naked_asm!( + "cld", // Check whether the exception happened in userspace. "test word ptr [rsp+16], 3", "je 66f", @@ -304,6 +306,7 @@ extern "x86-interrupt" fn general_protection_fault_handler( ) { unsafe { naked_asm!( + "cld", // Check whether the exception happened in userspace. "test word ptr [rsp+16], 3", "je {kernel_general_protection_fault_handler}", @@ -339,6 +342,7 @@ extern "x86-interrupt" fn double_fault_handler(frame: InterruptStackFrame, code: extern "x86-interrupt" fn vc_handler(frame: InterruptStackFrame, error_code: u64) { unsafe { naked_asm!( + "cld", // Check whether the exception happened in userspace. "test word ptr [rsp+16], 3", "je {kernel_vc_handler}", @@ -363,6 +367,7 @@ extern "x86-interrupt" fn vc_handler(frame: InterruptStackFrame, error_code: u64 extern "x86-interrupt" fn kernel_vc_handler(frame: InterruptStackFrame, code: u64) { unsafe { naked_asm!( + "cld", "push r11", "push r10", "push r9", @@ -465,6 +470,7 @@ extern "x86-interrupt" fn int0x80_handler(frame: InterruptStackFrame) { // continue when userspace exits. unsafe { naked_asm!( + "cld", "swapgs", "mov byte ptr gs:[{VECTOR_OFFSET}], 0x80", "jmp {exception_entry}", From fae4709f648917ed2ddddaaa06f06de54bfe6db3 Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 2 Nov 2024 08:32:07 +0000 Subject: [PATCH 7/8] implement wrapper for TDG.VP.VEINFO.GET --- tee/supervisor-tdx/src/tdcall.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tee/supervisor-tdx/src/tdcall.rs b/tee/supervisor-tdx/src/tdcall.rs index 9b35f928..c585ae01 100644 --- a/tee/supervisor-tdx/src/tdcall.rs +++ b/tee/supervisor-tdx/src/tdcall.rs @@ -244,6 +244,33 @@ impl Tdcall { (tdcall.rax, (tdcall.r11 >> 32) as u32) } + + pub fn vp_veinfo_get() -> VeInfo { + let mut tdcall = Self::new(3); + unsafe { + tdcall.execute(); + } + + assert_eq!(tdcall.rax, 0); + VeInfo { + exit_reason: tdcall.rcx as u32, + exit_qualification: tdcall.rdx, + guest_linear_address: tdcall.r8, + guest_physical_address: tdcall.r9, + instruction_length: tdcall.r10 as u32, + instruction_information: (tdcall.r10 >> 32) as u32, + } + } +} + +#[expect(dead_code)] +pub struct VeInfo { + pub exit_reason: u32, + pub exit_qualification: u64, + pub guest_linear_address: u64, + pub guest_physical_address: u64, + pub instruction_length: u32, + pub instruction_information: u32, } pub struct Vmcall { From 351dcba2b12586175efdf12f7204fb1acec302ed Mon Sep 17 00:00:00 2001 From: Tom Dohrmann Date: Sat, 2 Nov 2024 08:33:48 +0000 Subject: [PATCH 8/8] implement #VE handler for unsupported CPUID leaves --- tee/supervisor-tdx/src/exception.rs | 78 ++++++++++++++++++++++++++++- tee/supervisor-tdx/src/main.rs | 2 +- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/tee/supervisor-tdx/src/exception.rs b/tee/supervisor-tdx/src/exception.rs index 7d3bdd8c..c93bbe5b 100644 --- a/tee/supervisor-tdx/src/exception.rs +++ b/tee/supervisor-tdx/src/exception.rs @@ -1,11 +1,17 @@ +use core::arch::naked_asm; + use bit_field::BitField; use spin::Lazy; +use tdx_types::vmexit::VMEXIT_REASON_CPUID_INSTRUCTION; use x86_64::{ registers::model_specific::Msr, structures::idt::{InterruptDescriptorTable, InterruptStackFrame}, }; -use crate::{tdcall::Vmcall, tlb::flush_handler}; +use crate::{ + tdcall::{Tdcall, Vmcall}, + tlb::flush_handler, +}; pub const WAKEUP_VECTOR: u8 = 0x60; pub const FLUSH_VECTOR: u8 = 0x61; @@ -18,11 +24,81 @@ pub fn setup_idt() { static IDT: Lazy = Lazy::new(|| { let mut idt = InterruptDescriptorTable::new(); + idt.virtualization.set_handler_fn(virtualization_handler); idt[WAKEUP_VECTOR].set_handler_fn(wakeup_handler); idt[FLUSH_VECTOR].set_handler_fn(flush_handler); idt }); +/// The set of caller-saved registers + rbx. +#[repr(C)] +struct VeStackFrame { + rax: u64, + rbx: u64, + rcx: u64, + rdx: u64, + rsi: u64, + rdi: u64, + r8: u64, + r9: u64, + r10: u64, + r11: u64, + error_code: u64, + rip: u64, +} + +#[naked] +extern "x86-interrupt" fn virtualization_handler(_frame: InterruptStackFrame) { + unsafe { + naked_asm!( + "endbr64", + "cld", + "push r11", + "push r10", + "push r9", + "push r8", + "push rdi", + "push rsi", + "push rdx", + "push rcx", + "push rbx", + "push rax", + "mov rdi, rsp", + "sub rsp, 8", + // TODO: make sure alignment is correct. + "call {virtualization_handler_impl}", + "add rsp, 8", + "pop rax", + "pop rbx", + "pop rcx", + "pop rdx", + "pop rsi", + "pop rdi", + "pop r8", + "pop r9", + "pop r10", + "pop r11", + "iretq", + virtualization_handler_impl = sym virtualization_handler_impl, + ); + } +} + +extern "C" fn virtualization_handler_impl(frame: &mut VeStackFrame) { + let ve_info = Tdcall::vp_veinfo_get(); + match ve_info.exit_reason { + VMEXIT_REASON_CPUID_INSTRUCTION => { + // The TDX-module injects a #VE exception for unsupported CPUID + // leaves. Default to all-zeroes. + frame.rax = 0; + frame.rbx = 0; + frame.rcx = 0; + frame.rdx = 0; + } + reason => unimplemented!("unimplemented #VE reason: {reason}"), + } +} + extern "x86-interrupt" fn wakeup_handler(_frame: InterruptStackFrame) { // Don't do anything. // This handler only exists to move past `hlt`. diff --git a/tee/supervisor-tdx/src/main.rs b/tee/supervisor-tdx/src/main.rs index 4810ca4e..1c702a5d 100644 --- a/tee/supervisor-tdx/src/main.rs +++ b/tee/supervisor-tdx/src/main.rs @@ -1,6 +1,6 @@ #![no_std] #![no_main] -#![feature(abi_x86_interrupt, core_intrinsics, sync_unsafe_cell)] +#![feature(abi_x86_interrupt, core_intrinsics, naked_functions, sync_unsafe_cell)] #![allow(internal_features)] use exception::setup_idt;