diff --git a/tests/disas/winch/x64/atomic/load/i32_atomic_load.wat b/tests/disas/winch/x64/atomic/load/i32_atomic_load.wat index a671002d8a51..fb6e1f72dc06 100644 --- a/tests/disas/winch/x64/atomic/load/i32_atomic_load.wat +++ b/tests/disas/winch/x64/atomic/load/i32_atomic_load.wat @@ -13,23 +13,22 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x20, %r11 ;; cmpq %rsp, %r11 -;; ja 0x57 +;; ja 0x55 ;; 1c: movq %rdi, %r14 ;; subq $0x20, %rsp ;; movq %rdi, 0x18(%rsp) ;; movq %rsi, 0x10(%rsp) ;; movl %edx, 0xc(%rsp) ;; movl 0xc(%rsp), %eax -;; andl $3, %eax -;; cmpl $0, %eax -;; jne 0x59 -;; 41: movl 0xc(%rsp), %eax -;; movq 0x58(%r14), %r11 +;; andl $3, %r11d +;; cmpl $0, %r11d +;; jne 0x57 +;; 43: movq 0x58(%r14), %r11 ;; movq (%r11), %rcx ;; addq %rax, %rcx ;; movl (%rcx), %eax ;; addq $0x20, %rsp ;; popq %rbp ;; retq +;; 55: ud2 ;; 57: ud2 -;; 59: ud2 diff --git a/tests/disas/winch/x64/atomic/load/i32_atomic_load16_u.wat b/tests/disas/winch/x64/atomic/load/i32_atomic_load16_u.wat index 3c156158518b..2e4f28ef8a62 100644 --- a/tests/disas/winch/x64/atomic/load/i32_atomic_load16_u.wat +++ b/tests/disas/winch/x64/atomic/load/i32_atomic_load16_u.wat @@ -13,23 +13,22 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x20, %r11 ;; cmpq %rsp, %r11 -;; ja 0x5b +;; ja 0x59 ;; 1c: movq %rdi, %r14 ;; subq $0x20, %rsp ;; movq %rdi, 0x18(%rsp) ;; movq %rsi, 0x10(%rsp) ;; movl %edx, 0xc(%rsp) ;; movl 0xc(%rsp), %eax -;; andw $1, %ax -;; cmpw $0, %ax -;; jne 0x5d -;; 43: movl 0xc(%rsp), %eax -;; movq 0x58(%r14), %r11 +;; andw $1, %r11w +;; cmpw $0, %r11w +;; jne 0x5b +;; 45: movq 0x58(%r14), %r11 ;; movq (%r11), %rcx ;; addq %rax, %rcx ;; movzwq (%rcx), %rax ;; addq $0x20, %rsp ;; popq %rbp ;; retq +;; 59: ud2 ;; 5b: ud2 -;; 5d: ud2 diff --git a/tests/disas/winch/x64/atomic/load/i64_atomic_load.wat b/tests/disas/winch/x64/atomic/load/i64_atomic_load.wat index 6d67f5534ba1..e8cbbfb071cc 100644 --- a/tests/disas/winch/x64/atomic/load/i64_atomic_load.wat +++ b/tests/disas/winch/x64/atomic/load/i64_atomic_load.wat @@ -14,21 +14,20 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x54 +;; ja 0x4f ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movl $0, %eax -;; andq $7, %rax -;; cmpq $0, %rax -;; jne 0x56 -;; 3f: movl $0, %eax -;; movq 0x60(%r14), %rcx +;; andq $7, %r11 +;; cmpq $0, %r11 +;; jne 0x51 +;; 3f: movq 0x60(%r14), %rcx ;; addq %rax, %rcx ;; movq (%rcx), %rax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 54: ud2 -;; 56: ud2 +;; 4f: ud2 +;; 51: ud2 diff --git a/tests/disas/winch/x64/atomic/load/i64_atomic_load16_u.wat b/tests/disas/winch/x64/atomic/load/i64_atomic_load16_u.wat index dbd25c81996a..1c62828769db 100644 --- a/tests/disas/winch/x64/atomic/load/i64_atomic_load16_u.wat +++ b/tests/disas/winch/x64/atomic/load/i64_atomic_load16_u.wat @@ -14,21 +14,20 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x55 +;; ja 0x52 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movl $0, %eax -;; andw $1, %ax -;; cmpw $0, %ax -;; jne 0x57 -;; 3f: movl $0, %eax -;; movq 0x60(%r14), %rcx +;; andw $1, %r11w +;; cmpw $0, %r11w +;; jne 0x54 +;; 41: movq 0x60(%r14), %rcx ;; addq %rax, %rcx ;; movzwq (%rcx), %rax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 55: ud2 -;; 57: ud2 +;; 52: ud2 +;; 54: ud2 diff --git a/tests/disas/winch/x64/atomic/load/i64_atomic_load32_u.wat b/tests/disas/winch/x64/atomic/load/i64_atomic_load32_u.wat index a7ad511e0d37..88b0e886b3e8 100644 --- a/tests/disas/winch/x64/atomic/load/i64_atomic_load32_u.wat +++ b/tests/disas/winch/x64/atomic/load/i64_atomic_load32_u.wat @@ -14,21 +14,20 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x51 +;; ja 0x4e ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movl $0, %eax -;; andl $3, %eax -;; cmpl $0, %eax -;; jne 0x53 -;; 3d: movl $0, %eax -;; movq 0x60(%r14), %rcx +;; andl $3, %r11d +;; cmpl $0, %r11d +;; jne 0x50 +;; 3f: movq 0x60(%r14), %rcx ;; addq %rax, %rcx ;; movl (%rcx), %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 51: ud2 -;; 53: ud2 +;; 4e: ud2 +;; 50: ud2 diff --git a/tests/disas/winch/x64/atomic/store/i32_atomic_store.wat b/tests/disas/winch/x64/atomic/store/i32_atomic_store.wat index b6f98ecffc96..f664c721b7c8 100644 --- a/tests/disas/winch/x64/atomic/store/i32_atomic_store.wat +++ b/tests/disas/winch/x64/atomic/store/i32_atomic_store.wat @@ -11,18 +11,17 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x5c +;; ja 0x59 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movl $0x2a, %eax ;; movl $0, %ecx -;; andl $3, %ecx -;; cmpl $0, %ecx -;; jne 0x5e -;; 42: movl $0, %ecx -;; movq 0x58(%r14), %r11 +;; andl $3, %r11d +;; cmpl $0, %r11d +;; jne 0x5b +;; 44: movq 0x58(%r14), %r11 ;; movq (%r11), %rdx ;; addq %rcx, %rdx ;; movl %eax, (%rdx) @@ -30,5 +29,5 @@ ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 5c: ud2 -;; 5e: ud2 +;; 59: ud2 +;; 5b: ud2 diff --git a/tests/disas/winch/x64/atomic/store/i32_atomic_store16.wat b/tests/disas/winch/x64/atomic/store/i32_atomic_store16.wat index 3b48acb6bcf6..56b7933cae63 100644 --- a/tests/disas/winch/x64/atomic/store/i32_atomic_store16.wat +++ b/tests/disas/winch/x64/atomic/store/i32_atomic_store16.wat @@ -11,18 +11,17 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x5f +;; ja 0x5c ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movl $0x2a, %eax ;; movl $0, %ecx -;; andw $1, %cx -;; cmpw $0, %cx -;; jne 0x61 -;; 44: movl $0, %ecx -;; movq 0x58(%r14), %r11 +;; andw $1, %r11w +;; cmpw $0, %r11w +;; jne 0x5e +;; 46: movq 0x58(%r14), %r11 ;; movq (%r11), %rdx ;; addq %rcx, %rdx ;; movw %ax, (%rdx) @@ -30,5 +29,5 @@ ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 5f: ud2 -;; 61: ud2 +;; 5c: ud2 +;; 5e: ud2 diff --git a/tests/disas/winch/x64/atomic/store/i64_atomic_store.wat b/tests/disas/winch/x64/atomic/store/i64_atomic_store.wat index 9852a8983c31..1aa168325a8f 100644 --- a/tests/disas/winch/x64/atomic/store/i64_atomic_store.wat +++ b/tests/disas/winch/x64/atomic/store/i64_atomic_store.wat @@ -11,18 +11,17 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x61 +;; ja 0x5c ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movq $0x2a, %rax ;; movl $0, %ecx -;; andq $7, %rcx -;; cmpq $0, %rcx -;; jne 0x63 -;; 46: movl $0, %ecx -;; movq 0x58(%r14), %r11 +;; andq $7, %r11 +;; cmpq $0, %r11 +;; jne 0x5e +;; 46: movq 0x58(%r14), %r11 ;; movq (%r11), %rdx ;; addq %rcx, %rdx ;; movq %rax, (%rdx) @@ -30,5 +29,5 @@ ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 61: ud2 -;; 63: ud2 +;; 5c: ud2 +;; 5e: ud2 diff --git a/tests/disas/winch/x64/atomic/store/i64_atomic_store16.wat b/tests/disas/winch/x64/atomic/store/i64_atomic_store16.wat index c13d460fb61b..0699c95cb3c6 100644 --- a/tests/disas/winch/x64/atomic/store/i64_atomic_store16.wat +++ b/tests/disas/winch/x64/atomic/store/i64_atomic_store16.wat @@ -11,18 +11,17 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x61 +;; ja 0x5e ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movq $0x2a, %rax ;; movl $0, %ecx -;; andw $1, %cx -;; cmpw $0, %cx -;; jne 0x63 -;; 46: movl $0, %ecx -;; movq 0x58(%r14), %r11 +;; andw $1, %r11w +;; cmpw $0, %r11w +;; jne 0x60 +;; 48: movq 0x58(%r14), %r11 ;; movq (%r11), %rdx ;; addq %rcx, %rdx ;; movw %ax, (%rdx) @@ -30,5 +29,5 @@ ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 61: ud2 -;; 63: ud2 +;; 5e: ud2 +;; 60: ud2 diff --git a/tests/disas/winch/x64/atomic/store/i64_atomic_store32.wat b/tests/disas/winch/x64/atomic/store/i64_atomic_store32.wat index 8a4d056aa004..8278d67d57a5 100644 --- a/tests/disas/winch/x64/atomic/store/i64_atomic_store32.wat +++ b/tests/disas/winch/x64/atomic/store/i64_atomic_store32.wat @@ -11,18 +11,17 @@ ;; movq 0x10(%r11), %r11 ;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x5e +;; ja 0x5b ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) ;; movq $0x2a, %rax ;; movl $0, %ecx -;; andl $3, %ecx -;; cmpl $0, %ecx -;; jne 0x60 -;; 44: movl $0, %ecx -;; movq 0x58(%r14), %r11 +;; andl $3, %r11d +;; cmpl $0, %r11d +;; jne 0x5d +;; 46: movq 0x58(%r14), %r11 ;; movq (%r11), %rdx ;; addq %rcx, %rdx ;; movl %eax, (%rdx) @@ -30,5 +29,5 @@ ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 5e: ud2 -;; 60: ud2 +;; 5b: ud2 +;; 5d: ud2 diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index ef4a7ef0c9cc..5bd5b8973df9 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -648,12 +648,7 @@ where &mut self, memarg: &MemArg, access_size: OperandSize, - check_align: bool, ) -> Result> { - if check_align { - self.check_align(memarg, access_size)?; - } - let ptr_size: OperandSize = self.env.ptr_type().try_into()?; let enable_spectre_mitigation = self.env.heap_access_spectre_mitigation(); let add_offset_and_access_size = |offset: ImmOffset, access_size: OperandSize| { @@ -846,22 +841,25 @@ where } /// Emit checks to ensure that the address at `memarg` is correctly aligned for `size`. - fn check_align(&mut self, memarg: &MemArg, size: OperandSize) -> Result<()> { + fn emit_check_align(&mut self, memarg: &MemArg, size: OperandSize) -> Result<()> { if size.bytes() > 1 { - let addr = *self.context.stack.peek().unwrap(); - let tmp = self.context.any_gpr(self.masm)?; - self.context.move_val_to_reg(&addr, tmp, self.masm)?; - if memarg.offset != 0 { + let addr = self.context.pop_to_reg(self.masm, None)?; + let tmp = scratch!(M); + let effective_addr = if memarg.offset != 0 { self.masm.add( writable!(tmp), - tmp, + addr.reg, RegImm::Imm(Imm::I64(memarg.offset)), size, )?; + tmp + } else { + addr.reg }; + self.masm.and( writable!(tmp), - tmp, + effective_addr, RegImm::Imm(Imm::I32(size.bytes() - 1)), size, )?; @@ -869,12 +867,21 @@ where self.masm.cmp(tmp, RegImm::Imm(Imm::i64(0)), size)?; self.masm.trapif(IntCmpKind::Ne, TRAP_HEAP_MISALIGNED)?; - self.context.free_reg(tmp); + self.context.stack.push(addr.into()); } Ok(()) } + pub fn emit_compute_heap_address_align_checked( + &mut self, + memarg: &MemArg, + access_size: OperandSize, + ) -> Result> { + self.emit_check_align(memarg, access_size)?; + self.emit_compute_heap_address(memarg, access_size) + } + /// Emit a WebAssembly load. pub fn emit_wasm_load( &mut self, @@ -884,7 +891,12 @@ where kind: LoadKind, op_kind: MemOpKind, ) -> Result<()> { - if let Some(addr) = self.emit_compute_heap_address(&arg, size, op_kind.is_atomic())? { + let maybe_addr = match op_kind { + MemOpKind::Atomic => self.emit_compute_heap_address_align_checked(&arg, size)?, + MemOpKind::Normal => self.emit_compute_heap_address(&arg, size)?, + }; + + if let Some(addr) = maybe_addr { let dst = match ty { WasmValType::I32 | WasmValType::I64 => self.context.any_gpr(self.masm)?, WasmValType::F32 | WasmValType::F64 => self.context.any_fpr(self.masm)?, @@ -910,8 +922,13 @@ where op_kind: MemOpKind, ) -> Result<()> { let src = self.context.pop_to_reg(self.masm, None)?; - let addr = self.emit_compute_heap_address(&arg, size, op_kind.is_atomic())?; - if let Some(addr) = addr { + + let maybe_addr = match op_kind { + MemOpKind::Atomic => self.emit_compute_heap_address_align_checked(&arg, size)?, + MemOpKind::Normal => self.emit_compute_heap_address(&arg, size)?, + }; + + if let Some(addr) = maybe_addr { self.masm.wasm_store( src.reg.into(), self.masm.address_at_reg(addr, 0)?, diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs index 8b431821e569..ea6ded5f0d58 100644 --- a/winch/codegen/src/masm.rs +++ b/winch/codegen/src/masm.rs @@ -46,12 +46,6 @@ pub(crate) enum MemOpKind { Normal, } -impl MemOpKind { - pub(crate) fn is_atomic(&self) -> bool { - matches!(self, Self::Atomic) - } -} - #[derive(Eq, PartialEq)] pub(crate) enum MulWideKind { Signed,