diff --git a/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw16_andu.wat b/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw16_andu.wat index d741af7232eb..a868b5b31056 100644 --- a/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw16_andu.wat +++ b/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw16_andu.wat @@ -10,34 +10,30 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x14, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x80 +;; ja 0x72 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x2a, %eax -;; movl $0, %ecx -;; andw $1, %cx -;; cmpw $0, %cx -;; jne 0x82 -;; 44: movl $0, %ecx +;; movl $0x2a, %ecx +;; movl $0, %edx +;; andw $1, %dx +;; cmpw $0, %dx +;; jne 0x74 +;; 44: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movzwq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzwq (%rbx), %rax ;; movq %rax, %r11 ;; andq %rcx, %r11 -;; lock cmpxchgw %r11w, (%rdx) -;; jne 0x65 -;; 77: movzwl %ax, %eax +;; lock cmpxchgw %r11w, (%rbx) +;; jne 0x57 +;; 69: movzwl %ax, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 80: ud2 -;; 82: ud2 +;; 72: ud2 +;; 74: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw8_andu.wat b/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw8_andu.wat index 1530f1590e7e..e7dc10731acd 100644 --- a/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw8_andu.wat +++ b/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw8_andu.wat @@ -10,29 +10,25 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x14, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x6c +;; ja 0x5e ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x2a, %eax -;; movl $0, %ecx +;; movl $0x2a, %ecx +;; movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movzbq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzbq (%rbx), %rax ;; movq %rax, %r11 ;; andq %rcx, %r11 -;; lock cmpxchgb %r11b, (%rdx) -;; jne 0x52 -;; 63: movzbl %al, %eax +;; lock cmpxchgb %r11b, (%rbx) +;; jne 0x44 +;; 55: movzbl %al, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 6c: ud2 +;; 5e: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw_and.wat b/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw_and.wat index dc72c4624bab..413ce0f3df0c 100644 --- a/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw_and.wat +++ b/tests/disas/winch/x64/atomic/rmw/and/i32_atomic_rmw_and.wat @@ -10,33 +10,29 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x14, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x78 +;; ja 0x6a ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x2a, %eax -;; movl $0, %ecx -;; andl $3, %ecx -;; cmpl $0, %ecx -;; jne 0x7a -;; 42: movl $0, %ecx +;; movl $0x2a, %ecx +;; movl $0, %edx +;; andl $3, %edx +;; cmpl $0, %edx +;; jne 0x6c +;; 42: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movl (%rdx), %eax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movl (%rbx), %eax ;; movq %rax, %r11 ;; andq %rcx, %r11 -;; lock cmpxchgl %r11d, (%rdx) -;; jne 0x61 -;; 72: addq $0x10, %rsp +;; lock cmpxchgl %r11d, (%rbx) +;; jne 0x53 +;; 64: addq $0x10, %rsp ;; popq %rbp ;; retq -;; 78: ud2 -;; 7a: ud2 +;; 6a: ud2 +;; 6c: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw16_andu.wat b/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw16_andu.wat index f472a991e11f..5a47b6bb20a4 100644 --- a/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw16_andu.wat +++ b/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw16_andu.wat @@ -10,32 +10,30 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x77 +;; ja 0x75 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx -;; andw $1, %cx -;; cmpw $0, %cx -;; jne 0x79 -;; 46: movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx +;; andw $1, %dx +;; cmpw $0, %dx +;; jne 0x77 +;; 46: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movzwq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzwq (%rbx), %rax ;; movq %rax, %r11 ;; andq %rcx, %r11 -;; lock cmpxchgw %r11w, (%rdx) -;; jne 0x5b -;; 6d: movzwq %ax, %rax +;; lock cmpxchgw %r11w, (%rbx) +;; jne 0x59 +;; 6b: movzwq %ax, %rax ;; addq $0x10, %rsp ;; popq %rbp ;; retq +;; 75: ud2 ;; 77: ud2 -;; 79: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw32_andu.wat b/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw32_andu.wat index 0ea7edadbd72..193decb919a1 100644 --- a/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw32_andu.wat +++ b/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw32_andu.wat @@ -10,31 +10,29 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x6e +;; ja 0x6c ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx -;; andl $3, %ecx -;; cmpl $0, %ecx -;; jne 0x70 -;; 44: movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx +;; andl $3, %edx +;; cmpl $0, %edx +;; jne 0x6e +;; 44: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movl (%rdx), %eax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movl (%rbx), %eax ;; movq %rax, %r11 ;; andq %rcx, %r11 -;; lock cmpxchgl %r11d, (%rdx) -;; jne 0x57 -;; 68: addq $0x10, %rsp +;; lock cmpxchgl %r11d, (%rbx) +;; jne 0x55 +;; 66: addq $0x10, %rsp ;; popq %rbp ;; retq +;; 6c: ud2 ;; 6e: ud2 -;; 70: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw8_andu.wat b/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw8_andu.wat index 929e9c5cd729..27d75f6a962d 100644 --- a/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw8_andu.wat +++ b/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw8_andu.wat @@ -10,27 +10,25 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x63 +;; ja 0x61 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movzbq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzbq (%rbx), %rax ;; movq %rax, %r11 ;; andq %rcx, %r11 -;; lock cmpxchgb %r11b, (%rdx) -;; jne 0x48 -;; 59: movzbq %al, %rax +;; lock cmpxchgb %r11b, (%rbx) +;; jne 0x46 +;; 57: movzbq %al, %rax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 63: ud2 +;; 61: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw_and.wat b/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw_and.wat index f89677bc370e..5da403c95d38 100644 --- a/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw_and.wat +++ b/tests/disas/winch/x64/atomic/rmw/and/i64_atomic_rmw_and.wat @@ -10,31 +10,29 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x71 +;; ja 0x6f ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx -;; andq $7, %rcx -;; cmpq $0, %rcx -;; jne 0x73 -;; 46: movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx +;; andq $7, %rdx +;; cmpq $0, %rdx +;; jne 0x71 +;; 46: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movq (%rbx), %rax ;; movq %rax, %r11 ;; andq %rcx, %r11 -;; lock cmpxchgq %r11, (%rdx) -;; jne 0x5a -;; 6b: addq $0x10, %rsp +;; lock cmpxchgq %r11, (%rbx) +;; jne 0x58 +;; 69: addq $0x10, %rsp ;; popq %rbp ;; retq +;; 6f: ud2 ;; 71: ud2 -;; 73: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw16_cmpxchgu.wat b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw16_cmpxchgu.wat index 968a175c1631..b758557da464 100644 --- a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw16_cmpxchgu.wat +++ b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw16_cmpxchgu.wat @@ -10,35 +10,27 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x82 +;; ja 0x66 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x539, %eax -;; movl $0x2a, %ecx -;; movl $0, %edx -;; andw $1, %dx -;; cmpw $0, %dx -;; jne 0x84 -;; 49: movl $0, %edx +;; movl $0x539, %ecx +;; andw $1, %cx +;; cmpw $0, %cx +;; jne 0x68 +;; 3f: movl $0x539, %ecx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rbx -;; addq %rdx, %rbx -;; subq $4, %rsp -;; movl %ecx, (%rsp) -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movl (%rsp), %eax -;; addq $4, %rsp -;; lock cmpxchgw %cx, (%rbx) +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; movl $0x2a, %ecx +;; movl $0, %eax +;; lock cmpxchgw %cx, (%rdx) ;; movzwl %ax, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 82: ud2 -;; 84: ud2 +;; 66: ud2 +;; 68: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw8_cmpxchgu.wat b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw8_cmpxchgu.wat index d70743368e87..3ea7b177006b 100644 --- a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw8_cmpxchgu.wat +++ b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw8_cmpxchgu.wat @@ -10,30 +10,22 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x6e +;; ja 0x52 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x539, %eax -;; movl $0x2a, %ecx -;; movl $0, %edx +;; movl $0x539, %ecx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rbx -;; addq %rdx, %rbx -;; subq $4, %rsp -;; movl %ecx, (%rsp) -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movl (%rsp), %eax -;; addq $4, %rsp -;; lock cmpxchgb %cl, (%rbx) +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; movl $0x2a, %ecx +;; movl $0, %eax +;; lock cmpxchgb %cl, (%rdx) ;; movzbl %al, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 6e: ud2 +;; 52: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw_cmpxchg.wat b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw_cmpxchg.wat index e40c4a0c3576..ee308a7fe1b1 100644 --- a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw_cmpxchg.wat +++ b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i32_atomic_rmw_cmpxchg.wat @@ -10,34 +10,26 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x7c +;; ja 0x60 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x539, %eax -;; movl $0x2a, %ecx -;; movl $0, %edx -;; andl $3, %edx -;; cmpl $0, %edx -;; jne 0x7e -;; 47: movl $0, %edx +;; movl $0x539, %ecx +;; andl $3, %ecx +;; cmpl $0, %ecx +;; jne 0x62 +;; 3d: movl $0x539, %ecx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rbx -;; addq %rdx, %rbx -;; subq $4, %rsp -;; movl %ecx, (%rsp) -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movl (%rsp), %eax -;; addq $4, %rsp -;; lock cmpxchgl %ecx, (%rbx) +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; movl $0x2a, %ecx +;; movl $0, %eax +;; lock cmpxchgl %ecx, (%rdx) ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 7c: ud2 -;; 7e: ud2 +;; 60: ud2 +;; 62: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw16_cmpxchgu.wat b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw16_cmpxchgu.wat index 8f772dff3144..4dab62511f2e 100644 --- a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw16_cmpxchgu.wat +++ b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw16_cmpxchgu.wat @@ -10,31 +10,27 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x20, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x6f +;; ja 0x6d ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x539, %rax -;; movq $0x2a, %rcx -;; movl $0, %edx -;; andw $1, %dx -;; cmpw $0, %dx -;; jne 0x71 -;; 4d: movl $0, %edx +;; movq $0x539, %rcx +;; andw $1, %cx +;; cmpw $0, %cx +;; jne 0x6f +;; 41: movq $0x539, %rcx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rbx -;; addq %rdx, %rbx -;; pushq %rcx -;; pushq %rax -;; popq %rcx -;; popq %rax -;; lock cmpxchgw %cx, (%rbx) +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; movq $0x2a, %rcx +;; movl $0, %eax +;; lock cmpxchgw %cx, (%rdx) ;; movzwq %ax, %rax ;; addq $0x10, %rsp ;; popq %rbp ;; retq +;; 6d: ud2 ;; 6f: ud2 -;; 71: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw32_cmpxchgu.wat b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw32_cmpxchgu.wat index e4d52a25deed..a4bbaf0b46f2 100644 --- a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw32_cmpxchgu.wat +++ b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw32_cmpxchgu.wat @@ -10,30 +10,26 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x20, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x68 +;; ja 0x66 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x539, %rax -;; movq $0x2a, %rcx -;; movl $0, %edx -;; andl $3, %edx -;; cmpl $0, %edx -;; jne 0x6a -;; 4b: movl $0, %edx +;; movq $0x539, %rcx +;; andl $3, %ecx +;; cmpl $0, %ecx +;; jne 0x68 +;; 3f: movq $0x539, %rcx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rbx -;; addq %rdx, %rbx -;; pushq %rcx -;; pushq %rax -;; popq %rcx -;; popq %rax -;; lock cmpxchgl %ecx, (%rbx) +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; movq $0x2a, %rcx +;; movl $0, %eax +;; lock cmpxchgl %ecx, (%rdx) ;; addq $0x10, %rsp ;; popq %rbp ;; retq +;; 66: ud2 ;; 68: ud2 -;; 6a: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw8_cmpxchgu.wat b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw8_cmpxchgu.wat index 5379d2658778..7f975efbab41 100644 --- a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw8_cmpxchgu.wat +++ b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw8_cmpxchgu.wat @@ -10,26 +10,22 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x20, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x5b +;; ja 0x57 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x539, %rax -;; movq $0x2a, %rcx -;; movl $0, %edx +;; movq $0x539, %rcx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rbx -;; addq %rdx, %rbx -;; pushq %rcx -;; pushq %rax -;; popq %rcx -;; popq %rax -;; lock cmpxchgb %cl, (%rbx) +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; movq $0x2a, %rcx +;; movl $0, %eax +;; lock cmpxchgb %cl, (%rdx) ;; movzbq %al, %rax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 5b: ud2 +;; 57: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw_cmpxchg.wat b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw_cmpxchg.wat index 809439283b45..076d7993cb5c 100644 --- a/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw_cmpxchg.wat +++ b/tests/disas/winch/x64/atomic/rmw/cmpxchg/i64_atomic_rmw_cmpxchg.wat @@ -10,30 +10,26 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x20, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x6b +;; ja 0x69 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x539, %rax -;; movq $0x2a, %rcx -;; movl $0, %edx -;; andq $7, %rdx -;; cmpq $0, %rdx -;; jne 0x6d -;; 4d: movl $0, %edx +;; movq $0x539, %rcx +;; andq $7, %rcx +;; cmpq $0, %rcx +;; jne 0x6b +;; 41: movq $0x539, %rcx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rbx -;; addq %rdx, %rbx -;; pushq %rcx -;; pushq %rax -;; popq %rcx -;; popq %rax -;; lock cmpxchgq %rcx, (%rbx) +;; movq (%r11), %rdx +;; addq %rcx, %rdx +;; movq $0x2a, %rcx +;; movl $0, %eax +;; lock cmpxchgq %rcx, (%rdx) ;; addq $0x10, %rsp ;; popq %rbp ;; retq +;; 69: ud2 ;; 6b: ud2 -;; 6d: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw16_oru.wat b/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw16_oru.wat index 1dea6ca53f06..382e4d28cfe6 100644 --- a/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw16_oru.wat +++ b/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw16_oru.wat @@ -10,34 +10,30 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x14, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x80 +;; ja 0x72 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x2a, %eax -;; movl $0, %ecx -;; andw $1, %cx -;; cmpw $0, %cx -;; jne 0x82 -;; 44: movl $0, %ecx +;; movl $0x2a, %ecx +;; movl $0, %edx +;; andw $1, %dx +;; cmpw $0, %dx +;; jne 0x74 +;; 44: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movzwq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzwq (%rbx), %rax ;; movq %rax, %r11 ;; orq %rcx, %r11 -;; lock cmpxchgw %r11w, (%rdx) -;; jne 0x65 -;; 77: movzwl %ax, %eax +;; lock cmpxchgw %r11w, (%rbx) +;; jne 0x57 +;; 69: movzwl %ax, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 80: ud2 -;; 82: ud2 +;; 72: ud2 +;; 74: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw8_oru.wat b/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw8_oru.wat index 2376eed4b515..a3e634d4c32c 100644 --- a/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw8_oru.wat +++ b/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw8_oru.wat @@ -10,29 +10,25 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x14, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x6c +;; ja 0x5e ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x2a, %eax -;; movl $0, %ecx +;; movl $0x2a, %ecx +;; movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movzbq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzbq (%rbx), %rax ;; movq %rax, %r11 ;; orq %rcx, %r11 -;; lock cmpxchgb %r11b, (%rdx) -;; jne 0x52 -;; 63: movzbl %al, %eax +;; lock cmpxchgb %r11b, (%rbx) +;; jne 0x44 +;; 55: movzbl %al, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 6c: ud2 +;; 5e: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw_or.wat b/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw_or.wat index 2a32d5ed98e4..0bf71b87b134 100644 --- a/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw_or.wat +++ b/tests/disas/winch/x64/atomic/rmw/or/i32_atomic_rmw_or.wat @@ -10,33 +10,29 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x14, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x78 +;; ja 0x6a ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x2a, %eax -;; movl $0, %ecx -;; andl $3, %ecx -;; cmpl $0, %ecx -;; jne 0x7a -;; 42: movl $0, %ecx +;; movl $0x2a, %ecx +;; movl $0, %edx +;; andl $3, %edx +;; cmpl $0, %edx +;; jne 0x6c +;; 42: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movl (%rdx), %eax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movl (%rbx), %eax ;; movq %rax, %r11 ;; orq %rcx, %r11 -;; lock cmpxchgl %r11d, (%rdx) -;; jne 0x61 -;; 72: addq $0x10, %rsp +;; lock cmpxchgl %r11d, (%rbx) +;; jne 0x53 +;; 64: addq $0x10, %rsp ;; popq %rbp ;; retq -;; 78: ud2 -;; 7a: ud2 +;; 6a: ud2 +;; 6c: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw16_oru.wat b/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw16_oru.wat index 21725c07066f..5827c6208d36 100644 --- a/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw16_oru.wat +++ b/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw16_oru.wat @@ -10,32 +10,30 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x77 +;; ja 0x75 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx -;; andw $1, %cx -;; cmpw $0, %cx -;; jne 0x79 -;; 46: movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx +;; andw $1, %dx +;; cmpw $0, %dx +;; jne 0x77 +;; 46: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movzwq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzwq (%rbx), %rax ;; movq %rax, %r11 ;; orq %rcx, %r11 -;; lock cmpxchgw %r11w, (%rdx) -;; jne 0x5b -;; 6d: movzwq %ax, %rax +;; lock cmpxchgw %r11w, (%rbx) +;; jne 0x59 +;; 6b: movzwq %ax, %rax ;; addq $0x10, %rsp ;; popq %rbp ;; retq +;; 75: ud2 ;; 77: ud2 -;; 79: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw32_oru.wat b/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw32_oru.wat index 63c469f2c91b..2c5583295a73 100644 --- a/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw32_oru.wat +++ b/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw32_oru.wat @@ -10,31 +10,29 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x6e +;; ja 0x6c ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx -;; andl $3, %ecx -;; cmpl $0, %ecx -;; jne 0x70 -;; 44: movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx +;; andl $3, %edx +;; cmpl $0, %edx +;; jne 0x6e +;; 44: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movl (%rdx), %eax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movl (%rbx), %eax ;; movq %rax, %r11 ;; orq %rcx, %r11 -;; lock cmpxchgl %r11d, (%rdx) -;; jne 0x57 -;; 68: addq $0x10, %rsp +;; lock cmpxchgl %r11d, (%rbx) +;; jne 0x55 +;; 66: addq $0x10, %rsp ;; popq %rbp ;; retq +;; 6c: ud2 ;; 6e: ud2 -;; 70: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw8_oru.wat b/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw8_oru.wat index 3685312587b0..3cb50938cf61 100644 --- a/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw8_oru.wat +++ b/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw8_oru.wat @@ -10,27 +10,25 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x63 +;; ja 0x61 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movzbq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzbq (%rbx), %rax ;; movq %rax, %r11 ;; orq %rcx, %r11 -;; lock cmpxchgb %r11b, (%rdx) -;; jne 0x48 -;; 59: movzbq %al, %rax +;; lock cmpxchgb %r11b, (%rbx) +;; jne 0x46 +;; 57: movzbq %al, %rax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 63: ud2 +;; 61: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw_or.wat b/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw_or.wat index af01add2f99c..77dcb53caca4 100644 --- a/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw_or.wat +++ b/tests/disas/winch/x64/atomic/rmw/or/i64_atomic_rmw_or.wat @@ -10,31 +10,29 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x71 +;; ja 0x6f ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx -;; andq $7, %rcx -;; cmpq $0, %rcx -;; jne 0x73 -;; 46: movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx +;; andq $7, %rdx +;; cmpq $0, %rdx +;; jne 0x71 +;; 46: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movq (%rbx), %rax ;; movq %rax, %r11 ;; orq %rcx, %r11 -;; lock cmpxchgq %r11, (%rdx) -;; jne 0x5a -;; 6b: addq $0x10, %rsp +;; lock cmpxchgq %r11, (%rbx) +;; jne 0x58 +;; 69: addq $0x10, %rsp ;; popq %rbp ;; retq +;; 6f: ud2 ;; 71: ud2 -;; 73: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw16_xoru.wat b/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw16_xoru.wat index 960c94ba77ba..1e97e9f5897c 100644 --- a/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw16_xoru.wat +++ b/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw16_xoru.wat @@ -10,34 +10,30 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x14, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x80 +;; ja 0x72 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x2a, %eax -;; movl $0, %ecx -;; andw $1, %cx -;; cmpw $0, %cx -;; jne 0x82 -;; 44: movl $0, %ecx +;; movl $0x2a, %ecx +;; movl $0, %edx +;; andw $1, %dx +;; cmpw $0, %dx +;; jne 0x74 +;; 44: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movzwq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzwq (%rbx), %rax ;; movq %rax, %r11 ;; xorq %rcx, %r11 -;; lock cmpxchgw %r11w, (%rdx) -;; jne 0x65 -;; 77: movzwl %ax, %eax +;; lock cmpxchgw %r11w, (%rbx) +;; jne 0x57 +;; 69: movzwl %ax, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 80: ud2 -;; 82: ud2 +;; 72: ud2 +;; 74: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw8_xoru.wat b/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw8_xoru.wat index 2825964f1b1c..a983aa4ee516 100644 --- a/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw8_xoru.wat +++ b/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw8_xoru.wat @@ -10,29 +10,25 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x14, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x6c +;; ja 0x5e ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x2a, %eax -;; movl $0, %ecx +;; movl $0x2a, %ecx +;; movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movzbq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzbq (%rbx), %rax ;; movq %rax, %r11 ;; xorq %rcx, %r11 -;; lock cmpxchgb %r11b, (%rdx) -;; jne 0x52 -;; 63: movzbl %al, %eax +;; lock cmpxchgb %r11b, (%rbx) +;; jne 0x44 +;; 55: movzbl %al, %eax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 6c: ud2 +;; 5e: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw_xor.wat b/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw_xor.wat index 4aeb301db412..5075a57c95f2 100644 --- a/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw_xor.wat +++ b/tests/disas/winch/x64/atomic/rmw/xor/i32_atomic_rmw_xor.wat @@ -10,33 +10,29 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x14, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x78 +;; ja 0x6a ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movl $0x2a, %eax -;; movl $0, %ecx -;; andl $3, %ecx -;; cmpl $0, %ecx -;; jne 0x7a -;; 42: movl $0, %ecx +;; movl $0x2a, %ecx +;; movl $0, %edx +;; andl $3, %edx +;; cmpl $0, %edx +;; jne 0x6c +;; 42: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; subq $4, %rsp -;; movl %eax, (%rsp) -;; movl (%rsp), %ecx -;; addq $4, %rsp -;; movl (%rdx), %eax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movl (%rbx), %eax ;; movq %rax, %r11 ;; xorq %rcx, %r11 -;; lock cmpxchgl %r11d, (%rdx) -;; jne 0x61 -;; 72: addq $0x10, %rsp +;; lock cmpxchgl %r11d, (%rbx) +;; jne 0x53 +;; 64: addq $0x10, %rsp ;; popq %rbp ;; retq -;; 78: ud2 -;; 7a: ud2 +;; 6a: ud2 +;; 6c: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw16_xoru.wat b/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw16_xoru.wat index bfb623928a87..3d6b9bc38267 100644 --- a/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw16_xoru.wat +++ b/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw16_xoru.wat @@ -10,32 +10,30 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x77 +;; ja 0x75 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx -;; andw $1, %cx -;; cmpw $0, %cx -;; jne 0x79 -;; 46: movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx +;; andw $1, %dx +;; cmpw $0, %dx +;; jne 0x77 +;; 46: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movzwq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzwq (%rbx), %rax ;; movq %rax, %r11 ;; xorq %rcx, %r11 -;; lock cmpxchgw %r11w, (%rdx) -;; jne 0x5b -;; 6d: movzwq %ax, %rax +;; lock cmpxchgw %r11w, (%rbx) +;; jne 0x59 +;; 6b: movzwq %ax, %rax ;; addq $0x10, %rsp ;; popq %rbp ;; retq +;; 75: ud2 ;; 77: ud2 -;; 79: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw32_xoru.wat b/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw32_xoru.wat index da26bfb7bc1b..cad7d0e9fa6d 100644 --- a/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw32_xoru.wat +++ b/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw32_xoru.wat @@ -10,31 +10,29 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x6e +;; ja 0x6c ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx -;; andl $3, %ecx -;; cmpl $0, %ecx -;; jne 0x70 -;; 44: movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx +;; andl $3, %edx +;; cmpl $0, %edx +;; jne 0x6e +;; 44: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movl (%rdx), %eax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movl (%rbx), %eax ;; movq %rax, %r11 ;; xorq %rcx, %r11 -;; lock cmpxchgl %r11d, (%rdx) -;; jne 0x57 -;; 68: addq $0x10, %rsp +;; lock cmpxchgl %r11d, (%rbx) +;; jne 0x55 +;; 66: addq $0x10, %rsp ;; popq %rbp ;; retq +;; 6c: ud2 ;; 6e: ud2 -;; 70: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw8_xoru.wat b/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw8_xoru.wat index 738edc26804a..052f47df1cd3 100644 --- a/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw8_xoru.wat +++ b/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw8_xoru.wat @@ -10,27 +10,25 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x63 +;; ja 0x61 ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movzbq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movzbq (%rbx), %rax ;; movq %rax, %r11 ;; xorq %rcx, %r11 -;; lock cmpxchgb %r11b, (%rdx) -;; jne 0x48 -;; 59: movzbq %al, %rax +;; lock cmpxchgb %r11b, (%rbx) +;; jne 0x46 +;; 57: movzbq %al, %rax ;; addq $0x10, %rsp ;; popq %rbp ;; retq -;; 63: ud2 +;; 61: ud2 diff --git a/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw_xor.wat b/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw_xor.wat index d524762957ac..809972279c97 100644 --- a/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw_xor.wat +++ b/tests/disas/winch/x64/atomic/rmw/xor/i64_atomic_rmw_xor.wat @@ -10,31 +10,29 @@ ;; movq %rsp, %rbp ;; movq 8(%rdi), %r11 ;; movq 0x10(%r11), %r11 -;; addq $0x18, %r11 +;; addq $0x10, %r11 ;; cmpq %rsp, %r11 -;; ja 0x71 +;; ja 0x6f ;; 1c: movq %rdi, %r14 ;; subq $0x10, %rsp ;; movq %rdi, 8(%rsp) ;; movq %rsi, (%rsp) -;; movq $0x2a, %rax -;; movl $0, %ecx -;; andq $7, %rcx -;; cmpq $0, %rcx -;; jne 0x73 -;; 46: movl $0, %ecx +;; movq $0x2a, %rcx +;; movl $0, %edx +;; andq $7, %rdx +;; cmpq $0, %rdx +;; jne 0x71 +;; 46: movl $0, %edx ;; movq 0x58(%r14), %r11 -;; movq (%r11), %rdx -;; addq %rcx, %rdx -;; pushq %rax -;; popq %rcx -;; movq (%rdx), %rax +;; movq (%r11), %rbx +;; addq %rdx, %rbx +;; movq (%rbx), %rax ;; movq %rax, %r11 ;; xorq %rcx, %r11 -;; lock cmpxchgq %r11, (%rdx) -;; jne 0x5a -;; 6b: addq $0x10, %rsp +;; lock cmpxchgq %r11, (%rbx) +;; jne 0x58 +;; 69: addq $0x10, %rsp ;; popq %rbp ;; retq +;; 6f: ud2 ;; 71: ud2 -;; 73: ud2 diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index f2eb2f9764b4..8b67ce39de6b 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -623,265 +623,28 @@ where self.masm.bind(cont) } - /// Emits a series of instructions to bounds check and calculate the address - /// of the given WebAssembly memory. - /// This function returns a register containing the requested address. - /// - /// In essence, when computing the heap address for a WebAssembly load or - /// store instruction the objective is to ensure that such access is safe, - /// but also to perform the least amount of checks, and rely on the system to - /// detect illegal memory accesses where applicable. - /// - /// Winch follows almost the same principles as Cranelift when it comes to - /// bounds checks, for a more detailed explanation refer to - /// prepare_addr in wasmtime-cranelift. - /// - /// Winch implementation differs in that, it defaults to the general case - /// for dynamic heaps rather than optimizing for doing the least amount of - /// work possible at runtime, this is done to align with Winch's principle - /// of doing the least amount of work possible at compile time. For static - /// heaps, Winch does a bit more of work, given that some of the cases that - /// are checked against, can benefit compilation times, like for example, - /// detecting an out of bounds access at compile time. - pub fn emit_compute_heap_address( + fn make_compute_heap_address_context( &mut self, memarg: &MemArg, - access_size: OperandSize, - ) -> Result> { - let ptr_size: OperandSize = self.env.ptr_type().try_into()?; - let enable_spectre_mitigation = self.env.heap_access_spectre_mitigation(); - let add_offset_and_access_size = |offset: ImmOffset, access_size: OperandSize| { - (access_size.bytes() as u64) + (offset.as_u32() as u64) - }; - + ) -> Result { let memory_index = MemoryIndex::from_u32(memarg.memory); - let heap = self.env.resolve_heap(memory_index); - let index = Index::from_typed_reg(self.context.pop_to_reg(self.masm, None)?); - let offset = bounds::ensure_index_and_offset( - self.masm, - index, - memarg.offset, - heap.index_type().try_into()?, - )?; - let offset_with_access_size = add_offset_and_access_size(offset, access_size); - - let can_elide_bounds_check = heap - .memory - .can_elide_bounds_check(self.tunables, self.env.page_size_log2); - - let addr = if offset_with_access_size > heap.memory.maximum_byte_size().unwrap_or(u64::MAX) - { - // Detect at compile time if the access is out of bounds. - // Doing so will put the compiler in an unreachable code state, - // optimizing the work that the compiler has to do until the - // reachability is restored or when reaching the end of the - // function. - - self.emit_fuel_increment()?; - self.masm.trap(TrapCode::HEAP_OUT_OF_BOUNDS)?; - self.context.reachable = false; - None - } else if !can_elide_bounds_check { - // Account for the general case for bounds-checked memories. The - // access is out of bounds if: - // * index + offset + access_size overflows - // OR - // * index + offset + access_size > bound - let bounds = bounds::load_dynamic_heap_bounds::<_>( - &mut self.context, - self.masm, - &heap, - ptr_size, - )?; - - let index_reg = index.as_typed_reg().reg; - // Allocate a temporary register to hold - // index + offset + access_size - // which will serve as the check condition. - let index_offset_and_access_size = self.context.any_gpr(self.masm)?; - - // Move the value of the index to the - // index_offset_and_access_size register to perform the overflow - // check to avoid clobbering the initial index value. - // - // We derive size of the operation from the heap type since: - // - // * This is the first assignment to the - // `index_offset_and_access_size` register - // - // * The memory64 proposal specifies that the index is bound to - // the heap type instead of hardcoding it to 32-bits (i32). - self.masm.mov( - writable!(index_offset_and_access_size), - index_reg.into(), - heap.index_type().try_into()?, - )?; - // Perform - // index = index + offset + access_size, trapping if the - // addition overflows. - // - // We use the target's pointer size rather than depending on the heap - // type since we want to check for overflow; even though the - // offset and access size are guaranteed to be bounded by the heap - // type, when added, if used with the wrong operand size, their - // result could be clamped, resulting in an erroneus overflow - // check. - self.masm.checked_uadd( - writable!(index_offset_and_access_size), - index_offset_and_access_size, - RegImm::i64(offset_with_access_size as i64), - ptr_size, - TrapCode::HEAP_OUT_OF_BOUNDS, - )?; - - let addr = bounds::load_heap_addr_checked( - self.masm, - &mut self.context, - ptr_size, - &heap, - enable_spectre_mitigation, - bounds, - index, - offset, - |masm, bounds, _| { - let bounds_reg = bounds.as_typed_reg().reg; - masm.cmp( - index_offset_and_access_size.into(), - bounds_reg.into(), - // We use the pointer size to keep the bounds - // comparison consistent with the result of the - // overflow check above. - ptr_size, - )?; - Ok(IntCmpKind::GtU) - }, - )?; - self.context.free_reg(bounds.as_typed_reg().reg); - self.context.free_reg(index_offset_and_access_size); - Some(addr) - - // Account for the case in which we can completely elide the bounds - // checks. - // - // This case, makes use of the fact that if a memory access uses - // a 32-bit index, then we be certain that - // - // index <= u32::MAX - // - // Therefore if any 32-bit index access occurs in the region - // represented by - // - // bound + guard_size - (offset + access_size) - // - // We are certain that it's in bounds or that the underlying virtual - // memory subsystem will report an illegal access at runtime. - // - // Note: - // - // * bound - (offset + access_size) cannot wrap, because it's checked - // in the condition above. - // * bound + heap.offset_guard_size is guaranteed to not overflow if - // the heap configuration is correct, given that it's address must - // fit in 64-bits. - // * If the heap type is 32-bits, the offset is at most u32::MAX, so - // no adjustment is needed as part of - // [bounds::ensure_index_and_offset]. - } else if u64::from(u32::MAX) - <= self.tunables.memory_reservation + self.tunables.memory_guard_size - - offset_with_access_size - { - assert!(can_elide_bounds_check); - assert!(heap.index_type() == WasmValType::I32); - let addr = self.context.any_gpr(self.masm)?; - bounds::load_heap_addr_unchecked(self.masm, &heap, index, offset, addr, ptr_size)?; - Some(addr) - - // Account for the all remaining cases, aka. The access is out - // of bounds if: - // - // index > bound - (offset + access_size) - // - // bound - (offset + access_size) cannot wrap, because we already - // checked that (offset + access_size) > bound, above. - } else { - assert!(can_elide_bounds_check); - assert!(heap.index_type() == WasmValType::I32); - let bounds = Bounds::from_u64(self.tunables.memory_reservation); - let addr = bounds::load_heap_addr_checked( - self.masm, - &mut self.context, - ptr_size, - &heap, - enable_spectre_mitigation, - bounds, - index, - offset, - |masm, bounds, index| { - let adjusted_bounds = bounds.as_u64() - offset_with_access_size; - let index_reg = index.as_typed_reg().reg; - masm.cmp( - index_reg, - RegImm::i64(adjusted_bounds as i64), - // Similar to the dynamic heap case, even though the - // offset and access size are bound through the heap - // type, when added they can overflow, resulting in - // an erroneus comparison, therfore we rely on the - // target pointer size. - ptr_size, - )?; - Ok(IntCmpKind::GtU) - }, - )?; - Some(addr) - }; - self.context.free_reg(index.as_typed_reg().reg); - Ok(addr) + Ok(ComputeHeapAddrContext { + ptr_size: self.env.ptr_type().try_into()?, + enable_specte_mitigation: self.env.heap_access_spectre_mitigation(), + heap: self.env.resolve_heap(memory_index), + page_size_log2: self.env.page_size_log2, + emit_fuels: self.make_emit_fuel_context(), + }) } - /// Emit checks to ensure that the address at `memarg` is correctly aligned for `size`. - fn emit_check_align(&mut self, memarg: &MemArg, size: OperandSize) -> Result<()> { - if size.bytes() > 1 { - // Peek addr from top of the stack by popping and pushing. - let addr = *self - .context - .stack - .peek() - .ok_or_else(|| CodeGenError::missing_values_in_stack())?; - let tmp = self.context.any_gpr(self.masm)?; - self.context.move_val_to_reg(&addr, tmp, self.masm)?; - - if memarg.offset != 0 { - self.masm.add( - writable!(tmp), - tmp, - RegImm::Imm(Imm::I64(memarg.offset)), - size, - )?; - } - - self.masm.and( - writable!(tmp), - tmp, - RegImm::Imm(Imm::I32(size.bytes() - 1)), - size, - )?; - - self.masm.cmp(tmp, RegImm::Imm(Imm::i64(0)), size)?; - self.masm.trapif(IntCmpKind::Ne, TRAP_HEAP_MISALIGNED)?; - self.context.free_reg(tmp); + fn make_emit_fuel_context(&self) -> EmitFuelIncrement { + let limits_offset = self.env.vmoffsets.ptr.vmctx_runtime_limits(); + let fuel_offset = self.env.vmoffsets.ptr.vmruntime_limits_fuel_consumed(); + EmitFuelIncrement { + limits_offset, + fuel_offset, } - - Ok(()) - } - - pub fn emit_compute_heap_address_align_checked( - &mut self, - memarg: &MemArg, - access_size: OperandSize, - ) -> Result> { - self.emit_check_align(memarg, access_size)?; - self.emit_compute_heap_address(memarg, access_size) } /// Emit a WebAssembly load. @@ -892,13 +655,24 @@ where kind: LoadKind, op_kind: MemOpKind, ) -> Result<()> { + let mut compute_heap_ctx = self.make_compute_heap_address_context(arg)?; let maybe_addr = match op_kind { - MemOpKind::Atomic => { - self.emit_compute_heap_address_align_checked(&arg, kind.derive_operand_size())? - } - MemOpKind::Normal => { - self.emit_compute_heap_address(&arg, kind.derive_operand_size())? - } + MemOpKind::Atomic => compute_heap_ctx.emit_compute_heap_address_align_checked( + &mut self.context, + self.masm, + arg, + kind.derive_operand_size(), + &self.tunables, + &mut self.fuel_consumed, + )?, + MemOpKind::Normal => compute_heap_ctx.emit_compute_heap_address( + &mut self.context, + self.masm, + arg, + kind.derive_operand_size(), + &self.tunables, + &mut self.fuel_consumed, + )?, }; if let Some(addr) = maybe_addr { @@ -929,9 +703,24 @@ where ) -> Result<()> { let src = self.context.pop_to_reg(self.masm, None)?; + let mut compute_heap_ctx = self.make_compute_heap_address_context(arg)?; let maybe_addr = match op_kind { - MemOpKind::Atomic => self.emit_compute_heap_address_align_checked(&arg, size)?, - MemOpKind::Normal => self.emit_compute_heap_address(&arg, size)?, + MemOpKind::Atomic => compute_heap_ctx.emit_compute_heap_address_align_checked( + &mut self.context, + self.masm, + arg, + size, + &self.tunables, + &mut self.fuel_consumed, + )?, + MemOpKind::Normal => compute_heap_ctx.emit_compute_heap_address( + &mut self.context, + self.masm, + arg, + size, + &self.tunables, + &mut self.fuel_consumed, + )?, }; if let Some(addr) = maybe_addr { @@ -1242,58 +1031,17 @@ where /// Increments the fuel consumed in `VMRuntimeLimits` by flushing /// `self.fuel_consumed` to memory. fn emit_fuel_increment(&mut self) -> Result<()> { - let fuel_at_point = std::mem::replace(&mut self.fuel_consumed, 0); - if fuel_at_point == 0 { - return Ok(()); - } - - let limits_offset = self.env.vmoffsets.ptr.vmctx_runtime_limits(); - let fuel_offset = self.env.vmoffsets.ptr.vmruntime_limits_fuel_consumed(); - let limits_reg = self.context.any_gpr(self.masm)?; - - // Load `VMRuntimeLimits` into the `limits_reg` reg. - self.masm.load_ptr( - self.masm.address_at_vmctx(u32::from(limits_offset))?, - writable!(limits_reg), - )?; + self.make_emit_fuel_context() + .emit(self.masm, &mut self.context, &mut self.fuel_consumed) + } - // Load the fuel consumed at point into the scratch register. - self.masm.load( - self.masm - .address_at_reg(limits_reg, u32::from(fuel_offset))?, - writable!(scratch!(M)), - OperandSize::S64, - )?; - - // Add the fuel consumed at point with the value in the scratch - // register. - self.masm.add( - writable!(scratch!(M)), - scratch!(M), - RegImm::i64(fuel_at_point), - OperandSize::S64, - )?; - - // Store the updated fuel consumed to `VMRuntimeLimits`. - self.masm.store( - scratch!(M).into(), - self.masm - .address_at_reg(limits_reg, u32::from(fuel_offset))?, - OperandSize::S64, - )?; - - self.context.free_reg(limits_reg); - - Ok(()) - } - - /// Hook to handle fuel before visiting an operator. - fn fuel_before_visit_op(&mut self, op: &Operator) -> Result<()> { - if !self.context.reachable { - // `self.fuel_consumed` must be correctly flushed to memory when - // entering an unreachable state. - ensure!(self.fuel_consumed == 0, CodeGenError::illegal_fuel_state()) - } + /// Hook to handle fuel before visiting an operator. + fn fuel_before_visit_op(&mut self, op: &Operator) -> Result<()> { + if !self.context.reachable { + // `self.fuel_consumed` must be correctly flushed to memory when + // entering an unreachable state. + ensure!(self.fuel_consumed == 0, CodeGenError::illegal_fuel_state()) + } // Generally, most instructions require 1 fuel unit. // @@ -1370,17 +1118,30 @@ where size: OperandSize, extend: Option>, ) -> Result<()> { - // We need to pop-push the operand to compute the address before passing control over to - // masm, because some architectures may have specific requirements for the registers used - // in some atomic operations. - let operand = self.context.pop_to_reg(self.masm, None)?; - if let Some(addr) = self.emit_compute_heap_address_align_checked(arg, size)? { - let src = self.masm.address_at_reg(addr, 0)?; - self.context.stack.push(operand.into()); - self.masm - .atomic_rmw(&mut self.context, src, size, op, UNTRUSTED_FLAGS, extend)?; - self.context.free_reg(addr); - } + let mut compute_heap_ctx = self.make_compute_heap_address_context(arg)?; + let compute_addr = + |masm: &mut M, ctx: &mut CodeGenContext| -> Result> { + match compute_heap_ctx.emit_compute_heap_address_align_checked( + ctx, + masm, + arg, + size, + &self.tunables, + &mut self.fuel_consumed, + )? { + Some(addr) => Ok(Some(addr)), + None => Ok(None), + } + }; + + self.masm.atomic_rmw( + &mut self.context, + compute_addr, + size, + op, + UNTRUSTED_FLAGS, + extend, + )?; Ok(()) } @@ -1400,21 +1161,30 @@ where // the control to masm. The implementer of `atomic_cas` can expect to find `expected` and // `replacement` at the top the context's stack. - // pop the args - let replacement = self.context.pop_to_reg(self.masm, None)?; - let expected = self.context.pop_to_reg(self.masm, None)?; - - if let Some(addr) = self.emit_compute_heap_address_align_checked(arg, size)? { - // push back the args - self.context.stack.push(expected.into()); - self.context.stack.push(replacement.into()); + let mut compute_heap_ctx = self.make_compute_heap_address_context(arg)?; + let compute_addr = + |masm: &mut M, ctx: &mut CodeGenContext| -> Result> { + match compute_heap_ctx.emit_compute_heap_address_align_checked( + ctx, + masm, + arg, + size, + &self.tunables, + &mut self.fuel_consumed, + )? { + Some(addr) => Ok(Some(addr)), + None => Ok(None), + } + }; - let src = self.masm.address_at_reg(addr, 0)?; - self.masm - .atomic_cas(&mut self.context, src, size, UNTRUSTED_FLAGS, extend)?; + self.masm.atomic_cas( + &mut self.context, + compute_addr, + size, + UNTRUSTED_FLAGS, + extend, + )?; - self.context.free_reg(addr); - } Ok(()) } } @@ -1426,3 +1196,337 @@ pub fn control_index(depth: u32, control_length: usize) -> Result { .checked_sub(depth as usize) .ok_or_else(|| anyhow!(CodeGenError::control_frame_expected())) } + +struct ComputeHeapAddrContext { + ptr_size: OperandSize, + enable_specte_mitigation: bool, + heap: HeapData, + page_size_log2: u8, + emit_fuels: EmitFuelIncrement, +} + +impl ComputeHeapAddrContext { + /// Emits a series of instructions to bounds check and calculate the address + /// of the given WebAssembly memory. + /// This function returns a register containing the requested address. + /// + /// In essence, when computing the heap address for a WebAssembly load or + /// store instruction the objective is to ensure that such access is safe, + /// but also to perform the least amount of checks, and rely on the system to + /// detect illegal memory accesses where applicable. + /// + /// Winch follows almost the same principles as Cranelift when it comes to + /// bounds checks, for a more detailed explanation refer to + /// prepare_addr in wasmtime-cranelift. + /// + /// Winch implementation differs in that, it defaults to the general case + /// for dynamic heaps rather than optimizing for doing the least amount of + /// work possible at runtime, this is done to align with Winch's principle + /// of doing the least amount of work possible at compile time. For static + /// heaps, Winch does a bit more of work, given that some of the cases that + /// are checked against, can benefit compilation times, like for example, + /// detecting an out of bounds access at compile time. + pub fn emit_compute_heap_address( + &self, + context: &mut CodeGenContext, + masm: &mut M, + memarg: &MemArg, + access_size: OperandSize, + tunables: &Tunables, + fuel_consumed: &mut i64, + ) -> Result> { + let ptr_size: OperandSize = self.ptr_size; + let enable_spectre_mitigation = self.enable_specte_mitigation; + let add_offset_and_access_size = |offset: ImmOffset, access_size: OperandSize| { + (access_size.bytes() as u64) + (offset.as_u32() as u64) + }; + + let index = Index::from_typed_reg(context.pop_to_reg(masm, None)?); + let offset = bounds::ensure_index_and_offset( + masm, + index, + memarg.offset, + self.heap.index_type().try_into()?, + )?; + let offset_with_access_size = add_offset_and_access_size(offset, access_size); + + let can_elide_bounds_check = self + .heap + .memory + .can_elide_bounds_check(tunables, self.page_size_log2); + + let addr = if offset_with_access_size + > self.heap.memory.maximum_byte_size().unwrap_or(u64::MAX) + { + // Detect at compile time if the access is out of bounds. + // Doing so will put the compiler in an unreachable code state, + // optimizing the work that the compiler has to do until the + // reachability is restored or when reaching the end of the + // function. + + self.emit_fuels.emit(masm, context, fuel_consumed)?; + + masm.trap(TrapCode::HEAP_OUT_OF_BOUNDS)?; + context.reachable = false; + None + } else if !can_elide_bounds_check { + // Account for the general case for bounds-checked memories. The + // access is out of bounds if: + // * index + offset + access_size overflows + // OR + // * index + offset + access_size > bound + let bounds = + bounds::load_dynamic_heap_bounds::<_>(context, masm, &self.heap, ptr_size)?; + + let index_reg = index.as_typed_reg().reg; + // Allocate a temporary register to hold + // index + offset + access_size + // which will serve as the check condition. + let index_offset_and_access_size = context.any_gpr(masm)?; + + // Move the value of the index to the + // index_offset_and_access_size register to perform the overflow + // check to avoid clobbering the initial index value. + // + // We derive size of the operation from the heap type since: + // + // * This is the first assignment to the + // `index_offset_and_access_size` register + // + // * The memory64 proposal specifies that the index is bound to + // the heap type instead of hardcoding it to 32-bits (i32). + masm.mov( + writable!(index_offset_and_access_size), + index_reg.into(), + self.heap.index_type().try_into()?, + )?; + // Perform + // index = index + offset + access_size, trapping if the + // addition overflows. + // + // We use the target's pointer size rather than depending on the heap + // type since we want to check for overflow; even though the + // offset and access size are guaranteed to be bounded by the heap + // type, when added, if used with the wrong operand size, their + // result could be clamped, resulting in an erroneus overflow + // check. + masm.checked_uadd( + writable!(index_offset_and_access_size), + index_offset_and_access_size, + RegImm::i64(offset_with_access_size as i64), + ptr_size, + TrapCode::HEAP_OUT_OF_BOUNDS, + )?; + + let addr = bounds::load_heap_addr_checked( + masm, + context, + ptr_size, + &self.heap, + enable_spectre_mitigation, + bounds, + index, + offset, + |masm, bounds, _| { + let bounds_reg = bounds.as_typed_reg().reg; + masm.cmp( + index_offset_and_access_size.into(), + bounds_reg.into(), + // We use the pointer size to keep the bounds + // comparison consistent with the result of the + // overflow check above. + ptr_size, + )?; + Ok(IntCmpKind::GtU) + }, + )?; + context.free_reg(bounds.as_typed_reg().reg); + context.free_reg(index_offset_and_access_size); + Some(addr) + + // Account for the case in which we can completely elide the bounds + // checks. + // + // This case, makes use of the fact that if a memory access uses + // a 32-bit index, then we be certain that + // + // index <= u32::MAX + // + // Therefore if any 32-bit index access occurs in the region + // represented by + // + // bound + guard_size - (offset + access_size) + // + // We are certain that it's in bounds or that the underlying virtual + // memory subsystem will report an illegal access at runtime. + // + // Note: + // + // * bound - (offset + access_size) cannot wrap, because it's checked + // in the condition above. + // * bound + heap.offset_guard_size is guaranteed to not overflow if + // the heap configuration is correct, given that it's address must + // fit in 64-bits. + // * If the heap type is 32-bits, the offset is at most u32::MAX, so + // no adjustment is needed as part of + // [bounds::ensure_index_and_offset]. + } else if u64::from(u32::MAX) + <= tunables.memory_reservation + tunables.memory_guard_size - offset_with_access_size + { + assert!(can_elide_bounds_check); + assert!(self.heap.index_type() == WasmValType::I32); + let addr = context.any_gpr(masm)?; + bounds::load_heap_addr_unchecked(masm, &self.heap, index, offset, addr, ptr_size)?; + Some(addr) + + // Account for the all remaining cases, aka. The access is out + // of bounds if: + // + // index > bound - (offset + access_size) + // + // bound - (offset + access_size) cannot wrap, because we already + // checked that (offset + access_size) > bound, above. + } else { + assert!(can_elide_bounds_check); + assert!(self.heap.index_type() == WasmValType::I32); + let bounds = Bounds::from_u64(tunables.memory_reservation); + let addr = bounds::load_heap_addr_checked( + masm, + context, + ptr_size, + &self.heap, + enable_spectre_mitigation, + bounds, + index, + offset, + |masm, bounds, index| { + let adjusted_bounds = bounds.as_u64() - offset_with_access_size; + let index_reg = index.as_typed_reg().reg; + masm.cmp( + index_reg, + RegImm::i64(adjusted_bounds as i64), + // Similar to the dynamic heap case, even though the + // offset and access size are bound through the heap + // type, when added they can overflow, resulting in + // an erroneus comparison, therfore we rely on the + // target pointer size. + ptr_size, + )?; + Ok(IntCmpKind::GtU) + }, + )?; + Some(addr) + }; + + context.free_reg(index.as_typed_reg().reg); + Ok(addr) + } + + pub fn emit_compute_heap_address_align_checked( + &mut self, + context: &mut CodeGenContext, + masm: &mut M, + memarg: &MemArg, + access_size: OperandSize, + tunables: &Tunables, + fuel_consumed: &mut i64, + ) -> Result> { + self.emit_check_align(context, masm, memarg, access_size)?; + self.emit_compute_heap_address(context, masm, memarg, access_size, tunables, fuel_consumed) + } + + /// Emit checks to ensure that the address at `memarg` is correctly aligned for `size`. + fn emit_check_align( + &mut self, + context: &mut CodeGenContext, + masm: &mut M, + memarg: &MemArg, + size: OperandSize, + ) -> Result<()> { + if size.bytes() > 1 { + // Peek addr from top of the stack by popping and pushing. + let addr = *context + .stack + .peek() + .ok_or_else(|| CodeGenError::missing_values_in_stack())?; + let tmp = context.any_gpr(masm)?; + context.move_val_to_reg(&addr, tmp, masm)?; + + if memarg.offset != 0 { + masm.add( + writable!(tmp), + tmp, + RegImm::Imm(Imm::I64(memarg.offset)), + size, + )?; + } + + masm.and( + writable!(tmp), + tmp, + RegImm::Imm(Imm::I32(size.bytes() - 1)), + size, + )?; + + masm.cmp(tmp, RegImm::Imm(Imm::i64(0)), size)?; + masm.trapif(IntCmpKind::Ne, TRAP_HEAP_MISALIGNED)?; + context.free_reg(tmp); + } + + Ok(()) + } +} + +struct EmitFuelIncrement { + limits_offset: u8, + fuel_offset: u8, +} + +impl EmitFuelIncrement { + fn emit( + &self, + masm: &mut M, + context: &mut CodeGenContext, + fuel_consumed: &mut i64, + ) -> Result<()> { + let fuel_at_point = std::mem::replace(fuel_consumed, 0); + if fuel_at_point == 0 { + return Ok(()); + } + + let limits_reg = context.any_gpr(masm)?; + + // Load `VMRuntimeLimits` into the `limits_reg` reg. + masm.load_ptr( + masm.address_at_vmctx(u32::from(self.limits_offset))?, + writable!(limits_reg), + )?; + + // Load the fuel consumed at point into the scratch register. + masm.load( + masm.address_at_reg(limits_reg, u32::from(self.fuel_offset))?, + writable!(scratch!(M)), + OperandSize::S64, + )?; + + // Add the fuel consumed at point with the value in the scratch + // register. + masm.add( + writable!(scratch!(M)), + scratch!(M), + RegImm::i64(fuel_at_point), + OperandSize::S64, + )?; + + // Store the updated fuel consumed to `VMRuntimeLimits`. + masm.store( + scratch!(M).into(), + masm.address_at_reg(limits_reg, u32::from(self.fuel_offset))?, + OperandSize::S64, + )?; + + context.free_reg(limits_reg); + + Ok(()) + } +} diff --git a/winch/codegen/src/isa/aarch64/masm.rs b/winch/codegen/src/isa/aarch64/masm.rs index 65b0a4eefddf..c78fdde3e243 100644 --- a/winch/codegen/src/isa/aarch64/masm.rs +++ b/winch/codegen/src/isa/aarch64/masm.rs @@ -910,7 +910,7 @@ impl Masm for MacroAssembler { fn atomic_rmw( &mut self, _context: &mut CodeGenContext, - _addr: Self::Address, + _compute_addr: impl FnOnce(&mut Self, &mut CodeGenContext) -> Result>, _size: OperandSize, _op: RmwOp, _flags: MemFlags, @@ -932,7 +932,7 @@ impl Masm for MacroAssembler { fn atomic_cas( &mut self, _context: &mut CodeGenContext, - _addr: Self::Address, + _compute_addr: impl FnOnce(&mut Self, &mut CodeGenContext) -> Result>, _size: OperandSize, _flags: MemFlags, _extend: Option>, diff --git a/winch/codegen/src/isa/x64/masm.rs b/winch/codegen/src/isa/x64/masm.rs index 2250d7a256d6..9e91a7918ad1 100644 --- a/winch/codegen/src/isa/x64/masm.rs +++ b/winch/codegen/src/isa/x64/masm.rs @@ -1429,65 +1429,74 @@ impl Masm for MacroAssembler { fn atomic_rmw( &mut self, context: &mut CodeGenContext, - addr: Self::Address, + compute_addr: impl FnOnce(&mut Self, &mut CodeGenContext) -> Result>, size: OperandSize, op: RmwOp, flags: MemFlags, extend: Option>, ) -> Result<()> { - let res = match op { - RmwOp::Add => { - let operand = context.pop_to_reg(self, None)?; - self.asm - .lock_xadd(addr, operand.reg, writable!(operand.reg), size, flags); - operand.reg - } - RmwOp::Sub => { - let operand = context.pop_to_reg(self, None)?; - self.asm.neg(operand.reg, writable!(operand.reg), size); - self.asm - .lock_xadd(addr, operand.reg, writable!(operand.reg), size, flags); - operand.reg - } - RmwOp::Xchg => { - let operand = context.pop_to_reg(self, None)?; - self.asm - .xchg(addr, operand.reg, writable!(operand.reg), size, flags); - operand.reg - } - RmwOp::And | RmwOp::Or | RmwOp::Xor => { - let op = match op { - RmwOp::And => AtomicRmwSeqOp::And, - RmwOp::Or => AtomicRmwSeqOp::Or, - RmwOp::Xor => AtomicRmwSeqOp::Xor, - _ => unreachable!( - "invalid op for atomic_rmw_seq, should be one of `or`, `and` or `xor`" - ), - }; - let dst = context.reg(regs::rax(), self)?; - let operand = context.pop_to_reg(self, None)?; + // `atomic_rmw_seq` requires the rax register, reserve it now + if matches!(op, RmwOp::And | RmwOp::Or | RmwOp::Xor) { + context.reg(regs::rax(), self)?; + } - self.asm - .atomic_rmw_seq(addr, operand.reg, writable!(dst), size, flags, op); + let operand = context.pop_to_reg(self, None)?; - context.free_reg(operand.reg); - dst - } - }; + if let Some(addr) = compute_addr(self, context)? { + let src = self.address_at_reg(addr, 0)?; + let res = match op { + RmwOp::Add => { + self.asm + .lock_xadd(src, operand.reg, writable!(operand.reg), size, flags); + operand.reg + } + RmwOp::Sub => { + self.asm.neg(operand.reg, writable!(operand.reg), size); + self.asm + .lock_xadd(src, operand.reg, writable!(operand.reg), size, flags); + operand.reg + } + RmwOp::Xchg => { + self.asm + .xchg(src, operand.reg, writable!(operand.reg), size, flags); + operand.reg + } + RmwOp::And | RmwOp::Or | RmwOp::Xor => { + let op = match op { + RmwOp::And => AtomicRmwSeqOp::And, + RmwOp::Or => AtomicRmwSeqOp::Or, + RmwOp::Xor => AtomicRmwSeqOp::Xor, + _ => unreachable!( + "invalid op for atomic_rmw_seq, should be one of `or`, `and` or `xor`" + ), + }; + + // we have already reserved rax is already reserved + let dst = regs::rax(); - let dst_ty = match extend { - Some(ext) => { - // We don't need to zero-extend from 32 to 64bits. - if !(ext.from_bits() == 32 && ext.to_bits() == 64) { - self.asm.movzx_rr(res, writable!(res), ext.into()); + self.asm + .atomic_rmw_seq(src, operand.reg, writable!(dst), size, flags, op); + + context.free_reg(operand.reg); + dst } + }; - WasmValType::int_from_bits(ext.to_bits()) - } - None => WasmValType::int_from_bits(size.num_bits()), - }; + let dst_ty = match extend { + Some(ext) => { + // We don't need to zero-extend from 32 to 64bits. + if !(ext.from_bits() == 32 && ext.to_bits() == 64) { + self.asm.movzx_rr(res, writable!(res), ext.into()); + } - context.stack.push(TypedReg::new(dst_ty, res).into()); + WasmValType::int_from_bits(ext.to_bits()) + } + None => WasmValType::int_from_bits(size.num_bits()), + }; + + context.stack.push(TypedReg::new(dst_ty, res).into()); + context.free_reg(addr); + } Ok(()) } @@ -1543,7 +1552,7 @@ impl Masm for MacroAssembler { fn atomic_cas( &mut self, context: &mut CodeGenContext, - addr: Self::Address, + compute_addr: impl FnOnce(&mut Self, &mut CodeGenContext) -> Result>, size: OperandSize, flags: MemFlags, extend: Option>, @@ -1551,32 +1560,35 @@ impl Masm for MacroAssembler { // `cmpxchg` expects `expected` to be in the `*a*` register. // reserve rax for the expected argument. let rax = context.reg(regs::rax(), self)?; + if let Some(addr) = compute_addr(self, context)? { + let replacement = context.pop_to_reg(self, None)?; + let src = self.address_at_reg(addr, 0)?; - let replacement = context.pop_to_reg(self, None)?; - - // mark `rax` as allocatable again. - context.free_reg(rax); - let expected = context.pop_to_reg(self, Some(regs::rax()))?; - - self.asm.cmpxchg( - addr, - expected.reg, - replacement.reg, - writable!(expected.reg), - size, - flags, - ); + // mark `rax` as allocatable again. + context.free_reg(rax); + let expected = context.pop_to_reg(self, Some(regs::rax()))?; - if let Some(extend) = extend { - // We don't need to zero-extend from 32 to 64bits. - if !(extend.from_bits() == 32 && extend.to_bits() == 64) { - self.asm - .movzx_rr(expected.reg.into(), writable!(expected.reg.into()), extend); + self.asm.cmpxchg( + src, + expected.reg, + replacement.reg, + writable!(expected.reg), + size, + flags, + ); + + if let Some(extend) = extend { + // We don't need to zero-extend from 32 to 64bits. + if !(extend.from_bits() == 32 && extend.to_bits() == 64) { + self.asm + .movzx_rr(expected.reg.into(), writable!(expected.reg.into()), extend); + } } - } - context.stack.push(expected.into()); - context.free_reg(replacement); + context.stack.push(expected.into()); + context.free_reg(replacement); + context.free_reg(addr); + } Ok(()) } diff --git a/winch/codegen/src/masm.rs b/winch/codegen/src/masm.rs index 7a96248eab9a..834e3378caea 100644 --- a/winch/codegen/src/masm.rs +++ b/winch/codegen/src/masm.rs @@ -1412,7 +1412,7 @@ pub(crate) trait MacroAssembler { fn atomic_rmw( &mut self, context: &mut CodeGenContext, - addr: Self::Address, + compute_addr: impl FnOnce(&mut Self, &mut CodeGenContext) -> Result>, size: OperandSize, op: RmwOp, flags: MemFlags, @@ -1438,7 +1438,7 @@ pub(crate) trait MacroAssembler { fn atomic_cas( &mut self, context: &mut CodeGenContext, - addr: Self::Address, + compute_addr: impl FnOnce(&mut Self, &mut CodeGenContext) -> Result>, size: OperandSize, flags: MemFlags, extend: Option>,