Skip to content

Commit

Permalink
Init
Browse files Browse the repository at this point in the history
  • Loading branch information
@robehn
robehn committed Nov 21, 2024
1 parent 93aa7e2 commit dbdca0c
Show file tree
Hide file tree
Showing 2 changed files with 68 additions and 52 deletions.
115 changes: 66 additions & 49 deletions src/hotspot/cpu/riscv/macroAssembler_riscv.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3313,14 +3313,11 @@ void MacroAssembler::store_conditional(Register dst,
}


void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected,
Register new_val,
void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val,
enum operand_size size,
Register tmp1, Register tmp2, Register tmp3) {
Register shift, Register mask, Register aligned_addr) {
assert(size == int8 || size == int16, "unsupported operand size");

Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3;

andi(shift, addr, 3);
slli(shift, shift, 3);

Expand All @@ -3335,8 +3332,6 @@ void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expecte
}
sll(mask, mask, shift);

notr(not_mask, mask);

sll(expected, expected, shift);
andr(expected, expected, mask);

Expand All @@ -3353,35 +3348,46 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result, bool result_as_bool,
Register tmp1, Register tmp2, Register tmp3) {
Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0;
assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp);
cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3);
assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1);

Label retry, fail, done;
Register scratch0 = t0, aligned_addr = t1;
Register shift = tmp1, mask = tmp2, scratch1 = tmp3;

cmpxchg_narrow_value_helper(addr, expected, new_val, size, shift, mask, aligned_addr);

bind(retry);
Label retry, fail, done;

if (UseZacas) {
lw(old, aligned_addr);
lw(result, aligned_addr);

bind(retry); // amocas loads the current value into result
notr(scratch1, mask);

// if old & mask != expected
andr(tmp, old, mask);
bne(tmp, expected, fail);
andr(scratch0, result, scratch1); // scratch0 = word - cas bits
orr(scratch1, expected, scratch0); // scratch1 = non-cas bits + cas bits
bne(result, scratch1, fail); // cas bits differ, cas failed

andr(tmp, old, not_mask);
orr(tmp, tmp, new_val);
// result is the same as expected, use as expected value.

atomic_cas(old, tmp, aligned_addr, operand_size::int32, acquire, release);
bne(tmp, old, retry);
// scratch0 is still = word - cas bits
// Or in the new value to create complete new value.
orr(scratch0, scratch0, new_val);

mv(scratch1, result); // save our expected value
atomic_cas(result, scratch0, aligned_addr, operand_size::int32, acquire, release);
bne(scratch1, result, retry);
} else {
lr_w(old, aligned_addr, acquire);
andr(tmp, old, mask);
bne(tmp, expected, fail);
notr(scratch1, mask);
bind(retry);

lr_w(result, aligned_addr, acquire);
andr(scratch0, result, mask);
bne(scratch0, expected, fail);

andr(tmp, old, not_mask);
orr(tmp, tmp, new_val);
sc_w(tmp, tmp, aligned_addr, release);
bnez(tmp, retry);
andr(scratch0, result, scratch1); // scratch1 is ~mask
orr(scratch0, scratch0, new_val);
sc_w(scratch0, scratch0, aligned_addr, release);
bnez(scratch0, retry);
}

if (result_as_bool) {
Expand All @@ -3393,10 +3399,10 @@ void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,

bind(done);
} else {
andr(tmp, old, mask);

bind(fail);
srl(result, tmp, shift);

andr(scratch0, result, mask);
srl(result, scratch0, shift);

if (size == int8) {
sign_extend(result, result, 8);
Expand All @@ -3416,33 +3422,44 @@ void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result,
Register tmp1, Register tmp2, Register tmp3) {
Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0;
assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp);
cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3);
assert_different_registers(addr, expected, new_val, result, tmp1, tmp2, tmp3, t0, t1);

Register scratch0 = t0, aligned_addr = t1;
Register shift = tmp1, mask = tmp2, scratch1 = tmp3;

cmpxchg_narrow_value_helper(addr, expected, new_val, size, shift, mask, aligned_addr);

Label fail, done;

if (UseZacas) {
lw(old, aligned_addr);
lw(result, aligned_addr);

// if old & mask != expected
andr(tmp, old, mask);
bne(tmp, expected, fail);
notr(scratch1, mask);

andr(tmp, old, not_mask);
orr(tmp, tmp, new_val);
andr(scratch0, result, scratch1); // scratch0 = word - cas bits
orr(scratch1, expected, scratch0); // scratch1 = non-cas bits + cas bits
bne(result, scratch1, fail); // cas bits differ, cas failed

atomic_cas(tmp, new_val, addr, operand_size::int32, acquire, release);
bne(tmp, old, fail);
// result is the same as expected, use as expected value.

// scratch0 is still = word - cas bits
// Or in the new value to create complete new value.
orr(scratch0, scratch0, new_val);

mv(scratch1, result); // save our expected value
atomic_cas(result, scratch0, aligned_addr, operand_size::int32, acquire, release);
bne(scratch1, result, fail); // This weak, so just bail-out.
} else {
lr_w(old, aligned_addr, acquire);
andr(tmp, old, mask);
bne(tmp, expected, fail);

andr(tmp, old, not_mask);
orr(tmp, tmp, new_val);
sc_w(tmp, tmp, aligned_addr, release);
bnez(tmp, fail);
notr(scratch1, mask);

lr_w(result, aligned_addr, acquire);
andr(scratch0, result, mask);
bne(scratch0, expected, fail);

andr(scratch0, result, scratch1); // scratch1 is ~mask
orr(scratch0, scratch0, new_val);
sc_w(scratch0, scratch0, aligned_addr, release);
bnez(scratch0, fail);
}

// Success
Expand Down
5 changes: 2 additions & 3 deletions src/hotspot/cpu/riscv/macroAssembler_riscv.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1146,10 +1146,9 @@ class MacroAssembler: public Assembler {
enum operand_size size,
Assembler::Aqrl acquire, Assembler::Aqrl release,
Register result);
void cmpxchg_narrow_value_helper(Register addr, Register expected,
Register new_val,
void cmpxchg_narrow_value_helper(Register addr, Register expected, Register new_val,
enum operand_size size,
Register tmp1, Register tmp2, Register tmp3);
Register shift, Register mask, Register aligned_addr);
void cmpxchg_narrow_value(Register addr, Register expected,
Register new_val,
enum operand_size size,
Expand Down

0 comments on commit dbdca0c

Please sign in to comment.