From cd3532109c5fac197b6e0951a353ecbeb22bdc04 Mon Sep 17 00:00:00 2001 From: Robbin Ehn Date: Thu, 21 Mar 2024 14:57:44 +0100 Subject: [PATCH] revert, only short --- .../cpu/riscv/jvmciCodeInstaller_riscv.cpp | 1 - .../cpu/riscv/macroAssembler_riscv.cpp | 22 +++- src/hotspot/cpu/riscv/nativeInst_riscv.cpp | 117 ++++++++++++++++-- src/hotspot/cpu/riscv/nativeInst_riscv.hpp | 41 ++++-- src/hotspot/share/code/nmethod.cpp | 4 +- 5 files changed, 160 insertions(+), 25 deletions(-) diff --git a/src/hotspot/cpu/riscv/jvmciCodeInstaller_riscv.cpp b/src/hotspot/cpu/riscv/jvmciCodeInstaller_riscv.cpp index 784b29b5f7493..a7160bd724128 100644 --- a/src/hotspot/cpu/riscv/jvmciCodeInstaller_riscv.cpp +++ b/src/hotspot/cpu/riscv/jvmciCodeInstaller_riscv.cpp @@ -82,7 +82,6 @@ void CodeInstaller::pd_patch_DataSectionReference(int pc_offset, int data_offset void CodeInstaller::pd_relocate_ForeignCall(NativeInstruction* inst, jlong foreign_call_destination, JVMCI_TRAPS) { address pc = (address) inst; if (inst->is_jal()) { - assert(false, "Bad"); NativeCall* call = nativeCall_at(pc); call->set_destination((address) foreign_call_destination); _instructions->relocate(call->instruction_address(), runtime_call_Relocation::spec()); diff --git a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp index 6955cfcfc9360..e2e81eed86417 100644 --- a/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp +++ b/src/hotspot/cpu/riscv/macroAssembler_riscv.cpp @@ -3547,7 +3547,8 @@ address MacroAssembler::trampoline_call(Address entry) { } #endif relocate(entry.rspec(), [&] { - load_link(target, t0); +// load_link(target, t0); + jump_link(target, t0); }); postcond(pc() != badAddress); @@ -3624,6 +3625,21 @@ address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() + insts_call_instruction_offset); const int stub_start_offset = offset(); + relocate(rh, [&] { + // Now, create the trampoline stub's code: + // - load the call + // - call + Label target; + ld(t0, target); // auipc + ld + jr(t0); // jalr + bind(target); + assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset, + "should be"); + assert(offset() % wordSize == 0, "bad alignment"); + emit_int64((int64_t)dest); + }); + +/* relocate(rh, [&] { assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset, "%ld - %ld == %ld : should be", (long)offset(), (long)stub_start_offset, (long)NativeCallTrampolineStub::data_offset); @@ -3633,6 +3649,7 @@ address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, emit_int32((int32_t)0xabababababu); emit_int32((int32_t)0xdcdcdcdcdcu); }); +*/ const address stub_start_addr = addr_at(stub_start_offset); @@ -3644,7 +3661,8 @@ address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, int MacroAssembler::max_trampoline_stub_size() { // Max stub size: alignment nop, TrampolineStub. - return 2 * wordSize; + // return 2 * wordSize; + return NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size; } int MacroAssembler::static_call_stub_size() { diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp index 8ac16e2877459..6211fe4b28451 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.cpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.cpp @@ -85,6 +85,10 @@ bool NativeInstruction::is_load_pc_relative_at(address instr) { is_ld_at(instr + instruction_size) && // ld check_load_pc_relative_data_dependency(instr); } + +bool NativeInstruction::is_call_at(address addr) { + return NativeCall::is_at(addr); +} bool NativeInstruction::is_movptr_at(address instr) { return is_lui_at(instr) && // Lui @@ -127,14 +131,36 @@ bool NativeInstruction::is_li64_at(address instr) { address NativeShortCall::destination() const { address addr = (address)this; - assert(NativeShortCall::is_at(addr), "unexpected code at call site"); assert(NativeInstruction::is_jal_at(instruction_address()), "inst must be jal."); address destination = MacroAssembler::target_addr_for_insn(instruction_address()); + + // Do we use a trampoline stub for this call? + CodeBlob* cb = CodeCache::find_blob(addr); + assert(cb && cb->is_nmethod(), "sanity"); + nmethod *nm = (nmethod *)cb; + if (nm != nullptr && nm->stub_contains(destination) && is_NativeCallTrampolineStub_at(destination)) { + // Yes we do, so get the destination from the trampoline stub. + const address trampoline_stub_addr = destination; + destination = nativeCallTrampolineStub_at(trampoline_stub_addr)->destination(); + } + return destination; } void NativeShortCall::set_destination(address dest) { assert(NativeShortCall::is_at((address)this), "unexpected code at call site"); + assert(is_jal(), "Should be jal instruction!"); + intptr_t offset = (intptr_t)(dest - instruction_address()); + assert((offset & 0x1) == 0, "bad alignment"); + assert(Assembler::is_simm21(offset), "encoding constraint"); + unsigned int insn = 0b1101111; // jal + address pInsn = (address)(&insn); + Assembler::patch(pInsn, 31, 31, (offset >> 20) & 0x1); + Assembler::patch(pInsn, 30, 21, (offset >> 1) & 0x3ff); + Assembler::patch(pInsn, 20, 20, (offset >> 11) & 0x1); + Assembler::patch(pInsn, 19, 12, (offset >> 12) & 0xff); + Assembler::patch(pInsn, 11, 7, ra->encoding()); // Rd must be x1, need ra + set_int_at(0, insn); } void NativeShortCall::verify() { @@ -147,20 +173,50 @@ void NativeShortCall::print() { } void NativeShortCall::set_destination_mt_safe(address dest, bool assert_lock) { - assert(NativeShortCall::is_at((address)this), "unexpected code at call site"); assert(!assert_lock || (Patching_lock->is_locked() || SafepointSynchronize::is_at_safepoint()) || CompiledICLocker::is_safe(addr_at(0)), "concurrent code patching"); address addr_call = addr_at(0); + assert(NativeCall::is_call_at(addr_call), "unexpected code at call site"); + + // Patch the constant in the call's trampoline stub. + address trampoline_stub_addr = get_trampoline(); + if (trampoline_stub_addr != nullptr) { + assert (!is_NativeCallTrampolineStub_at(dest), "chained trampolines"); + nativeCallTrampolineStub_at(trampoline_stub_addr)->set_destination(dest); + } + // Patch the call. - assert(Assembler::reachable_from_branch_at(addr_call, dest), "we need a trampoline"); - set_destination(dest); + if (Assembler::reachable_from_branch_at(addr_call, dest)) { + set_destination(dest); + } else { + assert (trampoline_stub_addr != nullptr, "we need a trampoline"); + set_destination(trampoline_stub_addr); + } ICache::invalidate_range(addr_call, instruction_size); } +address NativeShortCall::get_trampoline() { + address call_addr = addr_at(0); + + CodeBlob *code = CodeCache::find_blob(call_addr); + assert(code != nullptr, "Could not find the containing code blob"); + + address jal_destination = MacroAssembler::pd_call_destination(call_addr); + if (code != nullptr && code->contains(jal_destination) && is_NativeCallTrampolineStub_at(jal_destination)) { + return jal_destination; + } + + if (code != nullptr && code->is_nmethod()) { + return trampoline_stub_Relocation::get_trampoline_for(call_addr, (nmethod*)code); + } + + return nullptr; +} + NativeShortCall* NativeShortCall::at(address addr) { assert_cond(addr != nullptr); assert(NativeShortCall::is_at(addr), "unexpected code at call site"); @@ -179,8 +235,7 @@ bool NativeShortCall::is_at(address addr) { } bool NativeShortCall::is_call_before(address return_address) { - assert(NativeShortCall::is_at(return_address - Assembler::instruction_size), "unexpected code at call site"); - return false; + return NativeShortCall::is_at(return_address - Assembler::instruction_size); } void NativeShortCall::insert(address code_pos, address entry) { @@ -238,13 +293,13 @@ void NativeFarCall::set_destination_mt_safe(address dest, bool assert_lock) { address addr_call = addr_at(0); assert(NativeFarCall::is_at(addr_call), "unexpected code at call site"); - +/* // Patch the constant in the call's trampoline stub. address trampoline_stub_addr = get_trampoline(); assert(trampoline_stub_addr != nullptr, "Must be"); assert (!is_NativeCallTrampolineStub_at(dest), "chained trampolines"); nativeCallTrampolineStub_at(trampoline_stub_addr)->set_destination(dest); - +*/ ICache::invalidate_range(addr_call, instruction_size); } @@ -311,46 +366,84 @@ address NativeCall::instruction_address() const { if (NativeShortCall::is_at(address(this))) { return NativeShortCall::at(address(this))->instruction_address(); } + assert(false, "bad"); return 0; } address NativeCall::next_instruction_address() const { - return 0; + if (NativeShortCall::is_at(address(this))) { + return NativeShortCall::at(address(this))->next_instruction_address(); + } + assert(false, "bad"); + return 0; } address NativeCall::return_address() const { + if (NativeShortCall::is_at(address(this))) { + return NativeShortCall::at(address(this))->return_address(); + } + assert(false, "bad"); return 0; } address NativeCall::destination() const { + if (NativeShortCall::is_at(address(this))) { + return NativeShortCall::at(address(this))->destination(); + } + assert(false, "bad"); return 0; } void NativeCall::set_destination(address dest) { + if (NativeShortCall::is_at(address(this))) { + NativeShortCall::at(address(this))->set_destination(dest); + } else { + assert(false, "bad"); + } } void NativeCall::verify() { + if (NativeShortCall::is_at(address(this))) { + NativeShortCall::at(address(this))->verify(); + } else { + assert(false, "bad"); + } } void NativeCall::print() { - tty->print_cr(PTR_FORMAT ": jal/auipc,ld,jalr x1, offset/reg", p2i(instruction_address())); + if (NativeShortCall::is_at(address(this))) { + NativeShortCall::at(address(this))->print(); + } else { + assert(false, "bad"); + } } -void NativeCall::set_destination_mt_safe(address dest, bool assert_lock = true) { +void NativeCall::set_destination_mt_safe(address dest, bool assert_lock) { + if (NativeShortCall::is_at(address(this))) { + NativeShortCall::at(address(this))->set_destination_mt_safe(dest, assert_lock); + } else { + assert(false, "bad"); + } } address NativeCall::get_trampoline() { + if (NativeShortCall::is_at(address(this))) { + return NativeShortCall::at(address(this))->get_trampoline(); + } + assert(false, "bad"); return 0; } bool NativeCall::is_call_before(address return_address) { - return false; + return NativeShortCall::is_call_before(return_address); } void NativeCall::insert(address code_pos, address entry) { + NativeShortCall::insert(code_pos, entry); } void NativeCall::replace_mt_safe(address instr_addr, address code_buffer) { + NativeShortCall::replace_mt_safe(instr_addr, code_buffer); } //------------------------------------------------------------------- diff --git a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp index e823d2516f8b8..f50f2dafc291c 100644 --- a/src/hotspot/cpu/riscv/nativeInst_riscv.hpp +++ b/src/hotspot/cpu/riscv/nativeInst_riscv.hpp @@ -248,11 +248,6 @@ NativeInstruction* nativeInstruction_at(address addr) { return (NativeInstruction*)addr; } -// The natural type of an RISCV instruction is uint32_t -NativeInstruction* nativeInstruction_at(uint32_t *addr) { - return (NativeInstruction*)addr; -} - NativeCall* nativeCall_at(address addr); NativeCall* nativeCall_before(address return_address); @@ -273,6 +268,8 @@ class NativeShortCall: public NativeInstruction { void print(); void set_destination_mt_safe(address dest, bool assert_lock = true); + + address get_trampoline(); static NativeShortCall* at(address addr); static bool is_at(address addr); @@ -336,6 +333,7 @@ class NativeCall: public NativeInstruction { void set_destination_mt_safe(address dest, bool assert_lock = true); address get_trampoline(); + static bool is_at(address addr) { return NativeShortCall::is_at(addr) || NativeFarCall::is_at(addr) ;} static bool is_call_before(address return_address); static void insert(address code_pos, address entry); static void replace_mt_safe(address instr_addr, address code_buffer); @@ -350,7 +348,7 @@ inline NativeCall* nativeCall_at(address addr) { inline NativeCall* nativeCall_before(address return_address) { assert_cond(return_address != nullptr); - NativeCall* call = (NativeCall*)(NativeInstruction::instruction_size); + NativeCall* call = (NativeCall*)(return_address - NativeInstruction::instruction_size); DEBUG_ONLY(call->verify()); return call; } @@ -533,8 +531,10 @@ class NativeCallTrampolineStub : public NativeInstruction { enum RISCV_specific_constants { // Refer to function emit_trampoline_stub. - instruction_size = 2 * wordSize, - data_offset = 0 * wordSize + instruction_size = 3 * NativeInstruction::instruction_size + wordSize, // auipc + ld + jr + target address + data_offset = 3 * NativeInstruction::instruction_size, // auipc + ld + jr + instruction_size2 = 2 * wordSize, + data_offset2 = 0 * wordSize }; address destination(nmethod *nm = nullptr) const; @@ -543,6 +543,31 @@ class NativeCallTrampolineStub : public NativeInstruction { }; inline bool is_NativeCallTrampolineStub_at(address addr) { + // Ensure that the stub is exactly + // ld t0, L--->auipc + ld + // jr t0 + // L: + + // judge inst + register + imm + // 1). check the instructions: auipc + ld + jalr + // 2). check if auipc[11:7] == t0 and ld[11:7] == t0 and ld[19:15] == t0 && jr[19:15] == t0 + // 3). check if the offset in ld[31:20] equals the data_offset + assert_cond(addr != nullptr); + const int instr_size = NativeInstruction::instruction_size; + if (NativeInstruction::is_auipc_at(addr) && + NativeInstruction::is_ld_at(addr + instr_size) && + NativeInstruction::is_jalr_at(addr + 2 * instr_size) && + (NativeInstruction::extract_rd(addr) == x5) && + (NativeInstruction::extract_rd(addr + instr_size) == x5) && + (NativeInstruction::extract_rs1(addr + instr_size) == x5) && + (NativeInstruction::extract_rs1(addr + 2 * instr_size) == x5) && + (Assembler::extract(Assembler::ld_instr(addr + 4), 31, 20) == NativeCallTrampolineStub::data_offset)) { + return true; + } + return false; +} + +inline bool is_NativeCallTrampolineStub_at2(address addr) { assert_cond(addr != nullptr); const int instr_size = NativeInstruction::instruction_size; uint32_t top = Assembler::ld_instr(addr); diff --git a/src/hotspot/share/code/nmethod.cpp b/src/hotspot/share/code/nmethod.cpp index 1028c1451ca87..b2834af1fca77 100644 --- a/src/hotspot/share/code/nmethod.cpp +++ b/src/hotspot/share/code/nmethod.cpp @@ -620,9 +620,9 @@ nmethod* nmethod::new_nmethod(const methodHandle& method, } // Do verification and logging outside CodeCache_lock. if (nm != nullptr) { - if (PrintNMethods) { + /*if (PrintNMethods) { nm->print_nmethod(true); - } + }*/ // Safepoints in nmethod::verify aren't allowed because nm hasn't been installed yet. DEBUG_ONLY(nm->verify();) nm->log_new_nmethod();