From 1d258ba6b08d73cf4bace0a9ed72437882354fba Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 4 Dec 2024 23:32:47 +0000 Subject: [PATCH 1/4] 8342409: [s390x] C1 unwind_handler fails to unlock synchronized methods with LM_MONITOR Backport-of: 9201e9fcc28cff37cf9996e8db38f9aee7511b1c --- src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp | 6 +++++- src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp index a7900042a9b..5908076c518 100644 --- a/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_LIRAssembler_s390.cpp @@ -230,7 +230,11 @@ int LIR_Assembler::emit_unwind_handler() { LIR_Opr lock = FrameMap::as_opr(Z_R1_scratch); monitor_address(0, lock); stub = new MonitorExitStub(lock, true, 0); - __ unlock_object(Rtmp1, Rtmp2, lock->as_register(), *stub->entry()); + if (LockingMode == LM_MONITOR) { + __ branch_optimized(Assembler::bcondAlways, *stub->entry()); + } else { + __ unlock_object(Rtmp1, Rtmp2, lock->as_register(), *stub->entry()); + } __ bind(*stub->continuation()); } diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp index c35b0923297..02b577c203c 100644 --- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp @@ -119,6 +119,8 @@ void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox branch_optimized(Assembler::bcondNotZero, slow_case); // done bind(done); + } else { + assert(false, "Unhandled LockingMode:%d", LockingMode); } } @@ -155,6 +157,8 @@ void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rb // If the object header was not pointing to the displaced header, // we do unlocking via runtime call. branch_optimized(Assembler::bcondNotEqual, slow_case); + } else { + assert(false, "Unhandled LockingMode:%d", LockingMode); } // done bind(done); From 38dc6db8145fb6a9834c857e2c56256bf589d708 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 4 Dec 2024 23:37:20 +0000 Subject: [PATCH 2/4] 8319947: Recursive lightweight locking: s390x implementation Backport-of: d457609f700bbb1fed233f1a04501c995852e5ac --- .../cpu/s390/c1_MacroAssembler_s390.cpp | 14 +- .../cpu/s390/c2_MacroAssembler_s390.cpp | 13 +- .../cpu/s390/c2_MacroAssembler_s390.hpp | 8 +- src/hotspot/cpu/s390/interp_masm_s390.cpp | 28 +- src/hotspot/cpu/s390/macroAssembler_s390.cpp | 493 +++++++++++++++--- src/hotspot/cpu/s390/macroAssembler_s390.hpp | 6 +- src/hotspot/cpu/s390/s390.ad | 34 ++ src/hotspot/cpu/s390/sharedRuntime_s390.cpp | 18 +- src/hotspot/cpu/s390/vm_version_s390.hpp | 2 + 9 files changed, 489 insertions(+), 127 deletions(-) diff --git a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp index 02b577c203c..90ae73ddde1 100644 --- a/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c1_MacroAssembler_s390.cpp @@ -67,9 +67,6 @@ void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox verify_oop(Roop, FILE_AND_LINE); - // Load object header. - z_lg(Rmark, Address(Roop, hdr_offset)); - // Save object being locked into the BasicObjectLock... z_stg(Roop, Address(Rbox, BasicObjectLock::obj_offset())); @@ -85,6 +82,10 @@ void C1_MacroAssembler::lock_object(Register Rmark, Register Roop, Register Rbox lightweight_lock(Roop, Rmark, tmp, slow_case); } else if (LockingMode == LM_LEGACY) { NearLabel done; + + // Load object header. + z_lg(Rmark, Address(Roop, hdr_offset)); + // and mark it as unlocked. z_oill(Rmark, markWord::unlocked_value); // Save unlocked object header into the displaced header location on the stack. @@ -143,12 +144,7 @@ void C1_MacroAssembler::unlock_object(Register Rmark, Register Roop, Register Rb verify_oop(Roop, FILE_AND_LINE); if (LockingMode == LM_LIGHTWEIGHT) { - const Register tmp = Z_R1_scratch; - z_lg(Rmark, Address(Roop, hdr_offset)); - z_lgr(tmp, Rmark); - z_nill(tmp, markWord::monitor_value); - branch_optimized(Assembler::bcondNotZero, slow_case); - lightweight_unlock(Roop, Rmark, tmp, slow_case); + lightweight_unlock(Roop, Rmark, Z_R1_scratch, slow_case); } else if (LockingMode == LM_LEGACY) { // Test if object header is pointing to the displaced header, and if so, restore // the displaced header in the object. If the object header is not pointing to diff --git a/src/hotspot/cpu/s390/c2_MacroAssembler_s390.cpp b/src/hotspot/cpu/s390/c2_MacroAssembler_s390.cpp index 62c1bd943b6..3641d82dabe 100644 --- a/src/hotspot/cpu/s390/c2_MacroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/c2_MacroAssembler_s390.cpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2017, 2022 SAP SE. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -33,6 +33,15 @@ #define BLOCK_COMMENT(str) block_comment(str) #define BIND(label) bind(label); BLOCK_COMMENT(#label ":") +void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register temp1, Register temp2) { + compiler_fast_lock_lightweight_object(obj, temp1, temp2); +} + + +void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Register temp1, Register temp2) { + compiler_fast_unlock_lightweight_object(obj, temp1, temp2); +} + //------------------------------------------------------ // Special String Intrinsics. Implementation //------------------------------------------------------ diff --git a/src/hotspot/cpu/s390/c2_MacroAssembler_s390.hpp b/src/hotspot/cpu/s390/c2_MacroAssembler_s390.hpp index a502e41ee08..aecb483f0a6 100644 --- a/src/hotspot/cpu/s390/c2_MacroAssembler_s390.hpp +++ b/src/hotspot/cpu/s390/c2_MacroAssembler_s390.hpp @@ -1,6 +1,6 @@ /* - * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2017, 2022 SAP SE. All rights reserved. + * Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2017, 2024 SAP SE. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -29,6 +29,10 @@ // C2_MacroAssembler contains high-level macros for C2 public: + // Code used by cmpFastLockLightweight and cmpFastUnlockLightweight mach instructions in s390.ad file. + void fast_lock_lightweight(Register obj, Register box, Register temp1, Register temp2); + void fast_unlock_lightweight(Register obj, Register box, Register temp1, Register temp2); + //------------------------------------------- // Special String Intrinsics Implementation. //------------------------------------------- diff --git a/src/hotspot/cpu/s390/interp_masm_s390.cpp b/src/hotspot/cpu/s390/interp_masm_s390.cpp index bc7996c270f..14bb98cea6a 100644 --- a/src/hotspot/cpu/s390/interp_masm_s390.cpp +++ b/src/hotspot/cpu/s390/interp_masm_s390.cpp @@ -1005,9 +1005,6 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) { // markWord header = obj->mark().set_unlocked(); - // Load markWord from object into header. - z_lg(header, hdr_offset, object); - if (DiagnoseSyncOnValueBasedClasses != 0) { load_klass(tmp, object); testbit(Address(tmp, Klass::access_flags_offset()), exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS)); @@ -1015,9 +1012,12 @@ void InterpreterMacroAssembler::lock_object(Register monitor, Register object) { } if (LockingMode == LM_LIGHTWEIGHT) { - lightweight_lock(object, /* mark word */ header, tmp, slow_case); + lightweight_lock(object, header, tmp, slow_case); } else if (LockingMode == LM_LEGACY) { + // Load markWord from object into header. + z_lg(header, hdr_offset, object); + // Set header to be (markWord of object | UNLOCK_VALUE). // This will not change anything if it was unlocked before. z_oill(header, markWord::unlocked_value); @@ -1153,26 +1153,8 @@ void InterpreterMacroAssembler::unlock_object(Register monitor, Register object) // If we still have a lightweight lock, unlock the object and be done. if (LockingMode == LM_LIGHTWEIGHT) { - // Check for non-symmetric locking. This is allowed by the spec and the interpreter - // must handle it. - - Register tmp = current_header; - - // First check for lock-stack underflow. - z_lgf(tmp, Address(Z_thread, JavaThread::lock_stack_top_offset())); - compareU32_and_branch(tmp, (unsigned)LockStack::start_offset(), Assembler::bcondNotHigh, slow_case); - - // Then check if the top of the lock-stack matches the unlocked object. - z_aghi(tmp, -oopSize); - z_lg(tmp, Address(Z_thread, tmp)); - compare64_and_branch(tmp, object, Assembler::bcondNotEqual, slow_case); - - z_lg(header, Address(object, hdr_offset)); - z_lgr(tmp, header); - z_nill(tmp, markWord::monitor_value); - z_brne(slow_case); - lightweight_unlock(object, header, tmp, slow_case); + lightweight_unlock(object, header, current_header, slow_case); z_bru(done); } else { diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.cpp b/src/hotspot/cpu/s390/macroAssembler_s390.cpp index 777625579fa..caab0bfd930 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.cpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.cpp @@ -3190,16 +3190,20 @@ void MacroAssembler::increment_counter_eq(address counter_address, Register tmp1 bind(l); } +// "The box" is the space on the stack where we copy the object mark. void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Register temp1, Register temp2) { + + assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_lock_lightweight"); + assert_different_registers(oop, box, temp1, temp2); + Register displacedHeader = temp1; - Register currentHeader = temp1; - Register temp = temp2; + Register currentHeader = temp1; + Register temp = temp2; + NearLabel done, object_has_monitor; const int hdr_offset = oopDesc::mark_offset_in_bytes(); - assert_different_registers(temp1, temp2, oop, box); - BLOCK_COMMENT("compiler_fast_lock_object {"); // Load markWord from oop into mark. @@ -3207,8 +3211,10 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis if (DiagnoseSyncOnValueBasedClasses != 0) { load_klass(temp, oop); - testbit(Address(temp, Klass::access_flags_offset()), exact_log2(JVM_ACC_IS_VALUE_BASED_CLASS)); - z_btrue(done); + z_l(temp, Address(temp, Klass::access_flags_offset())); + assert((JVM_ACC_IS_VALUE_BASED_CLASS & 0xFFFF) == 0, "or change following instruction"); + z_nilh(temp, JVM_ACC_IS_VALUE_BASED_CLASS >> 16); + z_brne(done); } // Handle existing monitor. @@ -3222,7 +3228,8 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis // From loading the markWord, we know that oop != nullptr z_ltgr(oop, oop); z_bru(done); - } else if (LockingMode == LM_LEGACY) { + } else { + assert(LockingMode == LM_LEGACY, "must be"); // Set mark to markWord | markWord::unlocked_value. z_oill(displacedHeader, markWord::unlocked_value); @@ -3251,10 +3258,6 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis z_stg(currentHeader/*==0 or not 0*/, BasicLock::displaced_header_offset_in_bytes(), box); - z_bru(done); - } else { - assert(LockingMode == LM_LIGHTWEIGHT, "must be"); - lightweight_lock(oop, displacedHeader, temp, done); z_bru(done); } @@ -3270,10 +3273,9 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis // Otherwise, register zero is filled with the current owner. z_lghi(zero, 0); z_csg(zero, Z_thread, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), monitor_tagged); - if (LockingMode != LM_LIGHTWEIGHT) { - // Store a non-null value into the box. - z_stg(box, BasicLock::displaced_header_offset_in_bytes(), box); - } + + // Store a non-null value into the box. + z_stg(box, BasicLock::displaced_header_offset_in_bytes(), box); z_bre(done); // acquired the lock for the first time. @@ -3295,14 +3297,16 @@ void MacroAssembler::compiler_fast_lock_object(Register oop, Register box, Regis } void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Register temp1, Register temp2) { + + assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight"); + assert_different_registers(oop, box, temp1, temp2); + Register displacedHeader = temp1; - Register currentHeader = temp2; - Register temp = temp1; + Register currentHeader = temp2; + Register temp = temp1; const int hdr_offset = oopDesc::mark_offset_in_bytes(); - assert_different_registers(temp1, temp2, oop, box); - Label done, object_has_monitor, not_recursive; BLOCK_COMMENT("compiler_fast_unlock_object {"); @@ -3326,18 +3330,14 @@ void MacroAssembler::compiler_fast_unlock_object(Register oop, Register box, Reg // Set NE to indicate 'failure' -> take slow-path z_ltgr(oop, oop); z_bru(done); - } else if (LockingMode == LM_LEGACY) { + } else { + assert(LockingMode == LM_LEGACY, "must be"); // Check if it is still a lightweight lock, this is true if we see // the stack address of the basicLock in the markWord of the object // copy box to currentHeader such that csg does not kill it. z_lgr(currentHeader, box); z_csg(currentHeader, displacedHeader, hdr_offset, oop); z_bru(done); // csg sets CR as desired. - } else { - assert(LockingMode == LM_LIGHTWEIGHT, "must be"); - - lightweight_unlock(oop, currentHeader, displacedHeader, done); - z_bru(done); } // In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0. @@ -5710,101 +5710,424 @@ SkipIfEqual::~SkipIfEqual() { } // Implements lightweight-locking. -// Branches to slow upon failure to lock the object. -// Falls through upon success. -// // - obj: the object to be locked, contents preserved. -// - hdr: the header, already loaded from obj, contents destroyed. +// - temp1, temp2: temporary registers, contents destroyed. // Note: make sure Z_R1 is not manipulated here when C2 compiler is in play -void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register temp, Label& slow_case) { +void MacroAssembler::lightweight_lock(Register obj, Register temp1, Register temp2, Label& slow) { assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking"); - assert_different_registers(obj, hdr, temp); + assert_different_registers(obj, temp1, temp2); + + Label push; + const Register top = temp1; + const Register mark = temp2; + const int mark_offset = oopDesc::mark_offset_in_bytes(); + const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset(); + + // Preload the markWord. It is important that this is the first + // instruction emitted as it is part of C1's null check semantics. + z_lg(mark, Address(obj, mark_offset)); + // First we need to check if the lock-stack has room for pushing the object reference. - z_lgf(temp, Address(Z_thread, JavaThread::lock_stack_top_offset())); + z_lgf(top, Address(Z_thread, ls_top_offset)); - compareU32_and_branch(temp, (unsigned)LockStack::end_offset()-1, bcondHigh, slow_case); + compareU32_and_branch(top, (unsigned)LockStack::end_offset(), bcondNotLow, slow); - // attempting a lightweight_lock - // Load (object->mark() | 1) into hdr - z_oill(hdr, markWord::unlocked_value); + // The underflow check is elided. The recursive check will always fail + // when the lock stack is empty because of the _bad_oop_sentinel field. - z_lgr(temp, hdr); + // Check for recursion: + z_aghi(top, -oopSize); + z_cg(obj, Address(Z_thread, top)); + z_bre(push); - // Clear lock-bits from hdr (locked state) - z_xilf(temp, markWord::unlocked_value); + // Check header for monitor (0b10). + z_tmll(mark, markWord::monitor_value); + branch_optimized(bcondNotAllZero, slow); - z_csg(hdr, temp, oopDesc::mark_offset_in_bytes(), obj); - branch_optimized(Assembler::bcondNotEqual, slow_case); + { // Try to lock. Transition lock bits 0b01 => 0b00 + const Register locked_obj = top; + z_oill(mark, markWord::unlocked_value); + z_lgr(locked_obj, mark); + // Clear lock-bits from locked_obj (locked state) + z_xilf(locked_obj, markWord::unlocked_value); + z_csg(mark, locked_obj, mark_offset, obj); + branch_optimized(Assembler::bcondNotEqual, slow); + } - // After successful lock, push object on lock-stack - z_lgf(temp, Address(Z_thread, JavaThread::lock_stack_top_offset())); - z_stg(obj, Address(Z_thread, temp)); - z_ahi(temp, oopSize); - z_st(temp, Address(Z_thread, JavaThread::lock_stack_top_offset())); + bind(push); - // as locking was successful, set CC to EQ - z_cr(temp, temp); + // After successful lock, push object on lock-stack + z_lgf(top, Address(Z_thread, ls_top_offset)); + z_stg(obj, Address(Z_thread, top)); + z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize); } // Implements lightweight-unlocking. -// Branches to slow upon failure. -// Falls through upon success. -// // - obj: the object to be unlocked -// - hdr: the (pre-loaded) header of the object, will be destroyed +// - temp1, temp2: temporary registers, will be destroyed // - Z_R1_scratch: will be killed in case of Interpreter & C1 Compiler -void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register tmp, Label& slow) { +void MacroAssembler::lightweight_unlock(Register obj, Register temp1, Register temp2, Label& slow) { assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking"); - assert_different_registers(obj, hdr, tmp); + assert_different_registers(obj, temp1, temp2); + + Label unlocked, push_and_slow; + const Register mark = temp1; + const Register top = temp2; + const int mark_offset = oopDesc::mark_offset_in_bytes(); + const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset(); #ifdef ASSERT - { - // Check that hdr is lightweight-locked. - Label hdr_ok; - z_lgr(tmp, hdr); - z_nill(tmp, markWord::lock_mask_in_place); - z_bre(hdr_ok); - stop("Header is not lightweight-locked"); - bind(hdr_ok); - } { // The following checks rely on the fact that LockStack is only ever modified by // its owning thread, even if the lock got inflated concurrently; removal of LockStack // entries after inflation will happen delayed in that case. // Check for lock-stack underflow. - Label stack_ok; - z_lgf(tmp, Address(Z_thread, JavaThread::lock_stack_top_offset())); - compareU32_and_branch(tmp, (unsigned)LockStack::start_offset(), Assembler::bcondHigh, stack_ok); + NearLabel stack_ok; + z_lgf(top, Address(Z_thread, ls_top_offset)); + compareU32_and_branch(top, (unsigned)LockStack::start_offset(), bcondNotLow, stack_ok); stop("Lock-stack underflow"); bind(stack_ok); } - { - // Check if the top of the lock-stack matches the unlocked object. - Label tos_ok; - z_aghi(tmp, -oopSize); - z_lg(tmp, Address(Z_thread, tmp)); - compare64_and_branch(tmp, obj, Assembler::bcondEqual, tos_ok); - stop("Top of lock-stack does not match the unlocked object"); - bind(tos_ok); +#endif // ASSERT + + // Check if obj is top of lock-stack. + z_lgf(top, Address(Z_thread, ls_top_offset)); + z_aghi(top, -oopSize); + z_cg(obj, Address(Z_thread, top)); + branch_optimized(bcondNotEqual, slow); + + // pop object from lock-stack +#ifdef ASSERT + const Register temp_top = temp1; // mark is not yet loaded, but be careful + z_agrk(temp_top, top, Z_thread); + z_xc(0, oopSize-1, temp_top, 0, temp_top); // wipe out lock-stack entry +#endif // ASSERT + z_alsi(in_bytes(ls_top_offset), Z_thread, -oopSize); // pop object + + // The underflow check is elided. The recursive check will always fail + // when the lock stack is empty because of the _bad_oop_sentinel field. + + // Check if recursive. (this is a check for the 2nd object on the stack) + z_aghi(top, -oopSize); + z_cg(obj, Address(Z_thread, top)); + branch_optimized(bcondEqual, unlocked); + + // Not recursive. Check header for monitor (0b10). + z_lg(mark, Address(obj, mark_offset)); + z_tmll(mark, markWord::monitor_value); + z_brnaz(push_and_slow); + +#ifdef ASSERT + // Check header not unlocked (0b01). + NearLabel not_unlocked; + z_tmll(mark, markWord::unlocked_value); + z_braz(not_unlocked); + stop("lightweight_unlock already unlocked"); + bind(not_unlocked); +#endif // ASSERT + + { // Try to unlock. Transition lock bits 0b00 => 0b01 + Register unlocked_obj = top; + z_lgr(unlocked_obj, mark); + z_oill(unlocked_obj, markWord::unlocked_value); + z_csg(mark, unlocked_obj, mark_offset, obj); + branch_optimized(Assembler::bcondEqual, unlocked); + } + + bind(push_and_slow); + + // Restore lock-stack and handle the unlock in runtime. + z_lgf(top, Address(Z_thread, ls_top_offset)); + DEBUG_ONLY(z_stg(obj, Address(Z_thread, top));) + z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize); + // set CC to NE + z_ltgr(obj, obj); // object shouldn't be null at this point + branch_optimized(bcondAlways, slow); + + bind(unlocked); +} + +void MacroAssembler::compiler_fast_lock_lightweight_object(Register obj, Register tmp1, Register tmp2) { + assert_different_registers(obj, tmp1, tmp2); + + // Handle inflated monitor. + NearLabel inflated; + // Finish fast lock successfully. MUST reach to with flag == NE + NearLabel locked; + // Finish fast lock unsuccessfully. MUST branch to with flag == EQ + NearLabel slow_path; + + if (DiagnoseSyncOnValueBasedClasses != 0) { + load_klass(tmp1, obj); + z_l(tmp1, Address(tmp1, Klass::access_flags_offset())); + assert((JVM_ACC_IS_VALUE_BASED_CLASS & 0xFFFF) == 0, "or change following instruction"); + z_nilh(tmp1, JVM_ACC_IS_VALUE_BASED_CLASS >> 16); + z_brne(slow_path); + } + + const Register mark = tmp1; + const int mark_offset = oopDesc::mark_offset_in_bytes(); + const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset(); + + BLOCK_COMMENT("compiler_fast_lightweight_locking {"); + { // lightweight locking + + // Push lock to the lock stack and finish successfully. MUST reach to with flag == EQ + NearLabel push; + + const Register top = tmp2; + + // Check if lock-stack is full. + z_lgf(top, Address(Z_thread, ls_top_offset)); + compareU32_and_branch(top, (unsigned) LockStack::end_offset() - 1, bcondHigh, slow_path); + + // The underflow check is elided. The recursive check will always fail + // when the lock stack is empty because of the _bad_oop_sentinel field. + + // Check if recursive. + z_aghi(top, -oopSize); + z_cg(obj, Address(Z_thread, top)); + z_bre(push); + + // Check for monitor (0b10) + z_lg(mark, Address(obj, mark_offset)); + z_tmll(mark, markWord::monitor_value); + z_brnaz(inflated); + + // not inflated + + { // Try to lock. Transition lock bits 0b01 => 0b00 + assert(mark_offset == 0, "required to avoid a lea"); + const Register locked_obj = top; + z_oill(mark, markWord::unlocked_value); + z_lgr(locked_obj, mark); + // Clear lock-bits from locked_obj (locked state) + z_xilf(locked_obj, markWord::unlocked_value); + z_csg(mark, locked_obj, mark_offset, obj); + branch_optimized(Assembler::bcondNotEqual, slow_path); + } + + bind(push); + + // After successful lock, push object on lock-stack. + z_lgf(top, Address(Z_thread, ls_top_offset)); + z_stg(obj, Address(Z_thread, top)); + z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize); + + z_cgr(obj, obj); // set the CC to EQ, as it could be changed by alsi + z_bru(locked); + } + BLOCK_COMMENT("} compiler_fast_lightweight_locking"); + + BLOCK_COMMENT("handle_inflated_monitor_lightweight_locking {"); + { // Handle inflated monitor. + bind(inflated); + + // mark contains the tagged ObjectMonitor*. + const Register tagged_monitor = mark; + const Register zero = tmp2; + + // Try to CAS m->owner from null to current thread. + // If m->owner is null, then csg succeeds and sets m->owner=THREAD and CR=EQ. + // Otherwise, register zero is filled with the current owner. + z_lghi(zero, 0); + z_csg(zero, Z_thread, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), tagged_monitor); + z_bre(locked); + + // Check if recursive. + z_cgr(Z_thread, zero); // zero contains the owner from z_csg instruction + z_brne(slow_path); + + // Recursive + z_agsi(Address(tagged_monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), 1ll); + z_cgr(zero, zero); + // z_bru(locked); + // Uncomment above line in the future, for now jump address is right next to us. } + BLOCK_COMMENT("} handle_inflated_monitor_lightweight_locking"); + + bind(locked); + +#ifdef ASSERT + // Check that locked label is reached with flag == EQ. + NearLabel flag_correct; + z_bre(flag_correct); + stop("CC is not set to EQ, it should be - lock"); +#endif // ASSERT + + bind(slow_path); + +#ifdef ASSERT + // Check that slow_path label is reached with flag == NE. + z_brne(flag_correct); + stop("CC is not set to NE, it should be - lock"); + bind(flag_correct); #endif // ASSERT - z_lgr(tmp, hdr); - z_oill(tmp, markWord::unlocked_value); - z_csg(hdr, tmp, oopDesc::mark_offset_in_bytes(), obj); - branch_optimized(Assembler::bcondNotEqual, slow); + // C2 uses the value of flag (NE vs EQ) to determine the continuation. +} + +void MacroAssembler::compiler_fast_unlock_lightweight_object(Register obj, Register tmp1, Register tmp2) { + assert_different_registers(obj, tmp1, tmp2); - // After successful unlock, pop object from lock-stack + // Handle inflated monitor. + NearLabel inflated, inflated_load_monitor; + // Finish fast unlock successfully. MUST reach to with flag == EQ. + NearLabel unlocked; + // Finish fast unlock unsuccessfully. MUST branch to with flag == NE. + NearLabel slow_path; + + const Register mark = tmp1; + const Register top = tmp2; + const int mark_offset = oopDesc::mark_offset_in_bytes(); + const ByteSize ls_top_offset = JavaThread::lock_stack_top_offset(); + + BLOCK_COMMENT("compiler_fast_lightweight_unlock {"); + { // Lightweight Unlock + + // Check if obj is top of lock-stack. + z_lgf(top, Address(Z_thread, ls_top_offset)); + + z_aghi(top, -oopSize); + z_cg(obj, Address(Z_thread, top)); + branch_optimized(bcondNotEqual, inflated_load_monitor); + + // Pop lock-stack. #ifdef ASSERT - z_lgf(tmp, Address(Z_thread, JavaThread::lock_stack_top_offset())); - z_aghi(tmp, -oopSize); - z_agr(tmp, Z_thread); - z_xc(0, oopSize-1, tmp, 0, tmp); // wipe out lock-stack entry + const Register temp_top = tmp1; // let's not kill top here, we can use for recursive check + z_agrk(temp_top, top, Z_thread); + z_xc(0, oopSize-1, temp_top, 0, temp_top); // wipe out lock-stack entry #endif - z_alsi(in_bytes(JavaThread::lock_stack_top_offset()), Z_thread, -oopSize); // pop object - z_cr(tmp, tmp); // set CC to EQ + z_alsi(in_bytes(ls_top_offset), Z_thread, -oopSize); // pop object + + // The underflow check is elided. The recursive check will always fail + // when the lock stack is empty because of the _bad_oop_sentinel field. + + // Check if recursive. + z_aghi(top, -oopSize); + z_cg(obj, Address(Z_thread, top)); + z_bre(unlocked); + + // Not recursive + + // Check for monitor (0b10). + z_lg(mark, Address(obj, mark_offset)); + z_tmll(mark, markWord::monitor_value); + z_brnaz(inflated); + +#ifdef ASSERT + // Check header not unlocked (0b01). + NearLabel not_unlocked; + z_tmll(mark, markWord::unlocked_value); + z_braz(not_unlocked); + stop("lightweight_unlock already unlocked"); + bind(not_unlocked); +#endif // ASSERT + + { // Try to unlock. Transition lock bits 0b00 => 0b01 + Register unlocked_obj = top; + z_lgr(unlocked_obj, mark); + z_oill(unlocked_obj, markWord::unlocked_value); + z_csg(mark, unlocked_obj, mark_offset, obj); + branch_optimized(Assembler::bcondEqual, unlocked); + } + + // Restore lock-stack and handle the unlock in runtime. + z_lgf(top, Address(Z_thread, ls_top_offset)); + DEBUG_ONLY(z_stg(obj, Address(Z_thread, top));) + z_alsi(in_bytes(ls_top_offset), Z_thread, oopSize); + // set CC to NE + z_ltgr(obj, obj); // object is not null here + z_bru(slow_path); + } + BLOCK_COMMENT("} compiler_fast_lightweight_unlock"); + + { // Handle inflated monitor. + + bind(inflated_load_monitor); + + z_lg(mark, Address(obj, mark_offset)); + +#ifdef ASSERT + z_tmll(mark, markWord::monitor_value); + z_brnaz(inflated); + stop("Fast Unlock not monitor"); +#endif // ASSERT + + bind(inflated); + +#ifdef ASSERT + NearLabel check_done, loop; + z_lgf(top, Address(Z_thread, ls_top_offset)); + bind(loop); + z_aghi(top, -oopSize); + compareU32_and_branch(top, in_bytes(JavaThread::lock_stack_base_offset()), + bcondLow, check_done); + z_cg(obj, Address(Z_thread, top)); + z_brne(loop); + stop("Fast Unlock lock on stack"); + bind(check_done); +#endif // ASSERT + + // mark contains the tagged ObjectMonitor*. + const Register monitor = mark; + + NearLabel not_recursive; + const Register recursions = tmp2; + + // Check if recursive. + load_and_test_long(recursions, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions))); + z_bre(not_recursive); // if 0 then jump, it's not recursive locking + + // Recursive unlock + z_agsi(Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)), -1ll); + z_cgr(monitor, monitor); // set the CC to EQUAL + z_bru(unlocked); + + bind(not_recursive); + + NearLabel not_ok; + // Check if the entry lists are empty. + load_and_test_long(tmp2, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(EntryList))); + z_brne(not_ok); + load_and_test_long(tmp2, Address(monitor, OM_OFFSET_NO_MONITOR_VALUE_TAG(cxq))); + z_brne(not_ok); + + z_release(); + z_stg(tmp2 /*=0*/, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), monitor); + + z_bru(unlocked); // CC = EQ here + + bind(not_ok); + + // The owner may be anonymous, and we removed the last obj entry in + // the lock-stack. This loses the information about the owner. + // Write the thread to the owner field so the runtime knows the owner. + z_stg(Z_thread, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner), monitor); + z_bru(slow_path); // CC = NE here + } + + bind(unlocked); + +#ifdef ASSERT + // Check that unlocked label is reached with flag == EQ. + NearLabel flag_correct; + z_bre(flag_correct); + stop("CC is not set to EQ, it should be - unlock"); +#endif // ASSERT + + bind(slow_path); + +#ifdef ASSERT + // Check that slow_path label is reached with flag == NE. + z_brne(flag_correct); + stop("CC is not set to NE, it should be - unlock"); + bind(flag_correct); +#endif // ASSERT + + // C2 uses the value of flag (NE vs EQ) to determine the continuation. } diff --git a/src/hotspot/cpu/s390/macroAssembler_s390.hpp b/src/hotspot/cpu/s390/macroAssembler_s390.hpp index 27a48b89b8b..0d7364ebdc9 100644 --- a/src/hotspot/cpu/s390/macroAssembler_s390.hpp +++ b/src/hotspot/cpu/s390/macroAssembler_s390.hpp @@ -726,8 +726,10 @@ class MacroAssembler: public Assembler { void compiler_fast_lock_object(Register oop, Register box, Register temp1, Register temp2); void compiler_fast_unlock_object(Register oop, Register box, Register temp1, Register temp2); - void lightweight_lock(Register obj, Register hdr, Register tmp, Label& slow); - void lightweight_unlock(Register obj, Register hdr, Register tmp, Label& slow); + void lightweight_lock(Register obj, Register tmp1, Register tmp2, Label& slow); + void lightweight_unlock(Register obj, Register tmp1, Register tmp2, Label& slow); + void compiler_fast_lock_lightweight_object(Register obj, Register tmp1, Register tmp2); + void compiler_fast_unlock_lightweight_object(Register obj, Register tmp1, Register tmp2); void resolve_jobject(Register value, Register tmp1, Register tmp2); void resolve_global_jobject(Register value, Register tmp1, Register tmp2); diff --git a/src/hotspot/cpu/s390/s390.ad b/src/hotspot/cpu/s390/s390.ad index 28cac16864d..be18bf2b668 100644 --- a/src/hotspot/cpu/s390/s390.ad +++ b/src/hotspot/cpu/s390/s390.ad @@ -9579,6 +9579,7 @@ instruct partialSubtypeCheck_vs_zero(flagsReg pcc, rarg2RegP sub, rarg3RegP supe // inlined locking and unlocking instruct cmpFastLock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRegP tmp2) %{ + predicate(LockingMode != LM_LIGHTWEIGHT); match(Set pcc (FastLock oop box)); effect(TEMP tmp1, TEMP tmp2); ins_cost(100); @@ -9589,6 +9590,7 @@ instruct cmpFastLock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRe %} instruct cmpFastUnlock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRegP tmp2) %{ + predicate(LockingMode != LM_LIGHTWEIGHT); match(Set pcc (FastUnlock oop box)); effect(TEMP tmp1, TEMP tmp2); ins_cost(100); @@ -9598,6 +9600,38 @@ instruct cmpFastUnlock(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, i ins_pipe(pipe_class_dummy); %} +instruct cmpFastLockLightweight(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRegP tmp2) %{ + predicate(LockingMode == LM_LIGHTWEIGHT); + match(Set pcc (FastLock oop box)); + effect(TEMP tmp1, TEMP tmp2); + ins_cost(100); + // TODO: s390 port size(VARIABLE_SIZE); + format %{ "FASTLOCK $oop, $box; KILL Z_ARG4, Z_ARG5" %} + ins_encode %{ + __ fast_lock_lightweight($oop$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register); + // If locking was successful, cc should indicate 'EQ'. + // The compiler generates a branch to the runtime call to + // _complete_monitor_locking_Java for the case where cc is 'NE'. + %} + ins_pipe(pipe_class_dummy); +%} + +instruct cmpFastUnlockLightweight(flagsReg pcc, iRegP_N2P oop, iRegP_N2P box, iRegP tmp1, iRegP tmp2) %{ + predicate(LockingMode == LM_LIGHTWEIGHT); + match(Set pcc (FastUnlock oop box)); + effect(TEMP tmp1, TEMP tmp2); + ins_cost(100); + // TODO: s390 port size(FIXED_SIZE); + format %{ "FASTUNLOCK $oop, $box; KILL Z_ARG4, Z_ARG5" %} + ins_encode %{ + __ fast_unlock_lightweight($oop$$Register, $box$$Register, $tmp1$$Register, $tmp2$$Register); + // If unlocking was successful, cc should indicate 'EQ'. + // The compiler generates a branch to the runtime call to + // _complete_monitor_unlocking_Java for the case where cc is 'NE'. + %} + ins_pipe(pipe_class_dummy); +%} + instruct inlineCallClearArrayConst(SSlenDW cnt, iRegP_N2P base, Universe dummy, flagsReg cr) %{ match(Set dummy (ClearArray cnt base)); effect(KILL cr); diff --git a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp index 87cf9cb600c..0ee88345282 100644 --- a/src/hotspot/cpu/s390/sharedRuntime_s390.cpp +++ b/src/hotspot/cpu/s390/sharedRuntime_s390.cpp @@ -1711,8 +1711,13 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, __ add2reg(r_box, lock_offset, Z_SP); // Try fastpath for locking. - // Fast_lock kills r_temp_1, r_temp_2. - __ compiler_fast_lock_object(r_oop, r_box, r_tmp1, r_tmp2); + if (LockingMode == LM_LIGHTWEIGHT) { + // Fast_lock kills r_temp_1, r_temp_2. + __ compiler_fast_lock_lightweight_object(r_oop, r_tmp1, r_tmp2); + } else { + // Fast_lock kills r_temp_1, r_temp_2. + __ compiler_fast_lock_object(r_oop, r_box, r_tmp1, r_tmp2); + } __ z_bre(done); //------------------------------------------------------------------------- @@ -1910,8 +1915,13 @@ nmethod *SharedRuntime::generate_native_wrapper(MacroAssembler *masm, __ add2reg(r_box, lock_offset, Z_SP); // Try fastpath for unlocking. - // Fast_unlock kills r_tmp1, r_tmp2. - __ compiler_fast_unlock_object(r_oop, r_box, r_tmp1, r_tmp2); + if (LockingMode == LM_LIGHTWEIGHT) { + // Fast_unlock kills r_tmp1, r_tmp2. + __ compiler_fast_unlock_lightweight_object(r_oop, r_tmp1, r_tmp2); + } else { + // Fast_unlock kills r_tmp1, r_tmp2. + __ compiler_fast_unlock_object(r_oop, r_box, r_tmp1, r_tmp2); + } __ z_bre(done); // Slow path for unlocking. diff --git a/src/hotspot/cpu/s390/vm_version_s390.hpp b/src/hotspot/cpu/s390/vm_version_s390.hpp index 7ac60a10ae7..4f963c4e485 100644 --- a/src/hotspot/cpu/s390/vm_version_s390.hpp +++ b/src/hotspot/cpu/s390/vm_version_s390.hpp @@ -413,6 +413,8 @@ class VM_Version: public Abstract_VM_Version { // s390 supports fast class initialization checks static bool supports_fast_class_init_checks() { return true; } + constexpr static bool supports_recursive_lightweight_locking() { return true; } + // CPU feature query functions static const char* get_model_string() { return _model_string; } static bool has_StoreFacilityListExtended() { return (_features[0] & StoreFacilityListExtendedMask) == StoreFacilityListExtendedMask; } From cb43c26600539b563131533c768507e670cc368d Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 4 Dec 2024 23:37:43 +0000 Subject: [PATCH 3/4] 8344164: [s390x] ProblemList hotspot/jtreg/runtime/NMT/VirtualAllocCommitMerge.java Backport-of: 3245f56e53792b3cfc9788799ba1594d6af15bea --- test/hotspot/jtreg/ProblemList.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test/hotspot/jtreg/ProblemList.txt b/test/hotspot/jtreg/ProblemList.txt index cdf8aeb3775..117432dc2c1 100644 --- a/test/hotspot/jtreg/ProblemList.txt +++ b/test/hotspot/jtreg/ProblemList.txt @@ -126,6 +126,7 @@ runtime/StackGuardPages/TestStackGuardPagesNative.java 8303612 linux-all runtime/ErrorHandling/TestDwarf.java#checkDecoder 8305489 linux-all runtime/ErrorHandling/MachCodeFramesInErrorFile.java 8313315 linux-ppc64le runtime/cds/appcds/customLoader/HelloCustom_JFR.java 8241075 linux-all,windows-x64 +runtime/NMT/VirtualAllocCommitMerge.java 8309698 linux-s390x applications/jcstress/copy.java 8229852 linux-all From 9101cc14972ce6bdeb966e67bcacc8b693c37d0a Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Wed, 4 Dec 2024 23:38:37 +0000 Subject: [PATCH 4/4] 8336012: Fix usages of jtreg-reserved properties Backport-of: e6c5aa7a6cb54c647d261facdcffa6a410849627 --- test/jdk/java/lang/invoke/PrivateInvokeTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/jdk/java/lang/invoke/PrivateInvokeTest.java b/test/jdk/java/lang/invoke/PrivateInvokeTest.java index 12edf8e3263..8ae78d96713 100644 --- a/test/jdk/java/lang/invoke/PrivateInvokeTest.java +++ b/test/jdk/java/lang/invoke/PrivateInvokeTest.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2009, 2016, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2009, 2024, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -67,8 +67,6 @@ public class PrivateInvokeTest { String vstr = System.getProperty(THIS_CLASS.getSimpleName()+".verbose"); if (vstr == null) vstr = System.getProperty(THIS_CLASS.getName()+".verbose"); - if (vstr == null) - vstr = System.getProperty("test.verbose"); if (vstr != null) verbose = Integer.parseInt(vstr); } private static int referenceKind(Method m) {