8325536: JVM crash during CDS archive creation with -XX:+AllowArchivi…

…ngWithJavaAgent

src/hotspot/share/oops/instanceKlass.cpp

@@ -972,6 +972,11 @@ void InstanceKlass::rewrite_class(TRAPS) {
   }
   Rewriter::rewrite(this, CHECK);
   set_rewritten();
+#if INCLUDE_CDS
+  if (CDSConfig::is_dumping_dynamic_archive()) {
+    assign_class_loader_type();
+  }
+#endif
 }
 
 // Now relocate and link method entry points after class is rewritten.

test/hotspot/jtreg/runtime/cds/appcds/jvmti/dumpingWithAgent/AppWithBMH.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+// Application which loads BoundMethodHandle species classes like the following:
+// java/lang/invoke/BoundMethodHandle$Species_LLLL
+import java.lang.management.ManagementFactory;
+
+public class AppWithBMH {
+  public static void main(String[] args) {
+    System.out.println("Hello world!");
+    ManagementFactory.getGarbageCollectorMXBeans();
+  }
+}

test/hotspot/jtreg/runtime/cds/appcds/jvmti/dumpingWithAgent/DumpingWithJavaAgent.java

@@ -28,7 +28,7 @@
  * @library /test/lib /test/hotspot/jtreg/runtime/cds/appcds /test/hotspot/jtreg/runtime/cds/appcds/test-classes
  * @requires vm.cds
  * @requires vm.jvmti
- * @build SimpleAgent Hello
+ * @build SimpleAgent Hello AppWithBMH
  * @run main/othervm DumpingWithJavaAgent
  */
 
@@ -40,9 +40,11 @@
 public class DumpingWithJavaAgent {
     public static String appClasses[] = {
         "Hello",
+        "AppWithBMH",
     };
     public static String agentClasses[] = {
         "SimpleAgent",
+        "SimpleAgent$1"
     };
 
     public static String warningMessages[] = {
@@ -65,8 +67,18 @@ public static void main(String[] args) throws Throwable {
         String appJar =
             ClassFileInstaller.writeJar("DumpingWithJavaAgent.jar", appClasses);
 
+        // CDS dumping with a java agent performing class transformation on BoundMethodHandle$Species classes
+        OutputAnalyzer output = TestCommon.testDump(appJar, TestCommon.list("AppWithBMH"),
+            "-XX:+UnlockDiagnosticVMOptions", diagnosticOption,
+            "-javaagent:" + agentJar + "=doTransform",
+            "AppWithBMH");
+        TestCommon.checkDump(output);
+        output.shouldContain(warningMessages[0]);
+        output.shouldContain(warningMessages[1]);
+        output.shouldContain("inside SimpleAgent");
+
         // CDS dumping with a java agent with the AllowArchvingWithJavaAgent diagnostic option.
-        OutputAnalyzer output = TestCommon.testDump(appJar, TestCommon.list("Hello"),
+        output = TestCommon.testDump(appJar, TestCommon.list("Hello"),
             "-XX:+UnlockDiagnosticVMOptions", diagnosticOption,
             "-javaagent:" + agentJar);
         TestCommon.checkDump(output);

test/hotspot/jtreg/runtime/cds/appcds/jvmti/dumpingWithAgent/SimpleAgent.java

@@ -21,14 +21,27 @@
  * questions.
  *
  */
+import java.lang.instrument.ClassFileTransformer;
 import java.lang.instrument.Instrumentation;
+import java.security.ProtectionDomain;
+import java.util.Arrays;
 
 public class SimpleAgent {
     public static void premain(String agentArg, Instrumentation instrumentation) throws Exception {
         System.out.println("inside SimpleAgent");
-        // Only load the class if the test requires it.
-        if (agentArg != null && agentArg.equals("OldSuper")) {
+        if (agentArg == null) return;
+        if (agentArg.equals("OldSuper")) {
+            // Only load the class if the test requires it.
             Class<?> cls = Class.forName("OldSuper", true, ClassLoader.getSystemClassLoader());
+        } else if (agentArg.equals("doTransform")) {
+            ClassFileTransformer transformer = new ClassFileTransformer() {
+            @Override
+            public byte[] transform(ClassLoader loader, String className, Class<?> classBeingRedefined, ProtectionDomain protectionDomain, byte[] classfileBuffer) {
+                 System.out.printf("%n Transforming %s", className);
+                 return Arrays.copyOf(classfileBuffer, classfileBuffer.length);
+               }
+             };
+             instrumentation.addTransformer(transformer);
         }
     }
 }