From d474d5cfaa300ba736b0df625d9833ac344525ff Mon Sep 17 00:00:00 2001 From: Yuri Mizushima Date: Tue, 10 Dec 2024 19:39:20 +0900 Subject: [PATCH] refactor: create new submodule native-library-common to merge NativeUtils --- circe-checksum/pom.xml | 5 + .../com/scurrilous/circe/crc/Sse42Crc32C.java | 4 +- cpu-affinity/pom.xml | 5 + .../util/affinity/impl/CpuAffinityImpl.java | 1 + .../util/affinity/impl/NativeUtils.java | 108 ------------------ native-io/pom.xml | 5 + .../common/util/nativeio/NativeIOJni.java | 1 + .../common/util/nativeio/NativeUtils.java | 104 ----------------- .../common/util/nativeio/package-info.java | 2 +- native-library-common/pom.xml | 53 +++++++++ .../common/util/nativelib}/NativeUtils.java | 40 +++---- .../common/util/nativelib/package-info.java | 22 ++++ .../util/nativelib/NativeUtilsTest.java | 4 +- pom.xml | 1 + shaded/bookkeeper-server-shaded/pom.xml | 1 + shaded/distributedlog-core-shaded/pom.xml | 1 + .../distributedlog-core-shaded-test/pom.xml | 4 + .../DistributedLogCoreShadedJarTest.java | 11 ++ 18 files changed, 134 insertions(+), 238 deletions(-) delete mode 100644 cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java delete mode 100644 native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/NativeUtils.java create mode 100644 native-library-common/pom.xml rename {circe-checksum/src/main/java/com/scurrilous/circe/utils => native-library-common/src/main/java/org/apache/bookkeeper/common/util/nativelib}/NativeUtils.java (83%) create mode 100644 native-library-common/src/main/java/org/apache/bookkeeper/common/util/nativelib/package-info.java rename circe-checksum/src/test/java/com/scurrilous/circe/utils/NativeUtilsTests.java => native-library-common/src/test/java/org/apache/bookkeeper/common/util/nativelib/NativeUtilsTest.java (94%) diff --git a/circe-checksum/pom.xml b/circe-checksum/pom.xml index 9f9ca5f0c6f..376619c6358 100644 --- a/circe-checksum/pom.xml +++ b/circe-checksum/pom.xml @@ -39,6 +39,11 @@ + + org.apache.bookkeeper + native-library-common + ${project.parent.version} + com.google.guava diff --git a/circe-checksum/src/main/java/com/scurrilous/circe/crc/Sse42Crc32C.java b/circe-checksum/src/main/java/com/scurrilous/circe/crc/Sse42Crc32C.java index 22379a0e32d..01a40e4b5c7 100644 --- a/circe-checksum/src/main/java/com/scurrilous/circe/crc/Sse42Crc32C.java +++ b/circe-checksum/src/main/java/com/scurrilous/circe/crc/Sse42Crc32C.java @@ -15,8 +15,8 @@ ******************************************************************************/ package com.scurrilous.circe.crc; -import static com.scurrilous.circe.utils.NativeUtils.loadLibraryFromJar; -import static com.scurrilous.circe.utils.NativeUtils.libType; +import static org.apache.bookkeeper.common.util.nativelib.NativeUtils.libType; +import static org.apache.bookkeeper.common.util.nativelib.NativeUtils.loadLibraryFromJar; import java.nio.ByteBuffer; import com.scurrilous.circe.IncrementalIntHash; diff --git a/cpu-affinity/pom.xml b/cpu-affinity/pom.xml index 5ba9bc95de5..5839c916cec 100644 --- a/cpu-affinity/pom.xml +++ b/cpu-affinity/pom.xml @@ -33,6 +33,11 @@ + + org.apache.bookkeeper + native-library-common + ${project.parent.version} + com.google.guava guava diff --git a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/CpuAffinityImpl.java b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/CpuAffinityImpl.java index 380509c2fb9..94c494f9032 100644 --- a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/CpuAffinityImpl.java +++ b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/CpuAffinityImpl.java @@ -34,6 +34,7 @@ import java.util.TreeSet; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; +import org.apache.bookkeeper.common.util.nativelib.NativeUtils; /** * Implementation of CPU Affinity functionality. diff --git a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java b/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java deleted file mode 100644 index 094c0056f75..00000000000 --- a/cpu-affinity/src/main/java/org/apache/bookkeeper/common/util/affinity/impl/NativeUtils.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - * - */ -package org.apache.bookkeeper.common.util.affinity.impl; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import lombok.NonNull; -import lombok.experimental.UtilityClass; - -/** - * Utility class to load jni library from inside a JAR. - */ -@UtilityClass -public class NativeUtils { - - public static final String TEMP_WORKDIR_PROPERTY_NAME = "org.apache.bookkeeper.native.workdir"; - - private static final String TEMP_DIR_NAME = "native"; - - /** - * loads given library from the this jar. ie: this jar contains: /lib/pulsar-checksum.jnilib - * - * @param path - * : absolute path of the library in the jar
- * if this jar contains: /lib/pulsar-checksum.jnilib then provide the same absolute path as input - * @throws Exception - */ - @SuppressFBWarnings( - value = "RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE", - justification = "work around for java 9: https://github.com/spotbugs/spotbugs/issues/493") - public static void loadLibraryFromJar(String path) throws Exception { - checkArgument(path.startsWith("/"), "absolute path must start with /"); - - String[] parts = path.split("/"); - checkArgument(parts.length > 0, "absolute path must contain file name"); - - String filename = parts[parts.length - 1]; - checkArgument(path.startsWith("/"), "absolute path must start with /"); - - // create the temp dir - final Path dir; - final String tempWorkDirName = System.getProperty(TEMP_WORKDIR_PROPERTY_NAME); - if (tempWorkDirName == null || tempWorkDirName.isEmpty()) { - dir = Files.createTempDirectory(TEMP_DIR_NAME); - } else { - final File tempWorkDir = new File(tempWorkDirName); - if (!tempWorkDir.exists() || !tempWorkDir.isDirectory()) { - throw new FileNotFoundException("The tempWorkDir doesn't exist: " + tempWorkDirName); - } - dir = Files.createTempDirectory(tempWorkDir.toPath(), TEMP_DIR_NAME); - } - dir.toFile().deleteOnExit(); - - // create the temp file - File temp = new File(dir.toString(), filename); - temp.deleteOnExit(); - - byte[] buffer = new byte[1024]; - int read; - - try (InputStream input = NativeUtils.class.getResourceAsStream(path); - OutputStream out = new FileOutputStream(temp)) { - if (input == null) { - throw new FileNotFoundException("Couldn't find file into jar " + path); - } - - while ((read = input.read(buffer)) != -1) { - out.write(buffer, 0, read); - } - } - - if (!temp.exists()) { - throw new FileNotFoundException("Failed to copy file from jar at " + temp.getAbsolutePath()); - } - - System.load(temp.getAbsolutePath()); - } - - private static void checkArgument(boolean expression, @NonNull Object errorMessage) { - if (!expression) { - throw new IllegalArgumentException(String.valueOf(errorMessage)); - } - } -} diff --git a/native-io/pom.xml b/native-io/pom.xml index 6c7c7fe9f0f..923ea43dbeb 100644 --- a/native-io/pom.xml +++ b/native-io/pom.xml @@ -34,6 +34,11 @@ + + org.apache.bookkeeper + native-library-common + ${project.parent.version} + org.apache.commons commons-lang3 diff --git a/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/NativeIOJni.java b/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/NativeIOJni.java index 77d2e80b019..beaeff24ac4 100644 --- a/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/NativeIOJni.java +++ b/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/NativeIOJni.java @@ -20,6 +20,7 @@ */ package org.apache.bookkeeper.common.util.nativeio; +import org.apache.bookkeeper.common.util.nativelib.NativeUtils; import org.apache.commons.lang3.SystemUtils; class NativeIOJni { diff --git a/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/NativeUtils.java b/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/NativeUtils.java deleted file mode 100644 index 5bce384cb8a..00000000000 --- a/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/NativeUtils.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.bookkeeper.common.util.nativeio; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import lombok.NonNull; -import lombok.experimental.UtilityClass; - -/** - * Utility class to load jni library from inside a JAR. - */ -@UtilityClass -class NativeUtils { - - public static final String TEMP_WORKDIR_PROPERTY_NAME = "org.apache.bookkeeper.native.workdir"; - - private static final String TEMP_DIR_NAME = "native"; - - /** - * loads given library from the this jar. ie: this jar contains: /lib/pulsar-checksum.jnilib - * - * @param path - * : absolute path of the library in the jar
- * if this jar contains: /lib/pulsar-checksum.jnilib then provide the same absolute path as input - * @throws Exception - */ - @SuppressFBWarnings( - value = "OBL_UNSATISFIED_OBLIGATION", - justification = "work around for java 9: https://github.com/spotbugs/spotbugs/issues/493") - static void loadLibraryFromJar(String path) throws Exception { - checkArgument(path.startsWith("/"), "absolute path must start with /"); - - String[] parts = path.split("/"); - checkArgument(parts.length > 0, "absolute path must contain file name"); - - String filename = parts[parts.length - 1]; - - // create the temp dir - final Path dir; - final String tempWorkDirName = System.getProperty(TEMP_WORKDIR_PROPERTY_NAME); - if (tempWorkDirName == null || tempWorkDirName.isEmpty()) { - dir = Files.createTempDirectory(TEMP_DIR_NAME); - } else { - final File tempWorkDir = new File(tempWorkDirName); - if (!tempWorkDir.exists() || !tempWorkDir.isDirectory()) { - throw new FileNotFoundException("The tempWorkDir doesn't exist: " + tempWorkDirName); - } - dir = Files.createTempDirectory(tempWorkDir.toPath(), TEMP_DIR_NAME); - } - dir.toFile().deleteOnExit(); - - // create the temp file - File temp = new File(dir.toString(), filename); - temp.deleteOnExit(); - - byte[] buffer = new byte[1024]; - int read; - - try (InputStream input = NativeUtils.class.getResourceAsStream(path); - OutputStream out = new FileOutputStream(temp)) { - if (input == null) { - throw new FileNotFoundException("Couldn't find file into jar " + path); - } - - while ((read = input.read(buffer)) != -1) { - out.write(buffer, 0, read); - } - } - - if (!temp.exists()) { - throw new FileNotFoundException("Failed to copy file from jar at " + temp.getAbsolutePath()); - } - - System.load(temp.getAbsolutePath()); - } - - public static void checkArgument(boolean expression, @NonNull Object errorMessage) { - if (!expression) { - throw new IllegalArgumentException(String.valueOf(errorMessage)); - } - }} diff --git a/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/package-info.java b/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/package-info.java index d215e470874..2ba42fcdac4 100644 --- a/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/package-info.java +++ b/native-io/src/main/java/org/apache/bookkeeper/common/util/nativeio/package-info.java @@ -17,6 +17,6 @@ */ /** - * Utiltiies to access I/O using JNI. + * Utilities to access I/O using JNI. */ package org.apache.bookkeeper.common.util.nativeio; diff --git a/native-library-common/pom.xml b/native-library-common/pom.xml new file mode 100644 index 00000000000..83a9c097950 --- /dev/null +++ b/native-library-common/pom.xml @@ -0,0 +1,53 @@ + + + 4.0.0 + + org.apache.bookkeeper + bookkeeper + 4.18.0-SNAPSHOT + .. + + + native-library-common + Apache BookKeeper :: Common for Native Libraries + Common for Native Libraries + + + + com.google.guava + guava + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + + + org.apache.maven.plugins + maven-compiler-plugin + + + org.apache.maven.plugins + maven-surefire-plugin + + + + diff --git a/circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java b/native-library-common/src/main/java/org/apache/bookkeeper/common/util/nativelib/NativeUtils.java similarity index 83% rename from circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java rename to native-library-common/src/main/java/org/apache/bookkeeper/common/util/nativelib/NativeUtils.java index 2a72cefb26b..7f8e42a3a51 100644 --- a/circe-checksum/src/main/java/com/scurrilous/circe/utils/NativeUtils.java +++ b/native-library-common/src/main/java/org/apache/bookkeeper/common/util/nativelib/NativeUtils.java @@ -1,4 +1,5 @@ /* + * * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information @@ -15,8 +16,9 @@ * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. + * */ -package com.scurrilous.circe.utils; +package org.apache.bookkeeper.common.util.nativelib; import static com.google.common.base.Preconditions.checkArgument; @@ -28,10 +30,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Locale; +import lombok.experimental.UtilityClass; /** - * Utils for loading native checksum library. + * Utility class to load jni library from inside a JAR. */ +@UtilityClass public class NativeUtils { public static final String OS_NAME = System.getProperty("os.name").toLowerCase(Locale.US); @@ -47,12 +51,14 @@ public class NativeUtils { * if this jar contains: /lib/pulsar-checksum.jnilib then provide the same absolute path as input * @throws Exception */ - public static void loadLibraryFromJar(String path) throws Exception { - checkArgument(path.startsWith("/"), "absolute path must start with /"); + public static void loadLibraryFromJar(String path) throws Exception { + checkArgument(path.startsWith("/"), "absolute path must start with /"); String[] parts = path.split("/"); - String filename = (parts.length > 0) ? parts[parts.length - 1] : null; + checkArgument(parts.length > 0, "absolute path must contain file name"); + + String filename = parts[parts.length - 1]; // create the temp dir final Path dir; @@ -75,22 +81,15 @@ public static void loadLibraryFromJar(String path) throws Exception { byte[] buffer = new byte[1024]; int read; - InputStream input = NativeUtils.class.getResourceAsStream(path); - if (input == null) { - throw new FileNotFoundException("Couldn't find file into jar " + path); - } + try (InputStream input = NativeUtils.class.getResourceAsStream(path); + OutputStream out = new FileOutputStream(temp)) { + if (input == null) { + throw new FileNotFoundException("Couldn't find file into jar " + path); + } - try { - OutputStream out = new FileOutputStream(temp); - try { - while ((read = input.read(buffer)) != -1) { - out.write(buffer, 0, read); - } - } finally { - out.close(); + while ((read = input.read(buffer)) != -1) { + out.write(buffer, 0, read); } - } finally { - input.close(); } if (!temp.exists()) { @@ -104,10 +103,9 @@ public static void loadLibraryFromJar(String path) throws Exception { * Returns jni library extension based on OS specification. Maven-nar generates jni library based on different OS : * http://mark.donszelmann.org/maven-nar-plugin/aol.html (jni.extension) * - * @return + * @return library type */ public static String libType() { - if (OS_NAME.indexOf("mac") >= 0) { return "jnilib"; } else if (OS_NAME.indexOf("nix") >= 0 || OS_NAME.indexOf("nux") >= 0 || OS_NAME.indexOf("aix") > 0) { diff --git a/native-library-common/src/main/java/org/apache/bookkeeper/common/util/nativelib/package-info.java b/native-library-common/src/main/java/org/apache/bookkeeper/common/util/nativelib/package-info.java new file mode 100644 index 00000000000..9c08198d08b --- /dev/null +++ b/native-library-common/src/main/java/org/apache/bookkeeper/common/util/nativelib/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Utilities for Native Libraries. + */ +package org.apache.bookkeeper.common.util.nativelib; diff --git a/circe-checksum/src/test/java/com/scurrilous/circe/utils/NativeUtilsTests.java b/native-library-common/src/test/java/org/apache/bookkeeper/common/util/nativelib/NativeUtilsTest.java similarity index 94% rename from circe-checksum/src/test/java/com/scurrilous/circe/utils/NativeUtilsTests.java rename to native-library-common/src/test/java/org/apache/bookkeeper/common/util/nativelib/NativeUtilsTest.java index 17bb45ace2a..d868ede1a33 100644 --- a/circe-checksum/src/test/java/com/scurrilous/circe/utils/NativeUtilsTests.java +++ b/native-library-common/src/test/java/org/apache/bookkeeper/common/util/nativelib/NativeUtilsTest.java @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -package com.scurrilous.circe.utils; +package org.apache.bookkeeper.common.util.nativelib; import java.io.FileNotFoundException; import org.junit.Assert; @@ -25,7 +25,7 @@ /** * Unit test of {@link NativeUtils}. */ -public class NativeUtilsTests { +public class NativeUtilsTest { @Test public void testLoadLibrary() throws Exception { diff --git a/pom.xml b/pom.xml index ee68a53127e..f8224775e77 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,7 @@ buildtools + native-library-common circe-checksum bookkeeper-common bookkeeper-common-allocator diff --git a/shaded/bookkeeper-server-shaded/pom.xml b/shaded/bookkeeper-server-shaded/pom.xml index 1fea94ef154..89e22a81baa 100644 --- a/shaded/bookkeeper-server-shaded/pom.xml +++ b/shaded/bookkeeper-server-shaded/pom.xml @@ -75,6 +75,7 @@ com.google.protobuf:protobuf-java org.apache.bookkeeper:bookkeeper-common org.apache.bookkeeper:bookkeeper-common-allocator + org.apache.bookkeeper:native-library-common org.apache.bookkeeper:cpu-affinity org.apache.bookkeeper:bookkeeper-tools-framework org.apache.bookkeeper:bookkeeper-proto diff --git a/shaded/distributedlog-core-shaded/pom.xml b/shaded/distributedlog-core-shaded/pom.xml index 36e18f6b641..37fa39f7d77 100644 --- a/shaded/distributedlog-core-shaded/pom.xml +++ b/shaded/distributedlog-core-shaded/pom.xml @@ -77,6 +77,7 @@ net.jpountz.lz4:lz4 org.apache.bookkeeper:bookkeeper-common org.apache.bookkeeper:bookkeeper-common-allocator + org.apache.bookkeeper:native-library-common org.apache.bookkeeper:cpu-affinity org.apache.bookkeeper:bookkeeper-tools-framework org.apache.bookkeeper:bookkeeper-proto diff --git a/tests/shaded/distributedlog-core-shaded-test/pom.xml b/tests/shaded/distributedlog-core-shaded-test/pom.xml index b2a2ed716a4..11d3c8668bc 100644 --- a/tests/shaded/distributedlog-core-shaded-test/pom.xml +++ b/tests/shaded/distributedlog-core-shaded-test/pom.xml @@ -48,6 +48,10 @@ org.apache.bookkeeper circe-checksum + + org.apache.bookkeeper + native-library-common + org.apache.distributedlog distributedlog-core diff --git a/tests/shaded/distributedlog-core-shaded-test/src/test/java/org/apache/bookkeeper/tests/shaded/DistributedLogCoreShadedJarTest.java b/tests/shaded/distributedlog-core-shaded-test/src/test/java/org/apache/bookkeeper/tests/shaded/DistributedLogCoreShadedJarTest.java index 9dc6ee01932..21463070a5c 100644 --- a/tests/shaded/distributedlog-core-shaded-test/src/test/java/org/apache/bookkeeper/tests/shaded/DistributedLogCoreShadedJarTest.java +++ b/tests/shaded/distributedlog-core-shaded-test/src/test/java/org/apache/bookkeeper/tests/shaded/DistributedLogCoreShadedJarTest.java @@ -111,6 +111,17 @@ public void testCirceChecksum() throws Exception { Class.forName("com.scurrilous.circe.checksum.Crc32cIntChecksum"); } + @Test + public void testBookKeeperNativeLibraryCommonShade() throws Exception { + Class.forName("dlshade.org.apache.bookkeeper.common.util.nativelib.NativeUtils"); + assertTrue(true); + } + + @Test(expected = ClassNotFoundException.class) + public void testBookKeeperNativeLibraryCommon() throws Exception { + Class.forName("org.apache.bookkeeper.common.util.nativelib.NativeUtils"); + } + @Test public void testCirceChecksumShade() throws Exception { Class.forName("dlshade.com.scurrilous.circe.checksum.Crc32cIntChecksum");