From 4adc1e27af1b92d309982c74f1ea265194f048e0 Mon Sep 17 00:00:00 2001 From: Michael Pollmeier Date: Wed, 6 Nov 2024 14:42:50 +0100 Subject: [PATCH 1/3] check tmp dir for noexec flag and warn user zstd requires a tmp dir without the `noexec` flag - probe for that and inform the user about their options --- .../flatgraph/storage/Deserialization.scala | 25 +++++----- .../flatgraph/storage/Serialization.scala | 2 +- .../scala/flatgraph/storage/ZstdWrapper.scala | 50 +++++++++++++++++++ 3 files changed, 63 insertions(+), 14 deletions(-) create mode 100644 core/src/main/scala/flatgraph/storage/ZstdWrapper.scala diff --git a/core/src/main/scala/flatgraph/storage/Deserialization.scala b/core/src/main/scala/flatgraph/storage/Deserialization.scala index 9dc42149..27f104ec 100644 --- a/core/src/main/scala/flatgraph/storage/Deserialization.scala +++ b/core/src/main/scala/flatgraph/storage/Deserialization.scala @@ -172,19 +172,18 @@ object Deserialization { } private def readPool(manifest: GraphItem, fileChannel: FileChannel): Array[String] = { - val stringPoolLength = Zstd - .decompress( + val stringPoolLength = ZstdWrapper( + Zstd.decompress( fileChannel.map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolLength.startOffset, manifest.stringPoolLength.compressedLength), manifest.stringPoolLength.decompressedLength - ) - .order(ByteOrder.LITTLE_ENDIAN) - val stringPoolBytes = Zstd - .decompress( - fileChannel - .map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolBytes.startOffset, manifest.stringPoolBytes.compressedLength), + ).order(ByteOrder.LITTLE_ENDIAN) + ) + val stringPoolBytes = ZstdWrapper( + Zstd.decompress( + fileChannel.map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolBytes.startOffset, manifest.stringPoolBytes.compressedLength), manifest.stringPoolBytes.decompressedLength - ) - .order(ByteOrder.LITTLE_ENDIAN) + ).order(ByteOrder.LITTLE_ENDIAN) + ) val poolBytes = new Array[Byte](manifest.stringPoolBytes.decompressedLength) stringPoolBytes.get(poolBytes) val pool = new Array[String](manifest.stringPoolLength.decompressedLength >> 2) @@ -214,9 +213,9 @@ object Deserialization { private def readArray(channel: FileChannel, ptr: OutlineStorage, nodes: Array[Array[GNode]], stringPool: Array[String]): Array[?] = { if (ptr == null) return null - val dec = Zstd - .decompress(channel.map(FileChannel.MapMode.READ_ONLY, ptr.startOffset, ptr.compressedLength), ptr.decompressedLength) - .order(ByteOrder.LITTLE_ENDIAN) + val dec = ZstdWrapper( + Zstd.decompress(channel.map(FileChannel.MapMode.READ_ONLY, ptr.startOffset, ptr.compressedLength), ptr.decompressedLength) + ).order(ByteOrder.LITTLE_ENDIAN) ptr.typ match { case StorageType.Bool => val bytes = new Array[Byte](dec.limit()) diff --git a/core/src/main/scala/flatgraph/storage/Serialization.scala b/core/src/main/scala/flatgraph/storage/Serialization.scala index 97023239..dc3a3f9b 100644 --- a/core/src/main/scala/flatgraph/storage/Serialization.scala +++ b/core/src/main/scala/flatgraph/storage/Serialization.scala @@ -180,7 +180,7 @@ object Serialization { private[flatgraph] def write(bytes: Array[Byte], res: OutlineStorage, filePtr: AtomicLong, fileChannel: FileChannel): OutlineStorage = { res.decompressedLength = bytes.length - val compressed = Zstd.compress(bytes) + val compressed = ZstdWrapper(Zstd.compress(bytes)) var outPos = filePtr.getAndAdd(compressed.length) res.startOffset = outPos diff --git a/core/src/main/scala/flatgraph/storage/ZstdWrapper.scala b/core/src/main/scala/flatgraph/storage/ZstdWrapper.scala new file mode 100644 index 00000000..32744cb0 --- /dev/null +++ b/core/src/main/scala/flatgraph/storage/ZstdWrapper.scala @@ -0,0 +1,50 @@ +package flatgraph.storage + +import org.slf4j.LoggerFactory + +import java.nio.file.{Files, Paths} +import scala.jdk.CollectionConverters.* +import scala.util.{Properties, Try} + +object ZstdWrapper { + private val logger = LoggerFactory.getLogger(getClass) + + /** + * zstd-jni ships system libraries that are being unpacked, loaded and executed from the system tmp directory. + * If that fails we get a rather obscure error message - this wrapper adds a check if the tmp dir is executable, + * and enhances the error message if the zstd invocation fails. + * + * This is where zstd-jni loads the system library: + * https://github.com/luben/zstd-jni/blob/9b08f1d0cdcf3b12b7a307cbba3d9f195149250b/src/main/java/com/github/luben/zstd/util/Native.java#L71 + */ + def apply[A](fun: => A): A = { + probeTmpMountOptions() + + try { + fun + } catch { case e => + throw new JniInvocationException( + "Error while trying to invoke zstd, i.e. cannot compress or decompress, which is required for flatgraph's storage", + Option(e) + ) + } + } + + private def probeTmpMountOptions(): Unit = { + val tmpDirPath = System.getProperty("java.io.tmpdir") + lazy val warnMessage = s"the configured temp directory ($tmpDirPath) is mounted with `noexec` flag - " + + "this will likely lead to an error when trying to invoke zstd. Please either remount it without `noexec` or " + + "configure a different tmp directory, e.g. via java system property `-Djava.io.tmpdir=/path/to/tmp`" + Try { + if (Properties.isLinux || Properties.isMac) { + val mounts = Files.readAllLines(Paths.get("/proc/mounts")) + if (mounts.asScala.exists { mountInfoLine => mountInfoLine.contains(s" $tmpDirPath ") && mountInfoLine.contains("noexec")}) + logger.warn(warnMessage) + } + } + // we're just probing here to warn the user and give some hints about fixing the situation + // it's fairly brittle as well, so if this fails we won't bother + } + + class JniInvocationException(message: String, cause: Option[Throwable]) extends RuntimeException(message, cause.orNull) +} From a056e80e506f1f4d7945a9c91a034ff042bc0705 Mon Sep 17 00:00:00 2001 From: Michael Pollmeier Date: Wed, 6 Nov 2024 14:52:12 +0100 Subject: [PATCH 2/3] fmt --- .../flatgraph/storage/Deserialization.scala | 20 ++++++++----- .../scala/flatgraph/storage/ZstdWrapper.scala | 30 +++++++++---------- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/core/src/main/scala/flatgraph/storage/Deserialization.scala b/core/src/main/scala/flatgraph/storage/Deserialization.scala index 27f104ec..0244e48c 100644 --- a/core/src/main/scala/flatgraph/storage/Deserialization.scala +++ b/core/src/main/scala/flatgraph/storage/Deserialization.scala @@ -173,16 +173,20 @@ object Deserialization { private def readPool(manifest: GraphItem, fileChannel: FileChannel): Array[String] = { val stringPoolLength = ZstdWrapper( - Zstd.decompress( - fileChannel.map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolLength.startOffset, manifest.stringPoolLength.compressedLength), - manifest.stringPoolLength.decompressedLength - ).order(ByteOrder.LITTLE_ENDIAN) + Zstd + .decompress( + fileChannel.map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolLength.startOffset, manifest.stringPoolLength.compressedLength), + manifest.stringPoolLength.decompressedLength + ) + .order(ByteOrder.LITTLE_ENDIAN) ) val stringPoolBytes = ZstdWrapper( - Zstd.decompress( - fileChannel.map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolBytes.startOffset, manifest.stringPoolBytes.compressedLength), - manifest.stringPoolBytes.decompressedLength - ).order(ByteOrder.LITTLE_ENDIAN) + Zstd + .decompress( + fileChannel.map(FileChannel.MapMode.READ_ONLY, manifest.stringPoolBytes.startOffset, manifest.stringPoolBytes.compressedLength), + manifest.stringPoolBytes.decompressedLength + ) + .order(ByteOrder.LITTLE_ENDIAN) ) val poolBytes = new Array[Byte](manifest.stringPoolBytes.decompressedLength) stringPoolBytes.get(poolBytes) diff --git a/core/src/main/scala/flatgraph/storage/ZstdWrapper.scala b/core/src/main/scala/flatgraph/storage/ZstdWrapper.scala index 32744cb0..fc4535f7 100644 --- a/core/src/main/scala/flatgraph/storage/ZstdWrapper.scala +++ b/core/src/main/scala/flatgraph/storage/ZstdWrapper.scala @@ -9,24 +9,24 @@ import scala.util.{Properties, Try} object ZstdWrapper { private val logger = LoggerFactory.getLogger(getClass) - /** - * zstd-jni ships system libraries that are being unpacked, loaded and executed from the system tmp directory. - * If that fails we get a rather obscure error message - this wrapper adds a check if the tmp dir is executable, - * and enhances the error message if the zstd invocation fails. - * - * This is where zstd-jni loads the system library: - * https://github.com/luben/zstd-jni/blob/9b08f1d0cdcf3b12b7a307cbba3d9f195149250b/src/main/java/com/github/luben/zstd/util/Native.java#L71 - */ + /** zstd-jni ships system libraries that are being unpacked, loaded and executed from the system tmp directory. If that fails we get a + * rather obscure error message - this wrapper adds a check if the tmp dir is executable, and enhances the error message if the zstd + * invocation fails. + * + * This is where zstd-jni loads the system library: + * https://github.com/luben/zstd-jni/blob/9b08f1d0cdcf3b12b7a307cbba3d9f195149250b/src/main/java/com/github/luben/zstd/util/Native.java#L71 + */ def apply[A](fun: => A): A = { probeTmpMountOptions() try { fun - } catch { case e => - throw new JniInvocationException( - "Error while trying to invoke zstd, i.e. cannot compress or decompress, which is required for flatgraph's storage", - Option(e) - ) + } catch { + case e => + throw new JniInvocationException( + "Error while trying to invoke zstd, i.e. cannot compress or decompress, which is required for flatgraph's storage", + Option(e) + ) } } @@ -38,10 +38,10 @@ object ZstdWrapper { Try { if (Properties.isLinux || Properties.isMac) { val mounts = Files.readAllLines(Paths.get("/proc/mounts")) - if (mounts.asScala.exists { mountInfoLine => mountInfoLine.contains(s" $tmpDirPath ") && mountInfoLine.contains("noexec")}) + if (mounts.asScala.exists { mountInfoLine => mountInfoLine.contains(s" $tmpDirPath ") && mountInfoLine.contains("noexec") }) logger.warn(warnMessage) - } } + } // we're just probing here to warn the user and give some hints about fixing the situation // it's fairly brittle as well, so if this fails we won't bother } From 0ed5a56153401dd70c59a4a794b1f3e9867635b7 Mon Sep 17 00:00:00 2001 From: Michael Pollmeier Date: Wed, 6 Nov 2024 15:01:59 +0100 Subject: [PATCH 3/3] readme --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ec06e8f6..888510a7 100644 --- a/README.md +++ b/README.md @@ -490,4 +490,14 @@ linked above). They exist only at compile time. Hence, it's safe to cast a given Yes, properties are no longer fields of stored nodes. Hence the debugger cannot find them. -But despair not! We have attached the `_debugChildren()` method to the GNode class. In order to see anything useful, you need to tell your debugger to use that in its object inspector. So in intellij, you need to add a custom java type renderer, make it apply to all `flatgraph.GNode` instances, and then tell it to use the expression `_debugChildren()` when expanding a node. See e.g. https://www.jetbrains.com/help/idea/customizing-views.html#renderers . \ No newline at end of file +But despair not! We have attached the `_debugChildren()` method to the GNode class. In order to see anything useful, you need to tell your debugger to use that in its object inspector. So in intellij, you need to add a custom java type renderer, make it apply to all `flatgraph.GNode` instances, and then tell it to use the expression `_debugChildren()` when expanding a node. See e.g. https://www.jetbrains.com/help/idea/customizing-views.html#renderers . + +## I got this strange `UnsatisfiedLinkError` with zstd, what's that all about? +We use https://github.com/luben/zstd-jni for storage compression, and that requires that you have not mounted your `tmp` partition with the `noexec` flag. If you really need that flag on your system tmp partition, you can workaround this by defining the java system property `-Djava.io.tmpdir=/path/to/tmp`. Before invoking zstd, flatgraph performs some basic checks and tries to help the user, should this go wrong. Here's how it looks like if it does (listed here for search engine indices): +``` +flatgraph.storage.ZstdWrapper$JniInvocationException: Error while trying to invoke zstd, i.e. cannot compress or decompress, which is required for flatgraph's storage +... +Cause: java.lang.ExceptionInInitializerError: Exception java.lang.UnsatisfiedLinkError: /tmp/libzstd-jni-1.5.6-77867291841665399714.so: /tmp/libzstd-jni-1.5.6-77867291841665399714.so: failed to map segment from shared object +[info] no zstd-jni-1.5.6-7 in java.library.path: /usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib +[info] Unsupported OS/arch, cannot find /linux/amd64/libzstd-jni-1.5.6-7.so or load zstd-jni-1.5.6-7 from system libraries. +```