From 95f152cd270554432e80ec59da4762e7f193b34e Mon Sep 17 00:00:00 2001 From: Romain Reuillon Date: Wed, 8 Nov 2023 14:17:14 +0100 Subject: [PATCH 1/4] Implement cache to skip bundle creation process when a bundle exists and has been produced using identical inputs --- .../scala/com/typesafe/sbt/osgi/Osgi.scala | 187 +++++++++++++----- .../com/typesafe/sbt/osgi/OsgiKeys.scala | 4 + .../scala/com/typesafe/sbt/osgi/SbtOsgi.scala | 8 +- 3 files changed, 145 insertions(+), 54 deletions(-) diff --git a/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala b/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala index 787484e..731fa1c 100644 --- a/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala +++ b/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala @@ -33,7 +33,7 @@ import scala.language.implicitConversions private object Osgi { - def bundleTask( + def cachedBundle( headers: OsgiManifestHeaders, additionalHeaders: Map[String, String], fullClasspath: Seq[File], @@ -44,63 +44,148 @@ private object Osgi { failOnUndecidedPackage: Boolean, sourceDirectories: Seq[File], packageOptions: scala.Seq[sbt.PackageOption], - streams: TaskStreams, - useJVMJar: Boolean): File = { - val builder = new Builder - - if (failOnUndecidedPackage) { - streams.log.info("Validating all packages are set private or exported for OSGi explicitly...") - val internal = headers.privatePackage - val exported = headers.exportPackage - validateAllPackagesDecidedAbout(internal, exported, sourceDirectories) - } - - builder.setClasspath(fullClasspath.toArray) - - val props = headersToProperties(headers, additionalHeaders) - addPackageOptions(props, packageOptions) - builder.setProperties(props) - - includeResourceProperty(resourceDirectories.filter(_.exists), embeddedJars, explodedJars) foreach (dirs => - builder.setProperty(INCLUDERESOURCE, dirs)) - bundleClasspathProperty(embeddedJars) foreach (jars => - builder.setProperty(BUNDLE_CLASSPATH, jars)) - // Write to a temporary file to prevent trying to simultaneously read from and write to the - // same jar file in exportJars mode (which causes a NullPointerException). - val tmpArtifactPath = file(artifactPath.absolutePath + ".tmp") - // builder.build is not thread-safe because it uses a static SimpleDateFormat. This ensures - // that all calls to builder.build are serialized. - val jar = synchronized { - builder.build + useJVMJar: Boolean, + cacheBundle: Boolean): Option[File] = { + + def footprint = { + val serialised = + s"""${headers} + |${additionalHeaders} + |${fullClasspath.map(f => FileInfo.lastModified(f).lastModified)} + |${artifactPath} + |${resourceDirectories.map(f => FileInfo.lastModified(f).lastModified)} + |${embeddedJars.map(f => FileInfo.lastModified(f).lastModified)} + |${explodedJars.map(f => FileInfo.lastModified(f).lastModified)} + |$failOnUndecidedPackage + |${sourceDirectories.map(f => FileInfo.lastModified(f).lastModified)} + |${packageOptions} + |$useJVMJar + |""".stripMargin + + Hash.apply(serialised).mkString("") } - val log = streams.log - builder.getWarnings.asScala.foreach(s => log.warn(s"bnd: $s")) - builder.getErrors.asScala.foreach(s => log.error(s"bnd: $s")) - if (!useJVMJar) jar.write(tmpArtifactPath) + if (!cacheBundle) None else { - val tmpArtifactDirectoryPath = file(artifactPath.absolutePath + "_tmpdir") - IO.delete(tmpArtifactDirectoryPath) - tmpArtifactDirectoryPath.mkdirs() - - val manifest = jar.getManifest - jar.writeFolder(tmpArtifactDirectoryPath) - - def content = { - import _root_.java.nio.file._ - import _root_.scala.collection.JavaConverters._ - val path = tmpArtifactDirectoryPath.toPath - Files.walk(path).iterator.asScala.map(f => f.toFile -> path.relativize(f).toString).filterNot { case (_, p) => p == "META-INF/MANIFEST.MF" }.toTraversable - } + val footprintValue = footprint + val bundleCacheFootprint = file(artifactPath.absolutePath + "_footprint") - IO.jar(content, tmpArtifactPath, manifest) - IO.delete(tmpArtifactDirectoryPath) + if(!bundleCacheFootprint.exists() || IO.read(bundleCacheFootprint) != footprintValue) { + IO.write(bundleCacheFootprint, footprintValue) + None + } else if(artifactPath.exists()) Some(artifactPath) else None } - - IO.move(tmpArtifactPath, artifactPath) - artifactPath } + def withCache( + headers: OsgiManifestHeaders, + additionalHeaders: Map[String, String], + fullClasspath: Seq[File], + artifactPath: File, + resourceDirectories: Seq[File], + embeddedJars: Seq[File], + explodedJars: Seq[File], + failOnUndecidedPackage: Boolean, + sourceDirectories: Seq[File], + packageOptions: scala.Seq[sbt.PackageOption], + useJVMJar: Boolean, + cacheBundle: Boolean)(produce: => File): File = + cachedBundle( + headers, + additionalHeaders, + fullClasspath, + artifactPath, + resourceDirectories, + embeddedJars, + explodedJars, + failOnUndecidedPackage, + sourceDirectories, + packageOptions, + useJVMJar, + cacheBundle + ).getOrElse(produce) + + def bundleTask( + headers: OsgiManifestHeaders, + additionalHeaders: Map[String, String], + fullClasspath: Seq[File], + artifactPath: File, + resourceDirectories: Seq[File], + embeddedJars: Seq[File], + explodedJars: Seq[File], + failOnUndecidedPackage: Boolean, + sourceDirectories: Seq[File], + packageOptions: scala.Seq[sbt.PackageOption], + useJVMJar: Boolean, + cacheBundle: Boolean, + streams: TaskStreams): File = + withCache(headers, + additionalHeaders, + fullClasspath, + artifactPath, + resourceDirectories, + embeddedJars, + explodedJars, + failOnUndecidedPackage, + sourceDirectories, + packageOptions, + useJVMJar, + cacheBundle) { + val builder = new Builder + + if (failOnUndecidedPackage) { + streams.log.info("Validating all packages are set private or exported for OSGi explicitly...") + val internal = headers.privatePackage + val exported = headers.exportPackage + validateAllPackagesDecidedAbout(internal, exported, sourceDirectories) + } + + builder.setClasspath(fullClasspath.toArray) + + val props = headersToProperties(headers, additionalHeaders) + addPackageOptions(props, packageOptions) + builder.setProperties(props) + + includeResourceProperty(resourceDirectories.filter(_.exists), embeddedJars, explodedJars) foreach (dirs => + builder.setProperty(INCLUDERESOURCE, dirs)) + bundleClasspathProperty(embeddedJars) foreach (jars => + builder.setProperty(BUNDLE_CLASSPATH, jars)) + // Write to a temporary file to prevent trying to simultaneously read from and write to the + // same jar file in exportJars mode (which causes a NullPointerException). + val tmpArtifactPath = file(artifactPath.absolutePath + ".tmp") + // builder.build is not thread-safe because it uses a static SimpleDateFormat. This ensures + // that all calls to builder.build are serialized. + val jar = synchronized { + builder.build + } + val log = streams.log + builder.getWarnings.asScala.foreach(s => log.warn(s"bnd: $s")) + builder.getErrors.asScala.foreach(s => log.error(s"bnd: $s")) + + if (!useJVMJar) jar.write(tmpArtifactPath) + else { + val tmpArtifactDirectoryPath = file(artifactPath.absolutePath + "_tmpdir") + IO.delete(tmpArtifactDirectoryPath) + tmpArtifactDirectoryPath.mkdirs() + + val manifest = jar.getManifest + jar.writeFolder(tmpArtifactDirectoryPath) + + def content = { + import _root_.java.nio.file._ + import _root_.scala.collection.JavaConverters._ + val path = tmpArtifactDirectoryPath.toPath + Files.walk(path).iterator.asScala.map(f => f.toFile -> path.relativize(f).toString).filterNot { case (_, p) => p == "META-INF/MANIFEST.MF" }.toTraversable + } + + IO.jar(content, tmpArtifactPath, manifest) + IO.delete(tmpArtifactDirectoryPath) + } + + IO.move(tmpArtifactPath, artifactPath) + artifactPath + } + private def addPackageOptions(props: Properties, packageOptions: Seq[PackageOption]) = { packageOptions .collect({ case attr: ManifestAttributes ⇒ attr.attributes }) diff --git a/src/main/scala/com/typesafe/sbt/osgi/OsgiKeys.scala b/src/main/scala/com/typesafe/sbt/osgi/OsgiKeys.scala index 7882485..7e096f2 100644 --- a/src/main/scala/com/typesafe/sbt/osgi/OsgiKeys.scala +++ b/src/main/scala/com/typesafe/sbt/osgi/OsgiKeys.scala @@ -107,6 +107,10 @@ object OsgiKeys { SettingKey[Boolean](prefix("PackageWithJVMJar"), "Use the JVM jar tools to craft the bundle instead of the one from BND." + "Without this setting the produced bundle are detected as corrupted by recent JVMs") + val cacheBundle: SettingKey[Boolean] = + SettingKey[Boolean](prefix("CacheBundle"), "Do not build a new bundle if a bundle already exists and has been crafted from identical inputs") + + private def prefix(key: String) = "osgi" + key } diff --git a/src/main/scala/com/typesafe/sbt/osgi/SbtOsgi.scala b/src/main/scala/com/typesafe/sbt/osgi/SbtOsgi.scala index ce5c6a8..2d0f2db 100644 --- a/src/main/scala/com/typesafe/sbt/osgi/SbtOsgi.scala +++ b/src/main/scala/com/typesafe/sbt/osgi/SbtOsgi.scala @@ -52,8 +52,9 @@ object SbtOsgi extends AutoPlugin { failOnUndecidedPackage.value, (sourceDirectories in Compile).value, (packageOptions in (Compile, packageBin)).value, - streams.value, - packageWithJVMJar.value), + packageWithJVMJar.value, + cacheBundle.value, + streams.value), Compile / sbt.Keys.packageBin := bundle.value, manifestHeaders := OsgiManifestHeaders( bundleActivator.value, @@ -87,6 +88,7 @@ object SbtOsgi extends AutoPlugin { additionalHeaders := Map.empty, embeddedJars := Nil, explodedJars := Nil, - packageWithJVMJar := false) + packageWithJVMJar := false, + cacheBundle := false) } } From 23998be10871b588d1f9a0ac1508fb437e8b8cb1 Mon Sep 17 00:00:00 2001 From: Romain Reuillon Date: Wed, 8 Nov 2023 16:24:54 +0100 Subject: [PATCH 2/4] Walk through directories and use hash instead of last modified --- .../scala/com/typesafe/sbt/osgi/Osgi.scala | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala b/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala index 731fa1c..aad7b3b 100644 --- a/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala +++ b/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala @@ -47,23 +47,26 @@ private object Osgi { useJVMJar: Boolean, cacheBundle: Boolean): Option[File] = { - def footprint = { - val serialised = - s"""${headers} - |${additionalHeaders} - |${fullClasspath.map(f => FileInfo.lastModified(f).lastModified)} - |${artifactPath} - |${resourceDirectories.map(f => FileInfo.lastModified(f).lastModified)} - |${embeddedJars.map(f => FileInfo.lastModified(f).lastModified)} - |${explodedJars.map(f => FileInfo.lastModified(f).lastModified)} - |$failOnUndecidedPackage - |${sourceDirectories.map(f => FileInfo.lastModified(f).lastModified)} - |${packageOptions} - |$useJVMJar - |""".stripMargin - - Hash.apply(serialised).mkString("") - } + def fileFootprint(file: File) = + if(!file.exists()) Seq() + else if(file.isDirectory) Files.walk(file.toPath).iterator().asScala.map(f => f.toAbsolutePath.toString -> FileInfo.hash(f.toFile).hash.mkString(" ")).toSeq + else Seq(file.absolutePath -> FileInfo.hash(file).hash.mkString(" ")) + + def serialized = + s"""${headers} + |${additionalHeaders} + |${fullClasspath.flatMap(fileFootprint)} + |${artifactPath} + |${resourceDirectories.flatMap(fileFootprint)} + |${embeddedJars.flatMap(fileFootprint)} + |${explodedJars.flatMap(fileFootprint)} + |$failOnUndecidedPackage + |${sourceDirectories.flatMap(fileFootprint)} + |${packageOptions} + |$useJVMJar + |""".stripMargin + + def footprint = Hash.apply(serialized).mkString("") if (!cacheBundle) None else { From abcba3bf64cbc2f67efa1187bf1b4d058d5bf79b Mon Sep 17 00:00:00 2001 From: Romain Reuillon Date: Wed, 8 Nov 2023 17:10:09 +0100 Subject: [PATCH 3/4] Serialize hash as hex string --- src/main/scala/com/typesafe/sbt/osgi/Osgi.scala | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala b/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala index aad7b3b..318a832 100644 --- a/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala +++ b/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala @@ -48,9 +48,9 @@ private object Osgi { cacheBundle: Boolean): Option[File] = { def fileFootprint(file: File) = - if(!file.exists()) Seq() - else if(file.isDirectory) Files.walk(file.toPath).iterator().asScala.map(f => f.toAbsolutePath.toString -> FileInfo.hash(f.toFile).hash.mkString(" ")).toSeq - else Seq(file.absolutePath -> FileInfo.hash(file).hash.mkString(" ")) + if (!file.exists()) Seq() + else if (file.isDirectory) Files.walk(file.toPath).iterator().asScala.map(f => f.toAbsolutePath.toString -> Hash.toHex(FileInfo.hash(f.toFile).hash.toArray)).toSeq + else Seq(file.absolutePath -> Hash.toHex(FileInfo.hash(file).hash.toArray)) def serialized = s"""${headers} @@ -73,13 +73,12 @@ private object Osgi { val footprintValue = footprint val bundleCacheFootprint = file(artifactPath.absolutePath + "_footprint") - if(!bundleCacheFootprint.exists() || IO.read(bundleCacheFootprint) != footprintValue) { + if (!bundleCacheFootprint.exists() || IO.read(bundleCacheFootprint) != footprintValue) { IO.write(bundleCacheFootprint, footprintValue) None - } else if(artifactPath.exists()) Some(artifactPath) else None + } else if (artifactPath.exists()) Some(artifactPath) else None } } - def withCache( headers: OsgiManifestHeaders, additionalHeaders: Map[String, String], From 0b5d97d480be23612d5ddfc3b9a3d55425257c6a Mon Sep 17 00:00:00 2001 From: Romain Reuillon Date: Thu, 9 Nov 2023 13:50:25 +0100 Subject: [PATCH 4/4] Implement altenative caching stategies --- .../scala/com/typesafe/sbt/osgi/Osgi.scala | 52 ++++++++++--------- .../com/typesafe/sbt/osgi/OsgiKeys.scala | 11 +++- .../scala/com/typesafe/sbt/osgi/SbtOsgi.scala | 4 +- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala b/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala index 318a832..294ef7e 100644 --- a/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala +++ b/src/main/scala/com/typesafe/sbt/osgi/Osgi.scala @@ -16,19 +16,19 @@ package com.typesafe.sbt.osgi -import java.nio.file.{ FileVisitOption, Files, Path } - +import java.nio.file.{FileVisitOption, Files, Path} import aQute.bnd.osgi.Builder -import aQute.bnd.osgi.Constants._ +import aQute.bnd.osgi.Constants.* +import com.typesafe.sbt.osgi.OsgiKeys.CacheStrategy + import java.util.Properties import java.util.function.Predicate import java.util.stream.Collectors - -import sbt._ -import sbt.Keys._ +import sbt.* +import sbt.Keys.* import sbt.Package.ManifestAttributes -import scala.collection.JavaConverters._ +import scala.collection.JavaConverters.* import scala.language.implicitConversions private object Osgi { @@ -45,12 +45,19 @@ private object Osgi { sourceDirectories: Seq[File], packageOptions: scala.Seq[sbt.PackageOption], useJVMJar: Boolean, - cacheBundle: Boolean): Option[File] = { + cacheStrategy: Option[CacheStrategy]): Option[File] = cacheStrategy.flatMap { strategy => + + def fileFootprint(file: File) = { + def footprint(f: File) = + strategy match { + case CacheStrategy.LastModified => FileInfo.lastModified(f).lastModified.toString + case CacheStrategy.Hash => Hash.toHex(FileInfo.hash(f).hash.toArray) + } - def fileFootprint(file: File) = if (!file.exists()) Seq() - else if (file.isDirectory) Files.walk(file.toPath).iterator().asScala.map(f => f.toAbsolutePath.toString -> Hash.toHex(FileInfo.hash(f.toFile).hash.toArray)).toSeq - else Seq(file.absolutePath -> Hash.toHex(FileInfo.hash(file).hash.toArray)) + else if (file.isDirectory) Files.walk(file.toPath).iterator().asScala.map(f => f.toAbsolutePath.toString -> footprint(f.toFile).toSeq) + else Seq(file.absolutePath -> footprint(file)) + } def serialized = s"""${headers} @@ -68,16 +75,13 @@ private object Osgi { def footprint = Hash.apply(serialized).mkString("") - if (!cacheBundle) None - else { - val footprintValue = footprint - val bundleCacheFootprint = file(artifactPath.absolutePath + "_footprint") + val footprintValue = footprint + val bundleCacheFootprint = file(artifactPath.absolutePath + "_footprint") - if (!bundleCacheFootprint.exists() || IO.read(bundleCacheFootprint) != footprintValue) { - IO.write(bundleCacheFootprint, footprintValue) - None - } else if (artifactPath.exists()) Some(artifactPath) else None - } + if (!bundleCacheFootprint.exists() || IO.read(bundleCacheFootprint) != footprintValue) { + IO.write(bundleCacheFootprint, footprintValue) + None + } else if (artifactPath.exists()) Some(artifactPath) else None } def withCache( headers: OsgiManifestHeaders, @@ -91,7 +95,7 @@ private object Osgi { sourceDirectories: Seq[File], packageOptions: scala.Seq[sbt.PackageOption], useJVMJar: Boolean, - cacheBundle: Boolean)(produce: => File): File = + cacheStrategy: Option[CacheStrategy])(produce: => File): File = cachedBundle( headers, additionalHeaders, @@ -104,7 +108,7 @@ private object Osgi { sourceDirectories, packageOptions, useJVMJar, - cacheBundle + cacheStrategy ).getOrElse(produce) def bundleTask( @@ -119,7 +123,7 @@ private object Osgi { sourceDirectories: Seq[File], packageOptions: scala.Seq[sbt.PackageOption], useJVMJar: Boolean, - cacheBundle: Boolean, + cacheStrategy: Option[CacheStrategy], streams: TaskStreams): File = withCache(headers, additionalHeaders, @@ -132,7 +136,7 @@ private object Osgi { sourceDirectories, packageOptions, useJVMJar, - cacheBundle) { + cacheStrategy) { val builder = new Builder if (failOnUndecidedPackage) { diff --git a/src/main/scala/com/typesafe/sbt/osgi/OsgiKeys.scala b/src/main/scala/com/typesafe/sbt/osgi/OsgiKeys.scala index 7e096f2..c5d8fb6 100644 --- a/src/main/scala/com/typesafe/sbt/osgi/OsgiKeys.scala +++ b/src/main/scala/com/typesafe/sbt/osgi/OsgiKeys.scala @@ -107,10 +107,17 @@ object OsgiKeys { SettingKey[Boolean](prefix("PackageWithJVMJar"), "Use the JVM jar tools to craft the bundle instead of the one from BND." + "Without this setting the produced bundle are detected as corrupted by recent JVMs") - val cacheBundle: SettingKey[Boolean] = - SettingKey[Boolean](prefix("CacheBundle"), "Do not build a new bundle if a bundle already exists and has been crafted from identical inputs") + val cacheStrategy: SettingKey[Option[CacheStrategy]] = + SettingKey[Option[CacheStrategy]](prefix("CacheBundle"), "Do not build a new bundle if a bundle already exists and has been crafted from identical inputs") private def prefix(key: String) = "osgi" + key + + sealed trait CacheStrategy + + object CacheStrategy { + object Hash extends CacheStrategy + object LastModified extends CacheStrategy + } } diff --git a/src/main/scala/com/typesafe/sbt/osgi/SbtOsgi.scala b/src/main/scala/com/typesafe/sbt/osgi/SbtOsgi.scala index 2d0f2db..e0aa655 100644 --- a/src/main/scala/com/typesafe/sbt/osgi/SbtOsgi.scala +++ b/src/main/scala/com/typesafe/sbt/osgi/SbtOsgi.scala @@ -53,7 +53,7 @@ object SbtOsgi extends AutoPlugin { (sourceDirectories in Compile).value, (packageOptions in (Compile, packageBin)).value, packageWithJVMJar.value, - cacheBundle.value, + cacheStrategy.value, streams.value), Compile / sbt.Keys.packageBin := bundle.value, manifestHeaders := OsgiManifestHeaders( @@ -89,6 +89,6 @@ object SbtOsgi extends AutoPlugin { embeddedJars := Nil, explodedJars := Nil, packageWithJVMJar := false, - cacheBundle := false) + cacheStrategy := None) } }