feat: Platform independent file and resource dependencies (#134)

* Add a mechanism for determining parts of the file names corresponding to a module dependency. Fixes #120
* Add a mechanism for overriding automatic filename determination for module dependencies. #121

build.sc

@@ -7,11 +7,10 @@ import $ivy.`com.lihaoyi::mill-contrib-buildinfo:`
 import mill.contrib.buildinfo.BuildInfo
 import com.github.lolgab.mill.mima._
 
-
 import $ivy.`com.lihaoyi::mill-contrib-scoverage:`
 import mill.contrib.scoverage.{ScoverageModule, ScoverageReport}
 
-object scoverage extends BaseModule with ScoverageReport 
+object scoverage extends BaseModule with ScoverageReport
 
 trait BaseModule extends ScoverageModule with ScalafmtModule {
   def scalaVersion = "3.3.0-RC3"
@@ -36,40 +35,62 @@ trait BaseModule extends ScoverageModule with ScalafmtModule {
     "-source:future",
     "-Ykind-projector",
     "-Vprofile"
-    )
+  )
 
-  trait BaseTest extends ScoverageTests with TestModule.Munit  {
+  trait BaseTest extends ScoverageTests with TestModule.Munit {
     def ivyDeps = Agg(
       ivy"org.scalameta::munit:$munitVersion",
       ivy"org.scalameta::munit-scalacheck:$munitVersion"
     )
 
+    val suffix = System.getProperty("os.name") match {
+      case "windows" => ".dll"
+      case _         => ".so"
+    }
 
     def compileLibs = T {
 
-      val suffix = System.getProperty("os.name") match {
-        case "windows" => ".dll"
-        case _ => ".so"
-      }
-
       os.proc(
         "clang",
         "-shared",
         "-fvisibility=default",
         "-Os",
         "-o",
-        s"libs/test$suffix",
+        s"libs/test_x64$suffix",
         "libs/test-code.c"
       ).spawn()
 
       os.pwd / "libs" / s"test$suffix"
     }
 
-    override def compile = T{
+    override def compile = T {
       compileLibs()
+      generateResources()
       super.compile()
     }
 
+    def generateResources = T {
+      val nativeResources =
+        resources().view.map(_.path / "native").filter(os.exists)
+      val cFiles = nativeResources.flatMap(f => os.walk(f).filter(_.ext == "c")).toSet
+
+      cFiles.map { file =>
+        val destination =
+          file / os.up / s"${file.last.stripSuffix(".c")}_x64$suffix"
+        T.log.info(s"Compiling ${destination.toString} from ${file.toString}")
+        os.proc(
+          "clang",
+          "-shared",
+          "-fvisibility=default",
+          "-Os",
+          "-o",
+          destination.toString,
+          file.toString
+        ).spawn()
+        PathRef(destination)
+      }
+    }
+
   }
 }
 object core
@@ -82,12 +103,13 @@ object core
 
   override def scalaDocOptions = T {
     super.scalaDocOptions() ++ Seq(
-      // "-project-logo", 
+      // "-project-logo",
       // (millSourcePath / "docs" / "_assets" / "images" / "logo.svg").toString,
-      "-project", "slinc"
+      "-project",
+      "slinc"
     )
   }
-  
+
   def pomSettings = pomTemplate("slinc-core")
 
   def specializationArity = 4
@@ -181,7 +203,7 @@ object `runtime` extends BaseModule with PublishableModule {
   object test extends Tests with TestModule.Munit {
     def ivyDeps = Agg(ivy"org.scalameta::munit:$munitVersion")
 
-    def jvm = T.input{ System.getProperty("java.version")}
+    def jvm = T.input { System.getProperty("java.version") }
     def moduleDeps = super.moduleDeps ++ Seq(core.test)
     def forkArgs = super.forkArgs() ++ Seq(
       "--enable-preview",

core/src/fr/hammons/slinc/annotations/NeedsFile.scala

@@ -10,7 +10,11 @@ final case class NeedsFile(val path: String)
       DependencyAnnotation:
   def toDependency: Dependency =
     val filePath = Paths.get(path).nn
-    Dependency.FilePath(filePath)
+    val fileName = filePath.getFileName().toString()
+    Dependency.FilePath(
+      filePath,
+      fileName.endsWith(".so") || fileName.endsWith(".dll")
+    )
 
 object NeedsFile:
   inline def apply[F]: List[NeedsFile] = ${

core/src/fr/hammons/slinc/annotations/NeedsResource.scala

@@ -10,8 +10,12 @@ final case class NeedsResource(val resourcePath: String)
     extends StaticAnnotation,
       DependencyAnnotation:
   def toDependency: Dependency = resourcePath match
-    case path @ s"${_}.c" => Dependency.CResource(Paths.get(path).nn)
-    case path             => Dependency.LibraryResource(Paths.get(path).nn)
+    case path @ s"${_}.c" => Dependency.CResource(path)
+    case path =>
+      Dependency.LibraryResource(
+        path,
+        path.endsWith(".so") || path.endsWith(".dll")
+      )
 
 object NeedsResource:
   inline def apply[L]: List[NeedsResource] = ${

core/src/fr/hammons/slinc/fset/Dependency.scala

@@ -3,7 +3,7 @@ package fr.hammons.slinc.fset
 import java.nio.file.Path
 
 enum Dependency:
-  case LibraryResource(path: Path)
-  case CResource(path: Path)
+  case LibraryResource(path: String, specific: Boolean)
+  case CResource(path: String)
   case PathLibrary(name: String)
-  case FilePath(path: Path)
+  case FilePath(path: Path, specific: Boolean)

core/src/fr/hammons/slinc/modules/CacheFile.scala

@@ -2,4 +2,4 @@ package fr.hammons.slinc.modules
 
 import java.nio.file.Path
 
-final case class CacheFile(origin: Path, cachePath: Path, updated: Boolean)
+final case class CacheFile(origin: String, cachePath: Path, updated: Boolean)

core/src/fr/hammons/slinc/modules/LinkageTools.scala

@@ -2,6 +2,7 @@ package fr.hammons.slinc.modules
 
 import java.nio.file.Paths
 import fr.hammons.slinc.types.{OS, os}
+import fr.hammons.slinc.types.{Arch, arch}
 import java.nio.file.Files
 import java.nio.file.Path
 import java.security.MessageDigest
@@ -15,11 +16,15 @@ import scala.sys.process.*
 object LinkageTools:
   private val dependenciesLoaded = AtomicReference(Set.empty[Dependency])
 
-  private val libSuffix = os match
+  private lazy val libSuffix = os match
     case OS.Linux | OS.Darwin => ".so"
     case OS.Windows           => ".dll"
     case OS.Unknown => throw Error("Lib suffix is unknown on this platform")
 
+  private lazy val potentialArchMarks = arch match
+    case Arch.X64         => Set("x64", "x86_64", "amd64")
+    case Arch.Unknown | _ => Set.empty
+
   private val resourcesLocation = "/native"
 
   private val md =
@@ -35,7 +40,7 @@ object LinkageTools:
         .nn
     case OS.Unknown => throw Error("Cache location is unknown on this platform")
 
-  def sendResourceToCache(location: Path): CacheFile =
+  def sendResourceToCache(location: String): CacheFile =
     if !Files.exists(cacheLocation) then Files.createDirectories(cacheLocation)
     val cacheFile = cacheLocation.resolve(location)
     val resourceHash = hash(
@@ -66,18 +71,46 @@ object LinkageTools:
     val currentDeps = dependenciesLoaded.get.nn
     if !currentDeps.contains(dependency) then
       dependency match
-        case Dependency.LibraryResource(path) =>
+        case Dependency.LibraryResource(path, true) =>
           val cachedFile = sendResourceToCache(path)
           load(cachedFile.cachePath)
+
+        case Dependency.LibraryResource(path, false) =>
+          val resolvedPath = potentialArchMarks.view
+            .map(archMark => s"${path}_$archMark$libSuffix")
+            .find(path =>
+              getClass().getResource(s"$resourcesLocation/$path") != null
+            )
+            .getOrElse(
+              throw Error(
+                s"No library resource found for $arch $os with path like $path"
+              )
+            )
+
+          val cachedFile = sendResourceToCache(resolvedPath)
+          load(cachedFile.cachePath)
+
         case Dependency.CResource(path) =>
           val cachedFile = sendResourceToCache(path)
           val compilationPath = compileCachedCCode(cachedFile)
           load(compilationPath)
         case Dependency.PathLibrary(name) =>
           loadLibrary(name)
-        case Dependency.FilePath(path) =>
+        case Dependency.FilePath(path, true) =>
           load(path)
 
+        case Dependency.FilePath(path, false) =>
+          val resolvedPath = potentialArchMarks.view
+            .map(archMark => Paths.get(s"${path}_$archMark$libSuffix").nn)
+            .find(Files.exists(_))
+            .getOrElse(
+              throw Error(
+                s"No library file found for $arch $os with path like $path"
+              )
+            )
+
+          load(resolvedPath)
+
       dependenciesLoaded.compareAndExchange(
         currentDeps,
         currentDeps + dependency
@@ -100,7 +133,7 @@ object LinkageTools:
       )
       if cmd.! != 0 then
         throw Error(
-          s"failed to compile resource ${cachedFile.origin.toAbsolutePath()}"
+          s"failed to compile resource ${cachedFile.origin}"
         )
 
     libLocation

core/test/src/fr/hammons/slinc/FSetRuntimeSpec.scala

@@ -39,7 +39,37 @@ trait FSetRuntimeSpec(val slinc: Slinc) extends munit.FunSuite:
         5
       )
 
-  test("Relative file loading works"):
+  test("Lib resource absolute loading works"):
+      if os == OS.Windows then
+        @NeedsResource("test_x64.dll")
+        trait L derives FSet:
+          def identity_int(i: CInt): CInt
+
+        assertEquals(
+          FSet.instance[L].identity_int(2),
+          2
+        )
+      else
+        @NeedsResource("test_x64.so")
+        trait L derives FSet:
+          def identity_int(i: CInt): CInt
+
+        assertEquals(
+          FSet.instance[L].identity_int(2),
+          2
+        )
+
+  test("Lib resource platform independent loading works"):
+      @NeedsResource("test")
+      trait L derives FSet:
+        def identity_int(i: CInt): CInt
+
+      assertEquals(
+        FSet.instance[L].identity_int(2),
+        2
+      )
+
+  test("Overridden relative file loading works"):
       if os == OS.Windows then
         @NeedsFile("libs/test.dll")
         trait L derives FSet:
@@ -59,6 +89,16 @@ trait FSetRuntimeSpec(val slinc: Slinc) extends munit.FunSuite:
           2
         )
 
+  test("Platform independent file loading works"):
+      @NeedsFile("libs/test")
+      trait L derives FSet:
+        def test_fn(i: CInt): CInt
+
+      assertEquals(
+        FSet.instance[L].test_fn(2),
+        2
+      )
+
   test("Absolute file loading works"):
       if os == OS.Linux then
         if !Files.exists(Paths.get("/tmp/test.so")) then

core/test/src/fr/hammons/slinc/FSetSpec.scala

@@ -94,12 +94,18 @@ class FSetSpec extends munit.FunSuite:
 
   test("library resources should be recorded for loading"):
       @NeedsResource("test")
+      @NeedsResource("test.so")
+      @NeedsResource("test.dll")
       trait L derives FSet:
         def abs(i: CInt): CInt
 
       assertEquals(
-        summon[FSet[L]].dependencies,
-        List(Dependency.LibraryResource(Paths.get("test").nn))
+        summon[FSet[L]].dependencies.toSet,
+        Set(
+          Dependency.LibraryResource("test", false),
+          Dependency.LibraryResource("test.so", true),
+          Dependency.LibraryResource("test.dll", true)
+        )
       )
 
   test("c resources should be recorded for loading"):
@@ -109,7 +115,7 @@ class FSetSpec extends munit.FunSuite:
 
       assertEquals(
         summon[FSet[L]].dependencies,
-        List(Dependency.CResource(Paths.get("test.c").nn))
+        List(Dependency.CResource("test.c"))
       )
 
   test("library dependencies should be recorded for loading"):
@@ -124,12 +130,17 @@ class FSetSpec extends munit.FunSuite:
 
   test("file dependencies should be recorded for loading"):
       @NeedsFile("core/test/resources/native/test.c")
+      @NeedsFile("test.so")
+      @NeedsFile("test.dll")
       trait L derives FSet:
         def abs(i: CInt): CInt
 
       assertEquals(
-        summon[FSet[L]].dependencies,
-        List(
-          Dependency.FilePath(Paths.get("core/test/resources/native/test.c").nn)
+        summon[FSet[L]].dependencies.toSet,
+        Set(
+          Dependency
+            .FilePath(Paths.get("core/test/resources/native/test.c").nn, false),
+          Dependency.FilePath(Paths.get("test.so").nn, true),
+          Dependency.FilePath(Paths.get("test.dll").nn, true)
         )
       )

core/test/src/fr/hammons/slinc/modules/LinkageToolsSpec.scala

@@ -1,12 +1,11 @@
 package fr.hammons.slinc.modules
 
 import java.nio.file.Files
-import java.nio.file.Paths
 import java.io.FileInputStream
 
 class LinkageToolsSpec extends munit.FunSuite:
   test("send resource to cache"):
-      val testC = Paths.get("test.c").nn
+      val testC = "test.c"
       val resultLocation =
         LinkageTools.sendResourceToCache(testC)
 
@@ -32,7 +31,7 @@ class LinkageToolsSpec extends munit.FunSuite:
       val hash2 = LinkageTools.hash(
         FileInputStream(
           LinkageTools
-            .sendResourceToCache(Paths.get("test.c").nn)
+            .sendResourceToCache("test.c")
             .cachePath
             .toString()
         )