Add support for generating code into a build with multiple projects (#73

) It's very common that you don't want to expose all the tables in one place in your system. You also don't want duplication of generated code or more than one script to generate database code. The solution is to pass a `ProjectGraph` structure to `generateFromDb` instead, in which you encode the structure of the (relevant) projects in your build. Dependencies between projects are encoded by nesting in the tree you pass (each node has a `downstream` member), with the root being the uppermost one. If multiple downstream projects want to generate the same code, it'll be pulled up to the level necessary to become visible for all of them.
oyvindberg · Dec 5, 2023 · 1cc0017 · 1cc0017
1 parent 9dbc582
commit 1cc0017
Show file tree

Hide file tree

Showing 29 changed files with 290 additions and 113 deletions.
diff --git a/...rces/typo-tester-anorm@jvm212/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala b/...rces/typo-tester-anorm@jvm212/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala
@@ -22,7 +22,7 @@ import testdb.hardcoded.myschema.marital_status.MaritalStatusRepoImpl
 import testdb.hardcoded.myschema.marital_status.MaritalStatusRow
 import testdb.hardcoded.myschema.person.PersonId
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def compositepkPerson(name: Option[String] = if (random.nextBoolean()) None else Some(random.alphanumeric.take(20).mkString),
                         one: Defaulted[Long] = Defaulted.UseDefault,
                         two: Defaulted[Option[String]] = Defaulted.UseDefault

diff --git a/...rces/typo-tester-anorm@jvm213/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala b/...rces/typo-tester-anorm@jvm213/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala
@@ -22,7 +22,7 @@ import testdb.hardcoded.myschema.marital_status.MaritalStatusRepoImpl
 import testdb.hardcoded.myschema.marital_status.MaritalStatusRow
 import testdb.hardcoded.myschema.person.PersonId
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def compositepkPerson(name: Option[String] = if (random.nextBoolean()) None else Some(random.alphanumeric.take(20).mkString),
                         one: Defaulted[Long] = Defaulted.UseDefault,
                         two: Defaulted[Option[String]] = Defaulted.UseDefault

diff --git a/...ources/typo-tester-anorm@jvm3/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala b/...ources/typo-tester-anorm@jvm3/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala
@@ -22,7 +22,7 @@ import testdb.hardcoded.myschema.marital_status.MaritalStatusRepoImpl
 import testdb.hardcoded.myschema.marital_status.MaritalStatusRow
 import testdb.hardcoded.myschema.person.PersonId
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def compositepkPerson(name: Option[String] = if (random.nextBoolean()) None else Some(random.alphanumeric.take(20).mkString),
                         one: Defaulted[Long] = Defaulted.UseDefault,
                         two: Defaulted[Option[String]] = Defaulted.UseDefault

diff --git a/...ces/typo-tester-doobie@jvm212/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala b/...ces/typo-tester-doobie@jvm212/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala
@@ -22,7 +22,7 @@ import testdb.hardcoded.myschema.marital_status.MaritalStatusRepoImpl
 import testdb.hardcoded.myschema.marital_status.MaritalStatusRow
 import testdb.hardcoded.myschema.person.PersonId
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def compositepkPerson(name: Option[String] = if (random.nextBoolean()) None else Some(random.alphanumeric.take(20).mkString),
                         one: Defaulted[Long] = Defaulted.UseDefault,
                         two: Defaulted[Option[String]] = Defaulted.UseDefault

diff --git a/...ces/typo-tester-doobie@jvm213/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala b/...ces/typo-tester-doobie@jvm213/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala
@@ -22,7 +22,7 @@ import testdb.hardcoded.myschema.marital_status.MaritalStatusRepoImpl
 import testdb.hardcoded.myschema.marital_status.MaritalStatusRow
 import testdb.hardcoded.myschema.person.PersonId
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def compositepkPerson(name: Option[String] = if (random.nextBoolean()) None else Some(random.alphanumeric.take(20).mkString),
                         one: Defaulted[Long] = Defaulted.UseDefault,
                         two: Defaulted[Option[String]] = Defaulted.UseDefault

diff --git a/...urces/typo-tester-doobie@jvm3/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala b/...urces/typo-tester-doobie@jvm3/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala
@@ -22,7 +22,7 @@ import testdb.hardcoded.myschema.marital_status.MaritalStatusRepoImpl
 import testdb.hardcoded.myschema.marital_status.MaritalStatusRow
 import testdb.hardcoded.myschema.person.PersonId
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def compositepkPerson(name: Option[String] = if (random.nextBoolean()) None else Some(random.alphanumeric.take(20).mkString),
                         one: Defaulted[Long] = Defaulted.UseDefault,
                         two: Defaulted[Option[String]] = Defaulted.UseDefault

diff --git a/...s/typo-tester-zio-jdbc@jvm212/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala b/...s/typo-tester-zio-jdbc@jvm212/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala
@@ -23,7 +23,7 @@ import testdb.hardcoded.myschema.person.PersonId
 import zio.ZIO
 import zio.jdbc.ZConnection
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def compositepkPerson(name: Option[String] = if (random.nextBoolean()) None else Some(random.alphanumeric.take(20).mkString),
                         one: Defaulted[Long] = Defaulted.UseDefault,
                         two: Defaulted[Option[String]] = Defaulted.UseDefault

diff --git a/...s/typo-tester-zio-jdbc@jvm213/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala b/...s/typo-tester-zio-jdbc@jvm213/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala
@@ -23,7 +23,7 @@ import testdb.hardcoded.myschema.person.PersonId
 import zio.ZIO
 import zio.jdbc.ZConnection
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def compositepkPerson(name: Option[String] = if (random.nextBoolean()) None else Some(random.alphanumeric.take(20).mkString),
                         one: Defaulted[Long] = Defaulted.UseDefault,
                         two: Defaulted[Option[String]] = Defaulted.UseDefault

diff --git a/...ces/typo-tester-zio-jdbc@jvm3/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala b/...ces/typo-tester-zio-jdbc@jvm3/scripts.GenHardcodedFiles/testdb/hardcoded/testInsert.scala
@@ -23,7 +23,7 @@ import testdb.hardcoded.myschema.person.PersonId
 import zio.ZIO
 import zio.jdbc.ZConnection
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def compositepkPerson(name: Option[String] = if (random.nextBoolean()) None else Some(random.alphanumeric.take(20).mkString),
                         one: Defaulted[Long] = Defaulted.UseDefault,
                         two: Defaulted[Option[String]] = Defaulted.UseDefault

diff --git a/site-in/other-features/generate-into-multiple-projects.md b/site-in/other-features/generate-into-multiple-projects.md
@@ -0,0 +1,77 @@
+---
+title: Generate code into multiple projects
+---
+
+It's very common that you don't want to expose all the tables in one place in your system. 
+You also don't want duplication of generated code or more than one script to generate database code. 
+
+The solution is to pass a `ProjectGraph` structure to `generateFromDb` instead, 
+in which you encode the structure of the (relevant) projects in your build.
+
+Dependencies between projects are encoded by nesting in the tree you pass (each node has a `downstream` member), 
+with the root being the uppermost one. 
+
+If multiple downstream projects want to generate the same code, it'll be pulled up to the level necessary to become visible for all of them.
+
+sample:
+```scala mdoc:silent
+import typo.*
+import java.nio.file.Path
+import java.sql.Connection
+
+def generate(implicit c: Connection): String = {
+  val cwd: Path = Path.of(sys.props("user.dir"))
+
+  val generated = generateFromDb(
+    Options(
+      pkg = "org.mypkg",
+      jsonLibs = Nil,
+      dbLib = Some(DbLibName.ZioJdbc)
+    ),
+    // setup a project graph. this outer-most project is the root project.
+    // if multiple downstream projects need the same relation, it'll be pulled up until it's visible for all.
+    // in this simple example it means `a.bicycle` will be pulled up here
+    ProjectGraph(
+      name = "a",
+      target = cwd.resolve("a/src/main/typo"),
+      value = Selector.None,
+      scripts = Nil,
+      downstream = List(
+        ProjectGraph(
+          name = "b",
+          target = cwd.resolve("b/src/main/typo"),
+          value = Selector.fullRelationNames(
+            "a.bicycle",
+            "b.person"
+          ),
+          // where to find sql files
+          scripts = List(cwd.resolve("b/src/main/sql")),
+          downstream = Nil
+        ),
+        ProjectGraph(
+          name = "c",
+          target = cwd.resolve("b/src/main/typo"),
+          value = Selector.fullRelationNames(
+            "a.bicycle",
+            "c.animal"
+          ),
+          scripts = List(cwd.resolve("b/src/main/sql")),
+          downstream = Nil
+        )
+      )
+    )
+  )
+
+  generated.foreach(_.overwriteFolder())
+
+  import scala.sys.process.*
+
+  (List("git", "add") ++ generated.map(_.folder.toString)).!!
+}
+```
+
+### todo:
+
+- [ ] `testInsert` (we'll need one for each project)
+- [ ] docs
+- [ ] tests
diff --git a/site-in/other-features/testing-with-random-values.md b/site-in/other-features/testing-with-random-values.md
@@ -4,7 +4,7 @@ title: Testing with random values
 
 This covers a lot of interesting ground, test-wise.
 
-If you enable `enableTestInserts` in `typo.Options` you now get an `testInsert` class, with a method to insert a row for each table Typo knows about. 
+If you enable `enableTestInserts` in `typo.Options` you now get an `TestInsert` class, with a method to insert a row for each table Typo knows about. 
 All values except ids, foreign keys and so on are *randomly generated*, but you can override them with named parameters.
 
 The idea is that you:
@@ -13,7 +13,7 @@ The idea is that you:
 - will get random values for the rest
 - are still forced to follow FKs to setup the data graph correctly
 - it's easy to follow those FKs, because after inserting a row you get the persisted version back, including generated IDs
-- can get the same values each time by hard coding the seed `new testInsert(new scala.util.Random(0L))`, or you can run it multiple times with different seeds to see that the random values really do not matter
+- can get the same values each time by hard coding the seed `new TestInsert(new scala.util.Random(0L))`, or you can run it multiple times with different seeds to see that the random values really do not matter
 - do not need to write *any* code to get all this available to you, like the rest of Typo.
 
 In summary, this is a fantastic way of setting up complex test scenarios in the database!
@@ -30,11 +30,11 @@ c.setAutoCommit(false)
 ```scala mdoc
 import adventureworks.customtypes.{Defaulted, TypoXml}
 import adventureworks.production.unitmeasure.UnitmeasureId
-import adventureworks.testInsert
+import adventureworks.TestInsert
 
 import scala.util.Random
 
-val testInsert = new testInsert(new Random(0))
+val testInsert = new TestInsert(new Random(0))
 
 val unitmeasure = testInsert.productionUnitmeasure(UnitmeasureId("kgg"))
 val productCategory = testInsert.productionProductcategory()

diff --git a/site-in/patterns/multi-repo.md b/site-in/patterns/multi-repo.md
@@ -123,12 +123,12 @@ Here is example usage:
 
 Note that we can easily create a deep dependency graph with random data due to [testInsert](other-features/testing-with-random-values.md).
 ```scala mdoc:silent
-import adventureworks.{testInsert, withConnection}
+import adventureworks.{TestInsert, withConnection}
 import adventureworks.userdefined.FirstName
 import scala.util.Random
 
 // set a fixed seed to get consistent values
-val testInsert = new testInsert(new Random(1))
+val testInsert = new TestInsert(new Random(1))
 
 val businessentityRow = testInsert.personBusinessentity()
 val personRow = testInsert.personPerson(businessentityRow.businessentityid, FirstName("name"), persontype = "SC")

diff --git a/site-in/setup.md b/site-in/setup.md
@@ -65,15 +65,18 @@ val scriptsFolder = location.resolve("sql")
 // you can use this to customize which relations you want to generate code for, see below
 val selector = Selector.ExcludePostgresInternal
 
-generateFromDb(options, selector = selector, scriptsPaths = List(scriptsFolder))
-  .overwriteFolder(folder = targetDir)
+generateFromDb(options, folder = targetDir, selector = selector, scriptsPaths = List(scriptsFolder))
+  .overwriteFolder()
 
 // add changed files to git, so you can keep them under control
 //scala.sys.process.Process(List("git", "add", targetDir.toString)).!!
 ```
 
 ## `selector`
 You can customize which relations you generate code for, see [customize selected relations](customization/customize-selected-relations.md)
-## sbt plugin
 
+## `ProjectGraph`
+If you want to split the generated code across multiple projects in your build, have a look at [Generate code into multiple projects](other-features/generate-into-multiple-projects.md)
+
+## sbt plugin
 It's natural to think an sbt plugin would be a good match for Typo. This will likely be added in the future.
diff --git a/site/sidebars.js b/site/sidebars.js
@@ -51,6 +51,7 @@ const sidebars = {
             collapsed: false,
             items: [
                 {type: "doc", id: "other-features/streaming-inserts"},
+                {type: "doc", id: "other-features/generate-into-multiple-projects"},
                 {type: "doc", id: "other-features/json"},
                 {type: "doc", id: "other-features/faster-compilation"},
                 {type: "doc", id: "other-features/flexible"},

diff --git a/typo-scripts/src/scala/scripts/CompileBenchmark.scala b/typo-scripts/src/scala/scripts/CompileBenchmark.scala
@@ -63,9 +63,14 @@ object CompileBenchmark extends BleepScript("CompileBenchmark") {
               enableStreamingInserts = false
             ),
             metadb,
-            readSqlFileDirectories(TypoLogger.Noop, buildDir.resolve("adventureworks_sql")),
-            Selector.ExcludePostgresInternal // All
-          ).overwriteFolder(targetSources)
+            ProjectGraph(
+              name = "",
+              targetSources,
+              Selector.ExcludePostgresInternal, // All
+              readSqlFileDirectories(TypoLogger.Noop, buildDir.resolve("adventureworks_sql")),
+              Nil
+            )
+          ).foreach(_.overwriteFolder())
 
           crossIds.map { crossId =>
             started.projectPaths(CrossProjectName(ProjectName(projectName), Some(crossId))).sourcesDirs.fromSourceLayout.foreach { p =>

diff --git a/typo-scripts/src/scala/scripts/GenHardcodedFiles.scala b/typo-scripts/src/scala/scripts/GenHardcodedFiles.scala
@@ -143,7 +143,7 @@ object GenHardcodedFiles extends BleepCodegenScript("GenHardcodedFiles") {
         TypeMapperDb(enums, domains)
       )
 
-      val generated: Generated =
+      val generated: List[Generated] =
         generate(
           Options(
             pkg = "testdb.hardcoded",
@@ -160,14 +160,14 @@ object GenHardcodedFiles extends BleepCodegenScript("GenHardcodedFiles") {
             silentBanner = true
           ),
           metaDb,
-          sqlFiles = Nil,
-          Selector.All
+          ProjectGraph(name = "", target.sources, Selector.All, scripts = Nil, Nil)
         )
 
-      generated.overwriteFolder(
-        target.sources,
-        // todo: bleep should use something better than timestamps
-        softWrite = SoftWrite.No
+      generated.foreach(
+        _.overwriteFolder(
+          // todo: bleep should use something better than timestamps
+          softWrite = SoftWrite.No
+        )
       )
       cli("add to git", target.sources, List("git", "add", "-f", target.sources.toString), Logger.DevNull, cli.Out.Raw)
     }

diff --git a/typo-scripts/src/scala/scripts/GeneratedAdventureWorks.scala b/typo-scripts/src/scala/scripts/GeneratedAdventureWorks.scala
@@ -53,7 +53,7 @@ object GeneratedAdventureWorks {
             val targetSources = buildDir.resolve(s"$projectPath/generated-and-checked-in")
 
             val newFiles: Generated =
-              generate(options, metadb, newSqlScripts, Selector.ExcludePostgresInternal)
+              generate(options, metadb, ProjectGraph(name = "", targetSources, Selector.ExcludePostgresInternal, newSqlScripts, Nil)).head
 
             val knownUnchanged: Set[RelPath] = {
               val oldFiles = oldFilesRef.get()
@@ -62,7 +62,7 @@ object GeneratedAdventureWorks {
             oldFilesRef.set(newFiles.files)
 
             newFiles
-              .overwriteFolder(targetSources, softWrite = FileSync.SoftWrite.Yes(knownUnchanged))
+              .overwriteFolder(softWrite = FileSync.SoftWrite.Yes(knownUnchanged))
               .filter { case (_, synced) => synced != FileSync.Synced.Unchanged }
               .foreach { case (path, synced) => logger.withContext(path).warn(synced.toString) }
 

diff --git a/typo-scripts/src/scala/scripts/GeneratedSources.scala b/typo-scripts/src/scala/scripts/GeneratedSources.scala
@@ -38,6 +38,7 @@ object GeneratedSources {
         fileHeader = header,
         debugTypes = true
       ),
+      typoSources,
       Selector.relationNames(
         "columns",
         "key_column_usage",
@@ -51,7 +52,7 @@ object GeneratedSources {
       List(buildDir.resolve("sql"))
     )
 
-    files.overwriteFolder(typoSources)
+    files.overwriteFolder()
 
     import scala.sys.process.*
     List("git", "add", "-f", typoSources.toString).!!

diff --git a/typo-tester-anorm/generated-and-checked-in/adventureworks/testInsert.scala b/typo-tester-anorm/generated-and-checked-in/adventureworks/testInsert.scala
@@ -296,7 +296,7 @@ import java.time.LocalTime
 import java.time.ZoneOffset
 import scala.util.Random
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def humanresourcesDepartment(name: Name = Name(random.alphanumeric.take(20).mkString),
                                groupname: Name = Name(random.alphanumeric.take(20).mkString),
                                departmentid: Defaulted[DepartmentId] = Defaulted.UseDefault,

diff --git a/typo-tester-anorm/src/scala/adventureworks/RecordTest.scala b/typo-tester-anorm/src/scala/adventureworks/RecordTest.scala
@@ -14,7 +14,7 @@ class RecordTest extends AnyFunSuite with TypeCheckedTripleEquals {
 
   test("works") {
     withConnection { implicit c =>
-      val testInsert = new testInsert(new Random(0))
+      val testInsert = new TestInsert(new Random(0))
       val businessentityRow = testInsert.personBusinessentity()
       val personRow = testInsert.personPerson(businessentityRow.businessentityid, FirstName("a"), persontype = "EM")
       testInsert.personEmailaddress(personRow.businessentityid, Some("[email protected]")): @nowarn

diff --git a/typo-tester-anorm/src/scala/adventureworks/person/MultiRepoTest.scala b/typo-tester-anorm/src/scala/adventureworks/person/MultiRepoTest.scala
@@ -1,6 +1,6 @@
 package adventureworks.person
 
-import adventureworks.{testInsert, withConnection}
+import adventureworks.{TestInsert, withConnection}
 import adventureworks.person.address.*
 import adventureworks.person.addresstype.*
 import adventureworks.person.businessentityaddress.*
@@ -89,7 +89,7 @@ class PersonWithAddressesTest extends AnyFunSuite with TypeCheckedTripleEquals {
   test("works") {
     withConnection { implicit c =>
       // insert randomly generated rows (with a fixed seed) we base the test on
-      val testInsert = new testInsert(new Random(1))
+      val testInsert = new TestInsert(new Random(1))
       val businessentityRow = testInsert.personBusinessentity()
       val personRow = testInsert.personPerson(businessentityRow.businessentityid, FirstName("name"), persontype = "SC")
       val countryregionRow = testInsert.personCountryregion(CountryregionId("NOR"))

diff --git a/typo-tester-anorm/src/scala/adventureworks/production/product/ProductTest.scala b/typo-tester-anorm/src/scala/adventureworks/production/product/ProductTest.scala
@@ -7,7 +7,7 @@ import adventureworks.production.productmodel.*
 import adventureworks.production.productsubcategory.*
 import adventureworks.production.unitmeasure.*
 import adventureworks.public.{Flag, Name}
-import adventureworks.{testInsert, withConnection}
+import adventureworks.{TestInsert, withConnection}
 import org.scalactic.TypeCheckedTripleEquals
 import org.scalatest.Assertion
 import org.scalatest.funsuite.AnyFunSuite
@@ -30,7 +30,7 @@ class ProductTest extends AnyFunSuite with TypeCheckedTripleEquals {
 
   test("foo") {
     withConnection { implicit c =>
-      val testInsert = new testInsert(new Random(0))
+      val testInsert = new TestInsert(new Random(0))
       val unitmeasure = testInsert.productionUnitmeasure(UnitmeasureId("kgg"))
       val productCategory = testInsert.productionProductcategory()
       val productSubcategory = testInsert.productionProductsubcategory(productCategory.productcategoryid)

diff --git a/typo-tester-doobie/generated-and-checked-in/adventureworks/testInsert.scala b/typo-tester-doobie/generated-and-checked-in/adventureworks/testInsert.scala
@@ -296,7 +296,7 @@ import java.time.LocalTime
 import java.time.ZoneOffset
 import scala.util.Random
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def humanresourcesDepartment(name: Name = Name(random.alphanumeric.take(20).mkString),
                                groupname: Name = Name(random.alphanumeric.take(20).mkString),
                                departmentid: Defaulted[DepartmentId] = Defaulted.UseDefault,

diff --git a/typo-tester-zio-jdbc/generated-and-checked-in/adventureworks/testInsert.scala b/typo-tester-zio-jdbc/generated-and-checked-in/adventureworks/testInsert.scala
@@ -297,7 +297,7 @@ import scala.util.Random
 import zio.ZIO
 import zio.jdbc.ZConnection
 
-class testInsert(random: Random) {
+class TestInsert(random: Random) {
   def humanresourcesDepartment(name: Name = Name(random.alphanumeric.take(20).mkString),
                                groupname: Name = Name(random.alphanumeric.take(20).mkString),
                                departmentid: Defaulted[DepartmentId] = Defaulted.UseDefault,