Support releasing multiple SDKs (#109)

Per [b/330880975](https://b.corp.google.com/issues/330880975),

This updates our release infrastructure to support our new multi project
repo (with `common` and `generativeai` living next to one another). Now,
we can release either in separation of the other.

This PR also includes the necessary changes to various tasks, plugins,
workflows, tools, and even documentation.

More specifically, this PR fixes the following:

- [b/332890790](https://b.corp.google.com/issues/332890790) -> Implement
a task to generate collective release notes
- [b/332891281](https://b.corp.google.com/issues/332891281) -> Implement
a task to generative collective api changes
- [b/332891563](https://b.corp.google.com/issues/332891563) -> Provide a
means for SDKs to generate their release artifacts in isolation of one
another
- [b/332892199](https://b.corp.google.com/issues/332892199) -> Separate
api changes according to sdk
- [b/332892499](https://b.corp.google.com/issues/332892499) -> Implement
a task to release all sdks with changes
- [b/332893058](https://b.corp.google.com/issues/332893058) -> Update
the api changes workflow to work for multiple sdks
- [b/332894984](https://b.corp.google.com/issues/332894984) -> Update
the build release workflow to work for multiple sdks
- [b/332897987](https://b.corp.google.com/issues/332897987) -> Update
DEVELOPING to reflect the multiple sdk structure
- [b/332898189](https://b.corp.google.com/issues/332898189) -> Update
change to allow specifying projects

.changes/generativeai/baseball-comfort-dock-appliance.json

@@ -0,0 +1 @@
+{"type":"MAJOR","changes":["Make totalTokens in CountTokensResponse nullable"]}

.changes/generativeai/cloud-camp-bait-calculator.json

@@ -0,0 +1 @@
+{"type":"MAJOR","changes":["Add function calling"]}

.changes/generativeai/crowd-birthday-drink-circle.json

@@ -0,0 +1 @@
+{"type":"PATCH","changes":["Implement error catching for unsupported Part types."]}

.github/workflows/build_release.yml

@@ -26,11 +26,27 @@ jobs:
           distribution: 'temurin'
           java-version: '17'
           cache: gradle
-      - name: Perform a Gradle build
+      - name: Build the generativeai release artifacts
         run: ./gradlew generativeai:publishAllPublicationsToMavenRepository
-      - name: Upload generated artifacts
+      - name: Upload the generativeai artifacts
         uses: actions/upload-artifact@v2
         with:
           name: generative-ai-android
           path: generativeai/m2
           retention-days: 5
+      - name: Build the common release artifacts
+        run: ./gradlew common:publishAllPublicationsToMavenRepository
+      - name: Upload the common artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: common-artifacts
+          path: common/m2
+          retention-days: 5
+      - name: Build the release artifacts for all sdks
+        run: ./gradlew publishAllPublicationsToMavenRepository
+      - name: Upload the generated artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: release-artifacts
+          path: m2
+          retention-days: 5

.github/workflows/check_for_api_changes.yml

@@ -20,10 +20,12 @@ jobs:
 
       - name: Generate the api file
         run: |
-          ./gradlew generativeai:exportApi --no-daemon
+          ./gradlew exportApi --no-daemon
 
       - name: Save public.api from master
-        run: mv generativeai/public.api ~/public.api
+        run: |
+          mv generativeai/public.api ~/generativeai/public.api && \
+          mv common/public.api ~/common/public.api
 
       - name: Checkout branch
         uses: actions/[email protected]
@@ -36,11 +38,13 @@ jobs:
           cache: gradle
 
       - name: Copy saved api to branch
-        run: mv ~/public.api generativeai/public.api
+        run: |
+          mv ~/generativeai/public.api generativeai/public.api && \
+          mv ~/common/public.api common/public.api
 
       - name: Run api warning task
         run: |
-          ./gradlew generativeai:warnAboutApiChanges --no-daemon
+          ./gradlew warnAboutApiChanges --no-daemon
 
       - name: Add PR Comment
         if: ${{ hashFiles('api_changes.md') != '' }}

DEVELOPING.md

@@ -15,15 +15,15 @@ hooks, run the following command at the root of your repo directory:
 
 To locally publish the m2 repo:
 
-`./gradlew generativeai:publishToMavenLocal`
+`./gradlew publishToMavenLocal`
 
 To generate a releasable m2 repo:
 
-`./gradlew generativeai:publishAllPublicationsToMavenRepository`
+`./gradlew publishAllPublicationsToMavenRepository`
 
-The m2 repo will be in `generativeai/m2`.
+The m2 repo will be in `/m2`.
 
-To generate Dokka:
+To generate Dokka for the generativeai project:
 
 `./gradlew generativeai:dokkaHtml`
 
@@ -60,6 +60,20 @@ change
 Both of these commands should generate a new file under the `.changes` directory with your message
 and impact.
 
+Note though, that this will naturally create release notes for *all* projects. If you only want
+to generate a change for the `generativeai` project, then prefix your message with the project name:
 
-To learn more, read the section on our changelog plugin in our [plugin readme](./plugins/README.md).
+```bash
+change generativeai "hello world!"
+```
 
+You can also do it for other projects as well, just supply the project name:
+
+```bash
+change common "hello world!"
+```
+
+Although, this does *not* support creating empty changes. You'll have to make that change manually,
+if that's what you want.
+
+To learn more, read the section on our changelog plugin in our [plugin readme](./plugins/README.md).

build.gradle.kts

@@ -21,6 +21,7 @@ plugins {
     kotlin("plugin.serialization") version "1.8.22" apply false
     id("com.ncorti.ktfmt.gradle") version "0.16.0" apply false
     id("license-plugin")
+    id("multi-project-plugin")
 }
 
 license {

change

@@ -17,6 +17,10 @@ if [ $# -eq 0 ]; then
     ./gradlew makeChange
 elif [ "$1" == "release" ] || [ "$1" == "compile" ]; then
     ./gradlew makeReleaseNotes
+elif [ $# -eq 2 ]; then
+    PROJECT="$1"
+    MESSAGE="$2"
+    ./gradlew ${PROJECT}:makeChange -PchangeMessage="${MESSAGE}"
 else
     MESSAGE="$1"
     ./gradlew makeChange -PchangeMessage="${MESSAGE}"

plugins/README.md

@@ -1,15 +1,17 @@
 # Plugins
 
-This composite module houses the gradle plugins that are used in the main SDK; to assist in a 
-number of monotonous tasks. You can read more on these plugins and the tasks they create below.
+This composite module houses the gradle plugins that are used in all the SDKs in this repo; to 
+assist in a number of monotonous tasks. You can read more on these plugins and the tasks they 
+create below.
 
 ## ChangelogPlugin
 
-Creates and manages changelog files. These files are used to signify changes made to the repo that 
+Creates and manages changelog files. These files are used to signify changes made to a project that 
 should invoke a release, alongside text to display in the release notes at release time.
 
-Change files are (by default) created under the `.changes` directory at the root of the repo. 
-These files are json encoded variants of a [Changelog](./src/main/java/com/google/gradle/types/Changelog.kt) instance- 
+Change files are (by default) created under the `.changes` directory at the root of the repo. They
+are then further filtered into sub directories according to the project name. These files are json 
+encoded variants of a [Changelog](./src/main/java/com/google/gradle/types/Changelog.kt) instance- 
 which is just an organization of what impact the change had (will it invoke a patch, minor, or 
 major bump?) and an (optional) end-user readable message to show alongside the other changes at 
 release time. By default, the files are saved as a random sequence of four words (to avoid 
@@ -38,27 +40,27 @@ generated by the `buildApi` task to a file at the project directory named `publi
 
 ### Tasks
 
-The ChangelogPlugin registers the four following tasks:
+The ChangelogPlugin registers the five following tasks:
 
 - [findChanges](./src/main/java/com/google/gradle/tasks/FindChangesTask.kt) -> Creates a new `.api` 
-file for the current project, and compares it to the existing `released.api` to create a diff of 
-changes made to the public api.
+file for the current project, and compares it to the latest `.api` file in the root `api` directory- 
+to create a diff of changes made to the public api.
 - [makeChange](./src/main/java/com/google/gradle/tasks/MakeChangeTask.kt) -> Creates a new `.json` 
-file in the `.changes` directory with an optional message, including the version impact inferred 
-from `findChanges`.
+file in the root `.changes` directory (under the subdirectory for this project) with an optional 
+message, including the version impact inferred from `findChanges`.
 - [warnAboutApiChanges](./src/main/java/com/google/gradle/tasks/WarnAboutApiChangesTask.kt) -> If 
 `findChanges` finds changes that will have version implications, this task will create a 
-`api_changes.md` in the root project's build directory, with a message to display to the developer 
-regarding what they changed, and the impact it will have on the public API. If no public API changes 
-are found, this task will just skip itself.
+markdown file corresponding to the project's name in the root project's build directory under 
+`api_changes`, with a message to display to the developer regarding what they changed, and the 
+impact it will have on the public API. If no public API changes are found, this task will just skip 
+itself.
 - [deleteChangeFiles](./src/main/java/com/google/gradle/tasks/MakeReleaseNotesTask.kt) -> Deletes 
 all the change files in the `.changes` directory, intended to be used after the release notes have 
 been created- or otherwise during a cleanup stage of the release.
 - [makeReleaseNotes](./src/main/java/com/google/gradle/tasks/MakeReleaseNotesTask.kt) -> Creates a 
-`release_notes.md` in the root project's build directory, containing a collection of all the changes 
-in the `.changes` directory; including the new project version at the top of file (inferred from the 
-highest impact change).
-
+markdown file corresponding to the project's name in the root project's build directory under
+`release_notes`, containing a collection of all the changes in the `.changes` directory; including 
+the new project version at the top of file (inferred from the highest impact change).
 
 ## LicensePlugin
 
@@ -91,3 +93,21 @@ notes.
 `.api` file in the root `api` directory; aligning with the current state of the public api.
 - [prepareRelease](./src/main/java/com/google/gradle/plugins/ReleasePlugin.kt) -> Does everything 
 needed to prepare a release; creates the release notes and runs the above tasks.
+
+## MultiProjectPlugin
+
+Works in conjunction with the `ReleasePlugin` to facilitate a release for all SDKs with changes.
+
+### Tasks
+
+The MultiProjectPlugin registers the three following tasks:
+
+- [warnAboutApiChanges](./src/main/java/com/google/gradle/tasks/CombineApiChangesTask.kt) -> Runs
+the `warnAboutApiChanges` task from the `ChangelogPlugin` on each SDK, and then combines their
+changes into a single file under the root project's build directory called `api_changes.md`.
+- [makeReleaseNotes](./src/main/java/com/google/gradle/tasks/CombineReleaseNotesTask.kt) -> Runs
+the `makeReleaseNotes` task from the `ChangelogPlugin` on each SDK, and then combines their
+release notes into a single file under the root project's build directory called `release_notes.md`.
+- [prepareRelease](./src/main/java/com/google/gradle/plugins/MultiProjectPlugin.kt) -> Does 
+everything needed to prepare a release for all SDKs; creates the combined release notes and runs the 
+above tasks.

plugins/build.gradle.kts

@@ -49,6 +49,10 @@ gradlePlugin {
             id = "release-plugin"
             implementationClass = "com.google.gradle.plugins.ReleasePlugin"
         }
+        register("multi-project-plugin") {
+            id = "multi-project-plugin"
+            implementationClass = "com.google.gradle.plugins.MultiProjectPlugin"
+        }
     }
 }
 

plugins/src/main/java/com/google/gradle/plugins/ApiPlugin.kt

@@ -26,7 +26,6 @@ import kotlinx.validation.KotlinApiBuildTask
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.provider.Property
-import org.gradle.api.tasks.Copy
 import org.gradle.api.tasks.Optional
 import org.gradle.kotlin.dsl.create
 import org.gradle.kotlin.dsl.register
@@ -69,7 +68,7 @@ abstract class ApiPlugin : Plugin<Project> {
 
   context(Project)
   private fun ApiPluginExtension.commonConfiguration() {
-    val latestApiFile = rootProject.file("api/${project.version}.api")
+    val latestApiFile = rootProject.file("api/${project.name}/${project.version}.api")
 
     apiFile.convention(latestApiFile)
   }

plugins/src/main/java/com/google/gradle/plugins/ChangelogPlugin.kt

@@ -25,18 +25,17 @@ import com.google.gradle.types.RandomWordsGenerator
 import com.google.gradle.util.apply
 import com.google.gradle.util.buildDir
 import com.google.gradle.util.childFile
+import com.google.gradle.util.file
 import com.google.gradle.util.moduleVersion
 import com.google.gradle.util.orElseIfNotExists
 import com.google.gradle.util.outputFile
 import com.google.gradle.util.provideProperty
 import com.google.gradle.util.tempFile
-import java.io.File
 import org.gradle.api.Plugin
 import org.gradle.api.Project
-import org.gradle.api.provider.Property
+import org.gradle.api.file.RegularFileProperty
 import org.gradle.api.tasks.Delete
 import org.gradle.api.tasks.Optional
-import org.gradle.kotlin.dsl.apply
 import org.gradle.kotlin.dsl.create
 import org.gradle.kotlin.dsl.getByType
 import org.gradle.kotlin.dsl.register
@@ -45,12 +44,13 @@ import org.gradle.kotlin.dsl.support.listFilesOrdered
 /**
  * A Gradle plugin for managing and creating [Changelog] files.
  *
- * By default, the root `.changes` directory will be used to save change files.
+ * By default, a subdirectory under the root `.changes` directory that corresponds to the
+ * [project name][Project.getName] will be used to save change files.
  *
  * Will also register the [ApiPlugin] if it's not already present, as it will handle the actual
  * generation of the API files.
  *
- * Registers four tasks:
+ * Registers the following tasks:
  * - `findChanges`
  * - `makeChange`
  * - `warnAboutApiChanges`
@@ -87,7 +87,7 @@ abstract class ChangelogPlugin : Plugin<Project> {
       tasks.register<MakeChangeTask>("makeChange") {
         val changeMessage = provideProperty<String>("changeMessage")
         val changeName = RandomWordsGenerator.generateString()
-        val changeOutput = extension.outputDirectory.childFile("$changeName.json")
+        val changeOutput = extension.outputDirectory.asFile.childFile("$changeName.json")
 
         changesFile.set(fileChanges)
         message.set(changeMessage)
@@ -96,11 +96,11 @@ abstract class ChangelogPlugin : Plugin<Project> {
 
       tasks.register<WarnAboutApiChangesTask>("warnAboutApiChanges") {
         changesFile.set(fileChanges)
-        outputFile.set(rootProject.buildDir("api_changes.md"))
+        outputFile.set(extension.apiChangesDirectory.asFile)
       }
 
       val changelogFiles =
-        extension.outputDirectory.map { it.listFilesOrdered { it.extension == "json" } }
+        extension.outputDirectory.map { it.asFile.listFilesOrdered { it.extension == "json" } }
 
       val deleteChangeFiles =
         tasks.register<Delete>("deleteChangeFiles") {
@@ -114,7 +114,8 @@ abstract class ChangelogPlugin : Plugin<Project> {
 
         changeFiles.set(changelogFiles)
         version.set(project.moduleVersion)
-        outputFile.set(rootProject.buildDir("release_notes.md"))
+        // TODO() move to extension config with convention (like .changes)
+        outputFile.set(extension.releaseNotesDirectory)
 
         finalizedBy(deleteChangeFiles)
       }
@@ -123,17 +124,23 @@ abstract class ChangelogPlugin : Plugin<Project> {
 
   context(Project)
   private fun ChangelogPluginExtension.commonConfiguration() {
-    outputDirectory.convention(rootProject.file(".changes"))
+    outputDirectory.convention(rootProject.layout.file(".changes/${project.name}"))
+    releaseNotesDirectory.convention(rootProject.buildDir("release_notes/${project.name}.md"))
+    apiChangesDirectory.convention(rootProject.buildDir("api_changes/${project.name}.md"))
   }
 }
 
 /**
  * Extension properties for the [ChangelogPlugin].
  *
  * @property outputDirectory The directory into which to store the [Changelog] files
+ * @property releaseNotesDirectory The directory into which to store the release notes file
+ * @property apiChangesDirectory The directory into which to store the api changes warning file
  */
 abstract class ChangelogPluginExtension {
-  @get:Optional abstract val outputDirectory: Property<File>
+  @get:Optional abstract val outputDirectory: RegularFileProperty
+  @get:Optional abstract val releaseNotesDirectory: RegularFileProperty
+  @get:Optional abstract val apiChangesDirectory: RegularFileProperty
 }
 
 /**