-
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use db as source of truth for repository metadata
The current behavior of S3PublishService when publishing an edit is to fetch the old repository metadata from the S3 bucket, patch it with new data, then republish the patch metadata over the old metadata. The problem with this approach is that it requires unnecessary data fetches and doesn't maintain a well-defined source of truth for the repository. Instead, store generated repository metadata in the database and patch _that_ when publishing edits so that we can consistently source the repository from a single location. The data is relatively small, so storing it in the database shouldn't have a significant performance impact. If we really need to in the future, we can store the metadata as files in a separate storage bucket. For consistency, additionally update the database repository metadata when publishing app updates.
- Loading branch information
1 parent
386720e
commit ce76e9d
Showing
8 changed files
with
148 additions
and
36 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
65 changes: 65 additions & 0 deletions
65
console/src/main/kotlin/db/migration/V10__Save_repodata_to_database.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| // Copyright 2024 Logan Magee | ||
| // | ||
| // SPDX-License-Identifier: AGPL-3.0-only | ||
|
|
||
| package db.migration | ||
|
|
||
| import app.accrescent.parcelo.console.Config | ||
| import app.accrescent.parcelo.console.data.App | ||
| import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider | ||
| import aws.sdk.kotlin.services.s3.S3Client | ||
| import aws.sdk.kotlin.services.s3.model.GetObjectRequest | ||
| import aws.smithy.kotlin.runtime.content.toInputStream | ||
| import aws.smithy.kotlin.runtime.net.url.Url | ||
| import kotlinx.coroutines.runBlocking | ||
| import org.flywaydb.core.api.migration.BaseJavaMigration | ||
| import org.flywaydb.core.api.migration.Context | ||
| import org.jetbrains.exposed.sql.statements.api.ExposedBlob | ||
| import org.jetbrains.exposed.sql.transactions.transaction | ||
| import org.koin.java.KoinJavaComponent.inject | ||
| import java.io.FileNotFoundException | ||
|
|
||
| /** | ||
| * A versioned migration which saves published repository to the database. | ||
| */ | ||
| class V10__Save_repodata_to_database : BaseJavaMigration() { | ||
| private val config: Config by inject(Config::class.java) | ||
|
|
||
| /** | ||
| * Downloads all published repository metadata, saving it to the database alongside its | ||
| * associated app. | ||
| */ | ||
| override fun migrate(context: Context) { | ||
| val oldRepoDataReqs = transaction { App.all().map { it.id.value } } | ||
| .map { appId -> | ||
| val req = GetObjectRequest { | ||
| bucket = config.s3.bucket | ||
| key = "apps/$appId/repodata.json" | ||
| } | ||
| Pair(appId, req) | ||
| } | ||
| S3Client { | ||
| endpointUrl = Url.parse(config.s3.endpointUrl) | ||
| region = config.s3.region | ||
| credentialsProvider = StaticCredentialsProvider { | ||
| accessKeyId = config.s3.accessKeyId | ||
| secretAccessKey = config.s3.secretAccessKey | ||
| } | ||
| }.use { s3Client -> | ||
| oldRepoDataReqs.forEach { (appId, req) -> | ||
| runBlocking { | ||
| s3Client.getObject(req) { resp -> | ||
| val data = resp.body | ||
| ?.toInputStream() | ||
| ?.use { it.readBytes() } | ||
| ?: throw FileNotFoundException() | ||
|
|
||
| transaction { | ||
| App.findById(appId)?.repositoryMetadata = ExposedBlob(data) | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
17 changes: 17 additions & 0 deletions
17
console/src/main/resources/db/migration/V11__Make_repository_metadata_not_null.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| -- Copyright 2024 Logan Magee | ||
| -- | ||
| -- SPDX-License-Identifier: AGPL-3.0-only | ||
|
|
||
| -- Add a NOT NULL constraint to the repository_metadata column | ||
| PRAGMA foreign_keys = OFF; | ||
|
|
||
| -- The previous apps table with the repository_metadata column made NOT NULL | ||
| CREATE TABLE apps2 (id TEXT NOT NULL PRIMARY KEY, version_code INT NOT NULL, version_name TEXT NOT NULL, file_id INT NOT NULL, review_issue_group_id INT NULL, updating BOOLEAN DEFAULT 0 NOT NULL, repository_metadata NOT NULL, CONSTRAINT fk_apps_file_id__id FOREIGN KEY (file_id) REFERENCES files(id) ON UPDATE RESTRICT, CONSTRAINT fk_apps_review_issue_group_id__id FOREIGN KEY (review_issue_group_id) REFERENCES review_issue_groups(id) ON UPDATE RESTRICT); | ||
|
|
||
| INSERT INTO apps2 SELECT * FROM apps; | ||
| DROP TABLE apps; | ||
| ALTER TABLE apps2 RENAME TO apps; | ||
|
|
||
| PRAGMA foreign_key_check; | ||
|
|
||
| PRAGMA foreign_keys = ON; |
5 changes: 5 additions & 0 deletions
5
console/src/main/resources/db/migration/V9__Add_repository_metadata_field_to_app.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| -- Copyright 2024 Logan Magee | ||
| -- | ||
| -- SPDX-License-Identifier: AGPL-3.0-only | ||
|
|
||
| ALTER TABLE apps ADD COLUMN repository_metadata BLOB; |