diff --git a/console/src/main/kotlin/app/accrescent/parcelo/console/data/App.kt b/console/src/main/kotlin/app/accrescent/parcelo/console/data/App.kt index f5cb0ed1..474fd143 100644 --- a/console/src/main/kotlin/app/accrescent/parcelo/console/data/App.kt +++ b/console/src/main/kotlin/app/accrescent/parcelo/console/data/App.kt @@ -20,6 +20,7 @@ object Apps : IdTable("apps") { val reviewIssueGroupId = reference("review_issue_group_id", ReviewIssueGroups, ReferenceOption.NO_ACTION).nullable() val updating = bool("updating").default(false) + val repositoryMetadata = blob("repository_metadata") override val primaryKey = PrimaryKey(id) } @@ -31,6 +32,7 @@ class App(id: EntityID) : Entity(id), ToSerializable + val metadata = storageService.loadFile(draft.fileId).use { draftStream -> storageService.loadFile(iconFileId).use { iconStream -> runBlocking { publishService.publishDraft(draftStream, iconStream, draft.shortDescription) @@ -45,6 +46,7 @@ fun registerPublishAppJob(draftId: UUID) { versionName = draft.versionName fileId = draft.fileId reviewIssueGroupId = draft.reviewIssueGroupId + repositoryMetadata = ExposedBlob(metadata) } Listing.new { appId = app.id @@ -72,7 +74,7 @@ fun registerPublishUpdateJob(updateId: UUID) { val update = transaction { UpdateDao.findById(updateId) } ?: return // Publish to the repository - storageService.loadFile(update.fileId).use { + val updatedMetadata = storageService.loadFile(update.fileId).use { runBlocking { publishService.publishUpdate(it, update.appId.value) } } @@ -81,6 +83,7 @@ fun registerPublishUpdateJob(updateId: UUID) { App.findById(update.appId)?.run { versionCode = update.versionCode versionName = update.versionName + repositoryMetadata = ExposedBlob(updatedMetadata) val oldAppFileId = fileId fileId = update.fileId diff --git a/console/src/main/kotlin/app/accrescent/parcelo/console/jobs/PublishEdit.kt b/console/src/main/kotlin/app/accrescent/parcelo/console/jobs/PublishEdit.kt index 4ccb7b1b..627c8e3c 100644 --- a/console/src/main/kotlin/app/accrescent/parcelo/console/jobs/PublishEdit.kt +++ b/console/src/main/kotlin/app/accrescent/parcelo/console/jobs/PublishEdit.kt @@ -11,6 +11,7 @@ import app.accrescent.parcelo.console.data.Listings import app.accrescent.parcelo.console.publish.PublishService import kotlinx.coroutines.runBlocking import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.statements.api.ExposedBlob import org.jetbrains.exposed.sql.transactions.transaction import org.koin.java.KoinJavaComponent.inject import java.util.UUID @@ -24,11 +25,15 @@ fun publishEdit(editId: UUID) { val edit = transaction { EditDao.findById(editId) } ?: return // Publish to the repository - runBlocking { publishService.publishEdit(edit.appId.value, edit.shortDescription) } + val updatedMetadata = + runBlocking { publishService.publishEdit(edit.appId.value, edit.shortDescription) } // Account for publication transaction { - App.findById(edit.appId)?.run { updating = false } + App.findById(edit.appId)?.run { + repositoryMetadata = ExposedBlob(updatedMetadata) + updating = false + } Listing .find { Listings.appId eq edit.appId and (Listings.locale eq "en-US") } .singleOrNull() diff --git a/console/src/main/kotlin/app/accrescent/parcelo/console/publish/PublishService.kt b/console/src/main/kotlin/app/accrescent/parcelo/console/publish/PublishService.kt index 9769aa88..4e7149ed 100644 --- a/console/src/main/kotlin/app/accrescent/parcelo/console/publish/PublishService.kt +++ b/console/src/main/kotlin/app/accrescent/parcelo/console/publish/PublishService.kt @@ -13,16 +13,26 @@ import java.io.InputStream interface PublishService { /** * Publishes a draft + * + * @return the repository metadata */ - suspend fun publishDraft(apkSet: InputStream, icon: InputStream, shortDescription: String) + suspend fun publishDraft( + apkSet: InputStream, + icon: InputStream, + shortDescription: String, + ): ByteArray /** * Publishes an app update + * + * @return the updated repository metadata */ - suspend fun publishUpdate(apkSet: InputStream, appId: String) + suspend fun publishUpdate(apkSet: InputStream, appId: String): ByteArray /** * Publishes an edit + * + * @return the updated repository metadata */ - suspend fun publishEdit(appId: String, shortDescription: String?) + suspend fun publishEdit(appId: String, shortDescription: String?): ByteArray } diff --git a/console/src/main/kotlin/app/accrescent/parcelo/console/publish/S3PublishService.kt b/console/src/main/kotlin/app/accrescent/parcelo/console/publish/S3PublishService.kt index f44cfdcc..b5fe5b37 100644 --- a/console/src/main/kotlin/app/accrescent/parcelo/console/publish/S3PublishService.kt +++ b/console/src/main/kotlin/app/accrescent/parcelo/console/publish/S3PublishService.kt @@ -6,6 +6,7 @@ package app.accrescent.parcelo.console.publish import app.accrescent.parcelo.apksparser.ApkSet import app.accrescent.parcelo.apksparser.ParseApkSetResult +import app.accrescent.parcelo.console.data.App import app.accrescent.parcelo.console.data.Listing import app.accrescent.parcelo.console.data.Listings import app.accrescent.parcelo.console.repo.RepoData @@ -14,12 +15,10 @@ import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider import aws.sdk.kotlin.services.s3.S3Client import aws.sdk.kotlin.services.s3.model.Delete import aws.sdk.kotlin.services.s3.model.DeleteObjectsRequest -import aws.sdk.kotlin.services.s3.model.GetObjectRequest import aws.sdk.kotlin.services.s3.model.ListObjectsRequest import aws.sdk.kotlin.services.s3.model.ObjectIdentifier import aws.sdk.kotlin.services.s3.model.PutObjectRequest import aws.smithy.kotlin.runtime.content.ByteStream -import aws.smithy.kotlin.runtime.content.toInputStream import aws.smithy.kotlin.runtime.net.url.Url import com.android.bundle.Targeting import io.ktor.utils.io.core.toByteArray @@ -29,8 +28,8 @@ import kotlinx.serialization.encodeToString import kotlinx.serialization.json.Json import kotlinx.serialization.json.decodeFromStream import org.jetbrains.exposed.sql.and +import org.jetbrains.exposed.sql.statements.api.ExposedBlob import org.jetbrains.exposed.sql.transactions.transaction -import java.io.FileNotFoundException import java.io.InputStream import java.util.zip.ZipFile @@ -68,7 +67,7 @@ class S3PublishService( apkSet: InputStream, icon: InputStream, shortDescription: String, - ) { + ): ByteArray { TempFile().use { tempApkSet -> tempApkSet.outputStream().use { apkSet.copyTo(it) } @@ -78,7 +77,7 @@ class S3PublishService( is ParseApkSetResult.Error -> throw Exception("APK set parsing failed") } - publish( + return publish( ZipFile(tempApkSet.path.toFile()), metadata, PublicationType.NewApp(icon, shortDescription), @@ -86,7 +85,7 @@ class S3PublishService( } } - override suspend fun publishUpdate(apkSet: InputStream, appId: String) { + override suspend fun publishUpdate(apkSet: InputStream, appId: String): ByteArray { TempFile().use { tempApkSet -> tempApkSet.outputStream().use { apkSet.copyTo(it) } @@ -96,16 +95,12 @@ class S3PublishService( is ParseApkSetResult.Error -> throw Exception("APK set parsing failed") } - publish(ZipFile(tempApkSet.path.toFile()), metadata, PublicationType.Update) + return publish(ZipFile(tempApkSet.path.toFile()), metadata, PublicationType.Update) } } @OptIn(ExperimentalSerializationApi::class) - override suspend fun publishEdit(appId: String, shortDescription: String?) { - val getOldDataReq = GetObjectRequest { - bucket = s3Bucket - key = "apps/$appId/repodata.json" - } + override suspend fun publishEdit(appId: String, shortDescription: String?): ByteArray { S3Client { endpointUrl = s3EndpointUrl region = s3Region @@ -114,32 +109,40 @@ class S3PublishService( secretAccessKey = s3SecretAccessKey } }.use { s3Client -> - val newRepoData = s3Client.getObject(getOldDataReq) { resp -> - val oldRepoData = - resp.body?.toInputStream()?.use { Json.decodeFromStream(it) } - ?: throw FileNotFoundException() - RepoData( - version = oldRepoData.version, - versionCode = oldRepoData.versionCode, - abiSplits = oldRepoData.abiSplits, - densitySplits = oldRepoData.densitySplits, - langSplits = oldRepoData.langSplits, - shortDescription = shortDescription ?: oldRepoData.shortDescription - ) - } + // Fetch the old app metadata from the database + val app = transaction { App.findById(appId) } ?: throw Exception("app not found") + val oldRepoData = app.repositoryMetadata.inputStream + .use { Json.decodeFromStream(it) } + + // Modify the old app metadata to produce the new app metadata + val newRepoData = RepoData( + version = oldRepoData.version, + versionCode = oldRepoData.versionCode, + abiSplits = oldRepoData.abiSplits, + densitySplits = oldRepoData.densitySplits, + langSplits = oldRepoData.langSplits, + shortDescription = shortDescription ?: oldRepoData.shortDescription + ).let { Json.encodeToString(it) }.toByteArray() + // Publish the new app metadata val updateDataReq = PutObjectRequest { bucket = s3Bucket key = "apps/$appId/repodata.json" - body = ByteStream.fromString(Json.encodeToString(newRepoData)) + body = ByteStream.fromBytes(newRepoData) } s3Client.putObject(updateDataReq) + + return newRepoData } } // Note that APKs which target multiple ABIs, multiple languages, or multiple screen densities // are not currently supported - private suspend fun publish(apkSetZip: ZipFile, metadata: ApkSet, type: PublicationType) { + private suspend fun publish( + apkSetZip: ZipFile, + metadata: ApkSet, + type: PublicationType, + ): ByteArray { val appId = metadata.metadata.packageName S3Client { @@ -273,12 +276,12 @@ class S3PublishService( densitySplits = densitySplits, shortDescription = shortDescription, ) - val repoDataString = Json.encodeToString(repoData) + val repoDataBytes = Json.encodeToString(repoData).toByteArray() val publishRepoDataRequest = PutObjectRequest { bucket = s3Bucket key = "apps/$appId/repodata.json" - body = ByteStream.fromBytes(repoDataString.toByteArray()) + body = ByteStream.fromBytes(repoDataBytes) } s3Client.putObject(publishRepoDataRequest) @@ -306,6 +309,8 @@ class S3PublishService( s3Client.deleteObjects(deleteRequest) } } + + return repoDataBytes } } } diff --git a/console/src/main/kotlin/db/migration/V10__Save_repodata_to_database.kt b/console/src/main/kotlin/db/migration/V10__Save_repodata_to_database.kt new file mode 100644 index 00000000..f61105f7 --- /dev/null +++ b/console/src/main/kotlin/db/migration/V10__Save_repodata_to_database.kt @@ -0,0 +1,65 @@ +// Copyright 2024 Logan Magee +// +// SPDX-License-Identifier: AGPL-3.0-only + +package db.migration + +import app.accrescent.parcelo.console.Config +import app.accrescent.parcelo.console.data.App +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.services.s3.S3Client +import aws.sdk.kotlin.services.s3.model.GetObjectRequest +import aws.smithy.kotlin.runtime.content.toInputStream +import aws.smithy.kotlin.runtime.net.url.Url +import kotlinx.coroutines.runBlocking +import org.flywaydb.core.api.migration.BaseJavaMigration +import org.flywaydb.core.api.migration.Context +import org.jetbrains.exposed.sql.statements.api.ExposedBlob +import org.jetbrains.exposed.sql.transactions.transaction +import org.koin.java.KoinJavaComponent.inject +import java.io.FileNotFoundException + +/** + * A versioned migration which saves published repository to the database. + */ +class V10__Save_repodata_to_database : BaseJavaMigration() { + private val config: Config by inject(Config::class.java) + + /** + * Downloads all published repository metadata, saving it to the database alongside its + * associated app. + */ + override fun migrate(context: Context) { + val oldRepoDataReqs = transaction { App.all().map { it.id.value } } + .map { appId -> + val req = GetObjectRequest { + bucket = config.s3.bucket + key = "apps/$appId/repodata.json" + } + Pair(appId, req) + } + S3Client { + endpointUrl = Url.parse(config.s3.endpointUrl) + region = config.s3.region + credentialsProvider = StaticCredentialsProvider { + accessKeyId = config.s3.accessKeyId + secretAccessKey = config.s3.secretAccessKey + } + }.use { s3Client -> + oldRepoDataReqs.forEach { (appId, req) -> + runBlocking { + s3Client.getObject(req) { resp -> + val data = resp.body + ?.toInputStream() + ?.use { it.readBytes() } + ?: throw FileNotFoundException() + + transaction { + App.findById(appId)?.repositoryMetadata = ExposedBlob(data) + } + } + } + } + } + } +} diff --git a/console/src/main/resources/db/migration/V11__Make_repository_metadata_not_null.sql b/console/src/main/resources/db/migration/V11__Make_repository_metadata_not_null.sql new file mode 100644 index 00000000..c2a90453 --- /dev/null +++ b/console/src/main/resources/db/migration/V11__Make_repository_metadata_not_null.sql @@ -0,0 +1,17 @@ +-- Copyright 2024 Logan Magee +-- +-- SPDX-License-Identifier: AGPL-3.0-only + +-- Add a NOT NULL constraint to the repository_metadata column +PRAGMA foreign_keys = OFF; + +-- The previous apps table with the repository_metadata column made NOT NULL +CREATE TABLE apps2 (id TEXT NOT NULL PRIMARY KEY, version_code INT NOT NULL, version_name TEXT NOT NULL, file_id INT NOT NULL, review_issue_group_id INT NULL, updating BOOLEAN DEFAULT 0 NOT NULL, repository_metadata NOT NULL, CONSTRAINT fk_apps_file_id__id FOREIGN KEY (file_id) REFERENCES files(id) ON UPDATE RESTRICT, CONSTRAINT fk_apps_review_issue_group_id__id FOREIGN KEY (review_issue_group_id) REFERENCES review_issue_groups(id) ON UPDATE RESTRICT); + +INSERT INTO apps2 SELECT * FROM apps; +DROP TABLE apps; +ALTER TABLE apps2 RENAME TO apps; + +PRAGMA foreign_key_check; + +PRAGMA foreign_keys = ON; diff --git a/console/src/main/resources/db/migration/V9__Add_repository_metadata_field_to_app.sql b/console/src/main/resources/db/migration/V9__Add_repository_metadata_field_to_app.sql new file mode 100644 index 00000000..959ad47e --- /dev/null +++ b/console/src/main/resources/db/migration/V9__Add_repository_metadata_field_to_app.sql @@ -0,0 +1,5 @@ +-- Copyright 2024 Logan Magee +-- +-- SPDX-License-Identifier: AGPL-3.0-only + +ALTER TABLE apps ADD COLUMN repository_metadata BLOB;