Improve lookup performance

libs.versions.toml

@@ -3,7 +3,7 @@ minecraft = "1.20.4"
 yarn-mappings = "1.20.4+build.1"
 fabric-loader = "0.15.1"
 
-fabric-api = "0.91.2+1.20.4"
+fabric-api = "0.91.3+1.20.4"
 
 # Kotlin
 kotlin = "1.8.22"
@@ -13,8 +13,8 @@ fabric-kotlin = "1.9.4+kotlin.1.8.21"
 fabric-permissions = "0.3-SNAPSHOT"
 translations = "2.2.0+1.20.3-rc1"
 
-exposed = "0.38.2"
-sqlite-jdbc = "3.42.0.0"
+exposed = "0.46.0"
+sqlite-jdbc = "3.44.1.0"
 
 konf = "1.1.2"
 

src/main/kotlin/com/github/quiltservertools/ledger/Ledger.kt

@@ -44,7 +44,7 @@ import com.github.quiltservertools.ledger.config.config as realConfig
 
 object Ledger : DedicatedServerModInitializer, CoroutineScope {
     const val MOD_ID = "ledger"
-    const val DEFAULT_DATABASE = SQLiteDialect.dialectName
+    val DEFAULT_DATABASE = SQLiteDialect.dialectName
 
     @JvmStatic
     val api: LedgerApi = LedgerApiImpl

src/main/kotlin/com/github/quiltservertools/ledger/actionutils/ActionSearchParams.kt

@@ -4,6 +4,7 @@ import com.github.quiltservertools.ledger.utility.Negatable
 import net.minecraft.util.Identifier
 import net.minecraft.util.math.BlockBox
 import java.time.Instant
+import java.util.UUID
 
 data class ActionSearchParams(
     val bounds: BlockBox?,
@@ -12,7 +13,7 @@ data class ActionSearchParams(
     var actions: MutableSet<Negatable<String>>?,
     var objects: MutableSet<Negatable<Identifier>>?,
     var sourceNames: MutableSet<Negatable<String>>?,
-    var sourcePlayerNames: MutableSet<Negatable<String>>?,
+    var sourcePlayerIds: MutableSet<Negatable<UUID>>?,
     var worlds: MutableSet<Negatable<Identifier>>?,
 ) {
     private constructor(builder: Builder) : this(
@@ -22,11 +23,11 @@ data class ActionSearchParams(
         builder.actions,
         builder.objects,
         builder.sourceNames,
-        builder.sourcePlayerNames,
+        builder.sourcePlayerIds,
         builder.worlds
     )
 
-    fun isEmpty() = listOf(bounds, before, after, actions, objects, sourceNames, sourcePlayerNames, worlds).all { it == null }
+    fun isEmpty() = listOf(bounds, before, after, actions, objects, sourceNames, sourcePlayerIds, worlds).all { it == null }
 
     companion object {
         inline fun build(block: Builder.() -> Unit) = Builder().apply(block).build()
@@ -39,7 +40,7 @@ data class ActionSearchParams(
         var actions: MutableSet<Negatable<String>>? = null
         var objects: MutableSet<Negatable<Identifier>>? = null
         var sourceNames: MutableSet<Negatable<String>>? = null
-        var sourcePlayerNames: MutableSet<Negatable<String>>? = null
+        var sourcePlayerIds: MutableSet<Negatable<UUID>>? = null
         var worlds: MutableSet<Negatable<Identifier>>? = null
 
         fun build() = ActionSearchParams(this)

src/main/kotlin/com/github/quiltservertools/ledger/commands/arguments/SearchParamArgument.kt

@@ -137,11 +137,17 @@ object SearchParamArgument {
                             builder.sourceNames!!.add(nonPlayer)
                         }
                     } else {
-                        if (builder.sourcePlayerNames == null) {
-                            builder.sourcePlayerNames =
-                                mutableSetOf(sourceInput)
-                        } else {
-                            builder.sourcePlayerNames!!.add(sourceInput)
+                        val profile = source.server.userCache?.findByName(sourceInput.property)
+                        val id = profile?.orElse(null)?.id
+
+                        if (id != null) {
+                            val playerIdEntry = Negatable(id, sourceInput.allowed)
+                            if (builder.sourcePlayerIds == null) {
+                                builder.sourcePlayerIds =
+                                    mutableSetOf(playerIdEntry)
+                            } else {
+                                builder.sourcePlayerIds!!.add(playerIdEntry)
+                            }
                         }
                     }
                 }

src/main/kotlin/com/github/quiltservertools/ledger/commands/parameters/SourceParameter.kt

@@ -1,12 +1,13 @@
 package com.github.quiltservertools.ledger.commands.parameters
 
+import com.github.quiltservertools.ledger.database.DatabaseManager
 import com.mojang.brigadier.StringReader
 import com.mojang.brigadier.context.CommandContext
 import com.mojang.brigadier.suggestion.Suggestions
 import com.mojang.brigadier.suggestion.SuggestionsBuilder
+import java.util.concurrent.CompletableFuture
 import net.minecraft.command.CommandSource
 import net.minecraft.server.command.ServerCommandSource
-import java.util.concurrent.CompletableFuture
 
 class SourceParameter : SimpleParameter<String>() {
     override fun parse(stringReader: StringReader): String {
@@ -27,7 +28,9 @@ class SourceParameter : SimpleParameter<String>() {
         stringReader.cursor = builder.start
 
         val players = context.source.playerNames
-        players.add("@")
+        DatabaseManager.getKnownSources().forEach {
+            players.add("@$it")
+        }
         // TODO suggest non-player sources
 
         return CommandSource.suggestMatching(

src/main/kotlin/com/github/quiltservertools/ledger/database/DatabaseManager.kt

@@ -15,18 +15,25 @@ import com.github.quiltservertools.ledger.utility.NbtUtils
 import com.github.quiltservertools.ledger.utility.Negatable
 import com.github.quiltservertools.ledger.utility.PlayerResult
 import com.mojang.authlib.GameProfile
+import java.time.Instant
+import java.time.temporal.ChronoUnit
+import java.util.*
+import javax.sql.DataSource
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.sync.Mutex
 import net.minecraft.nbt.StringNbtReader
 import net.minecraft.util.Identifier
 import net.minecraft.util.WorldSavePath
 import net.minecraft.util.math.BlockPos
+import org.jetbrains.exposed.dao.id.EntityID
 import org.jetbrains.exposed.sql.Column
 import org.jetbrains.exposed.sql.Database
 import org.jetbrains.exposed.sql.Op
 import org.jetbrains.exposed.sql.Query
 import org.jetbrains.exposed.sql.SchemaUtils
 import org.jetbrains.exposed.sql.SortOrder
+import org.jetbrains.exposed.sql.SqlExpressionBuilder.inSubQuery
+import org.jetbrains.exposed.sql.SqlExpressionBuilder.lessEq
 import org.jetbrains.exposed.sql.SqlLogger
 import org.jetbrains.exposed.sql.Transaction
 import org.jetbrains.exposed.sql.addLogger
@@ -41,18 +48,13 @@ import org.jetbrains.exposed.sql.insertIgnore
 import org.jetbrains.exposed.sql.lowerCase
 import org.jetbrains.exposed.sql.or
 import org.jetbrains.exposed.sql.orWhere
-import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.selectAll
 import org.jetbrains.exposed.sql.statements.StatementContext
 import org.jetbrains.exposed.sql.statements.expandArgs
 import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.jetbrains.exposed.sql.update
 import org.sqlite.SQLiteDataSource
-import java.time.Instant
-import java.time.temporal.ChronoUnit
-import java.util.UUID
-import javax.sql.DataSource
 import kotlin.io.path.pathString
 import kotlin.math.ceil
 
@@ -214,64 +216,64 @@ object DatabaseManager {
             op = op.and { Tables.Actions.timestamp.greaterEq(params.after) }
         }
 
+        val sourceNames = ArrayList<Negatable<Int>>()
+        params.sourceNames?.forEach {
+            val sourceId = getSourceId(it.property)
+            if (sourceId != null) {
+                sourceNames.add(Negatable(sourceId, it.allowed))
+            } else {
+                // Unknown source name
+                op = Op.FALSE
+            }
+        }
+
         op = addParameters(
             op,
-            params.sourceNames,
-            Tables.Sources.name
+            sourceNames,
+            Tables.Actions.sourceName
         )
 
         op = addParameters(
             op,
-            params.actions,
-            Tables.ActionIdentifiers.actionIdentifier
+            params.actions?.map { Negatable(getActionId(it.property), it.allowed) },
+            Tables.Actions.actionIdentifier
         )
 
         op = addParameters(
             op,
-            params.worlds?.map {
-                if (it.allowed) {
-                    Negatable.allow(it.property.toString())
-                } else {
-                    Negatable.deny(it.property.toString())
-                }
-            },
-            Tables.Worlds.identifier
+            params.worlds?.map { Negatable(getWorldId(it.property), it.allowed) },
+            Tables.Actions.world
         )
 
         op = addParameters(
             op,
-            params.objects?.map {
-                if (it.allowed) {
-                    Negatable.allow(it.property.toString())
-                } else {
-                    Negatable.deny(it.property.toString())
-                }
-            },
-            Tables.ObjectIdentifiers.identifier,
-            Tables.oldObjectTable[Tables.ObjectIdentifiers.identifier]
+            params.objects?.map { Negatable(getRegistryKeyId(it.property), it.allowed) },
+            Tables.Actions.objectId,
+            Tables.Actions.oldObjectId
         )
 
         op = addParameters(
             op,
-            params.sourcePlayerNames,
-            Tables.Players.playerName
+            params.sourcePlayerIds?.map { Negatable(selectPlayerId(it.property), it.allowed) },
+            Tables.Actions.sourcePlayer
         )
 
         return op
     }
 
-    private fun <E> addParameters(
+    private fun <E : Comparable<E>, C : EntityID<E>?> addParameters(
         op: Op<Boolean>,
         paramSet: Collection<Negatable<E>>?,
-        column: Column<E>,
-        orColumn: Column<E>? = null
+        column: Column<C>,
+        orColumn: Column<C>? = null,
     ): Op<Boolean> {
         fun addAllowedParameters(
             allowed: Collection<E>,
             op: Op<Boolean>
         ): Op<Boolean> {
             if (allowed.isEmpty()) return op
 
+
             var operator = if (orColumn != null) {
                 Op.build { column eq allowed.first() or (orColumn eq allowed.first()) }
             } else {
@@ -440,7 +442,7 @@ object DatabaseManager {
             .selectAll()
             .andWhere { buildQueryParams(params) }
 
-        totalActions = query.copy().count()
+        totalActions = countActions(params)
         if (totalActions == 0L) return SearchResults(actions, params, page, 0)
 
         query = query.orderBy(Tables.Actions.id, SortOrder.DESC)
@@ -457,15 +459,6 @@ object DatabaseManager {
     }
 
     private fun Transaction.countActions(params: ActionSearchParams): Long = Tables.Actions
-        .innerJoin(Tables.ActionIdentifiers)
-        .innerJoin(Tables.Worlds)
-        .leftJoin(Tables.Players)
-        .innerJoin(
-            Tables.oldObjectTable,
-            { Tables.Actions.oldObjectId },
-            { Tables.oldObjectTable[Tables.ObjectIdentifiers.id] })
-        .innerJoin(Tables.ObjectIdentifiers, { Tables.Actions.objectId }, { Tables.ObjectIdentifiers.id })
-        .innerJoin(Tables.Sources)
         .selectAll()
         .andWhere { buildQueryParams(params) }
         .count()
@@ -541,7 +534,7 @@ object DatabaseManager {
         return actions
     }
 
-    private fun Transaction.selectPlayerId(playerId: UUID): Int {
+    private fun selectPlayerId(playerId: UUID): Int {
         cache.playerKeys.getIfPresent(playerId)?.let { return it }
 
         return Tables.Player.find {
@@ -552,35 +545,46 @@ object DatabaseManager {
     private fun Transaction.selectPlayer(playerName: String) =
         Tables.Player.find { Tables.Players.playerName.lowerCase() eq playerName }.firstOrNull()
 
-    private fun Transaction.getOrCreateSourceId(source: String): Int {
+    fun getKnownSources() =
+        cache.sourceKeys.asMap().keys
+
+    private fun getSourceId(source: String): Int? {
+        cache.sourceKeys.getIfPresent(source)?.let { return it }
+
+        return Tables.Source.find { Tables.Sources.name eq source }.firstOrNull()?.id?.value.also {
+            it?.let { cache.sourceKeys.put(source, it) }
+        }
+    }
+
+    private fun getOrCreateSourceId(source: String): Int {
         cache.sourceKeys.getIfPresent(source)?.let { return it }
 
         Tables.Source.find { Tables.Sources.name eq source }.firstOrNull()?.let { return it.id.value }
 
         return Tables.Source[
             Tables.Sources.insertAndGetId {
-            it[name] = source
-        }
+                it[name] = source
+            }
         ].id.value.also { cache.sourceKeys.put(source, it) }
     }
 
-    private fun Transaction.getActionId(actionTypeId: String): Int {
+    private fun getActionId(actionTypeId: String): Int {
         cache.actionIdentifierKeys.getIfPresent(actionTypeId)?.let { return it }
 
         return Tables.ActionIdentifier.find { Tables.ActionIdentifiers.actionIdentifier eq actionTypeId }
             .first().id.value
             .also { cache.actionIdentifierKeys.put(actionTypeId, it) }
     }
 
-    private fun Transaction.getRegistryKeyId(identifier: Identifier): Int {
+    private fun getRegistryKeyId(identifier: Identifier): Int {
         cache.objectIdentifierKeys.getIfPresent(identifier)?.let { return it }
 
         return Tables.ObjectIdentifier.find { Tables.ObjectIdentifiers.identifier eq identifier.toString() }
             .limit(1).first().id.value
             .also { cache.objectIdentifierKeys.put(identifier, it) }
     }
 
-    private fun Transaction.getWorldId(identifier: Identifier): Int {
+    private fun getWorldId(identifier: Identifier): Int {
         cache.worldIdentifierKeys.getIfPresent(identifier)?.let { return it }
 
         return Tables.World.find { Tables.Worlds.identifier eq identifier.toString() }.limit(1).first().id.value
@@ -590,8 +594,8 @@ object DatabaseManager {
     // Workaround because can't delete from a join in exposed https://kotlinlang.slack.com/archives/C0CG7E0A1/p1605866974117400
     private fun Transaction.purgeActions(params: ActionSearchParams) = Tables.Actions
         .deleteWhere {
-            Tables.Actions.id inSubQuery Tables.Actions.joinTheTables().slice(Tables.Actions.id)
-                .select { buildQueryParams(params) }
+            Tables.Actions.id inSubQuery Tables.Actions.select(Tables.Actions.id)
+                .where(buildQueryParams(params))
         }
 
     private fun Transaction.selectPlayers(players: Set<GameProfile>): List<PlayerResult> {
@@ -603,11 +607,4 @@ object DatabaseManager {
         return Tables.Player.wrapRows(query).toList().map { PlayerResult.fromRow(it) }
     }
 
-    private fun Tables.Actions.joinTheTables() = this
-        .innerJoin(Tables.ActionIdentifiers)
-        .innerJoin(Tables.Worlds)
-        .leftJoin(Tables.Players)
-        .innerJoin(Tables.oldObjectTable, { oldObjectId }, { Tables.oldObjectTable[Tables.ObjectIdentifiers.id] })
-        .innerJoin(Tables.ObjectIdentifiers, { objectId }, { Tables.ObjectIdentifiers.id })
-        .innerJoin(Tables.Sources)
 }