Merge pull request #126 from JetBrains/merge/release-3.0

Merge/release 3.0

benchmarks/build.gradle.kts

@@ -1,11 +1,6 @@
 val currentJmhVersion = "1.35"
 
 buildscript {
-    repositories {
-        maven {
-            url = uri("https://cache-redirector.jetbrains.com/plugins.gradle.org/m2")
-        }
-    }
     dependencies {
         classpath("org.apache.commons:commons-lang3:3.5")
     }
@@ -44,4 +39,4 @@ val deleteEmptyBenchmarkList = tasks.register<Delete>("deleteEmptyBenchmarkList"
 
 tasks.named("jmhCompileGeneratedClasses") {
     finalizedBy(deleteEmptyBenchmarkList)
-}
+}

build.gradle.kts

@@ -63,8 +63,7 @@ nexusStaging {
 
 allprojects {
     repositories {
-        maven { url = uri("https://cache-redirector.jetbrains.com/repo1.maven.org/maven2") }
-        maven { url = uri("https://maven.pkg.jetbrains.space/public/p/kotlinx-html/maven") }
+        mavenCentral()
     }
 }
 

entity-store/src/main/java/jetbrains/exodus/entitystore/PersistentEntityStoreImpl.java

@@ -2043,14 +2043,21 @@ public void renameEntityType(@NotNull final String oldEntityTypeName, @NotNull f
         entityTypes.rename(getAndCheckCurrentTransaction(), oldEntityTypeName, newEntityTypeName);
     }
 
-    public void deleteEntityType(@NotNull final String entityTypeName) {
-        final PersistentStoreTransaction txn = getAndCheckCurrentTransaction();
-        final int entityTypeId = entityTypes.delete(txn, entityTypeName);
-
+    public void deleteEntityType(int entityTypeId) {
         if (entityTypeId < 0) {
             return;
         }
+        PersistentStoreTransaction txn = getAndCheckCurrentTransaction();
+        final String entityTypeName = entityTypes.getName(txn, entityTypeId);
+        if (entityTypeName == null) {
+            return;
+        }
+
+        entityTypes.delete(entityTypeName, entityTypeId);
+        deleteEntityType(entityTypeId, txn);
+    }
 
+    private void deleteEntityType(int entityTypeId, final PersistentStoreTransaction txn) {
         entitiesTables.remove(entityTypeId);
         propertiesTables.remove(entityTypeId);
         linksTables.remove(entityTypeId);
@@ -2085,6 +2092,16 @@ public void remove() { // don't give a damn
                 }
         );
     }
+    public void deleteEntityType(@NotNull final String entityTypeName) {
+        final PersistentStoreTransaction txn = getAndCheckCurrentTransaction();
+        final int entityTypeId = entityTypes.delete(txn, entityTypeName);
+
+        if (entityTypeId < 0) {
+            return;
+        }
+
+        deleteEntityType(entityTypeId, txn);
+    }
 
     private void truncateStores(@NotNull final PersistentStoreTransaction txn, @NotNull Iterable<String> unsafe, @NotNull Iterable<String> safe) {
         final Transaction envTxn = txn.getEnvironmentTransaction();

entity-store/src/main/java/jetbrains/exodus/entitystore/PersistentSequentialDictionary.java

@@ -138,8 +138,7 @@ public String getName(@NotNull final TxnProvider txnProvider, final int id) {
         return result;
     }
 
-    public int delete(@NotNull final PersistentStoreTransaction txn, @NotNull final String name) {
-        final int id = getId(txn, name);
+    public int delete(@NotNull final String name, final int id) {
         if (id < 0) {
             // type doesn't exist, and it's ok
             return -1;
@@ -157,6 +156,12 @@ void persist(final Transaction txn) {
         }
     }
 
+
+    public int delete(@NotNull final PersistentStoreTransaction txn, @NotNull final String name) {
+        final int id = getId(txn, name);
+        return delete(name, id);
+    }
+
     public void rename(@NotNull final PersistentStoreTransaction txn, @NotNull final String oldName, @NotNull final String newName) {
         if (oldName.equals(newName)) {
             return;

entity-store/src/test/kotlin/jetbrains/exodus/entitystore/EntityTests.kt

@@ -49,7 +49,8 @@ class EntityTests : EntityStoreTestBase() {
                 "testAddPhantomLink",
                 "testTooBigProperty",
                 "testTransactionAt",
-                "testTransactionAtIsIdempotent"
+                "testTransactionAtIsIdempotent",
+                "testCreateAndRemoveEntityType"
         )
     }
 
@@ -125,6 +126,20 @@ class EntityTests : EntityStoreTestBase() {
         Assert.assertEquals(entity.id, sameEntity.id)
     }
 
+    fun testCreateAndRemoveEntityType() {
+        var typeId = entityStore.computeInTransaction { txn ->
+            entityStore.getEntityTypeId(txn as PersistentStoreTransaction, "Issue", true)
+        }
+        Assert.assertNotEquals(-1, typeId)
+        entityStore.computeInTransaction {
+            entityStore.deleteEntityType(typeId)
+        }
+        typeId = entityStore.computeInTransaction { txn ->
+            entityStore.getEntityTypeId(txn as PersistentStoreTransaction, "Issue", false)
+        }
+        Assert.assertEquals(-1, typeId)
+    }
+
     fun testRawProperty() {
         val txn = storeTransaction
         val entity = txn.newEntity("Issue")

environment/src/main/kotlin/jetbrains/exodus/env/Environments.kt

@@ -44,7 +44,8 @@ object Environments : KLogging() {
     fun newInstance(dir: String): Environment = newInstance(dir, EnvironmentConfig())
 
     @JvmStatic
-    fun newInstance(log: Log, ec: EnvironmentConfig): Environment = prepare { EnvironmentImpl(log, ec) }
+    fun newInstance(log: Log, ec: EnvironmentConfig): Environment =
+        prepare { EnvironmentImpl(log, ec) }
 
     @JvmStatic
     fun newInstance(dir: String, ec: EnvironmentConfig): Environment =
@@ -69,20 +70,26 @@ object Environments : KLogging() {
 
     @Suppress("unused")
     @JvmStatic
-    fun newContextualInstance(dir: String, subDir: String, ec: EnvironmentConfig): ContextualEnvironment =
+    fun newContextualInstance(
+        dir: String,
+        subDir: String,
+        ec: EnvironmentConfig
+    ): ContextualEnvironment =
         prepare { ContextualEnvironmentImpl(newLogInstance(File(dir, subDir), ec), ec) }
 
     @Suppress("unused")
     @JvmStatic
-    fun newContextualInstance(dir: String): ContextualEnvironment = newContextualInstance(dir, EnvironmentConfig())
+    fun newContextualInstance(dir: String): ContextualEnvironment =
+        newContextualInstance(dir, EnvironmentConfig())
 
     @JvmStatic
     fun newContextualInstance(dir: String, ec: EnvironmentConfig): ContextualEnvironment =
         prepare { ContextualEnvironmentImpl(newLogInstance(File(dir), ec), ec) }
 
     @Suppress("unused")
     @JvmStatic
-    fun newContextualInstance(dir: File): ContextualEnvironment = newContextualInstance(dir, EnvironmentConfig())
+    fun newContextualInstance(dir: File): ContextualEnvironment =
+        newContextualInstance(dir, EnvironmentConfig())
 
     @JvmStatic
     fun newContextualInstance(dir: File, ec: EnvironmentConfig): ContextualEnvironment =
@@ -93,7 +100,8 @@ object Environments : KLogging() {
         prepare { ContextualEnvironmentImpl(newLogInstance(config, ec), ec) }
 
     @JvmStatic
-    fun newLogInstance(dir: File, ec: EnvironmentConfig): Log = newLogInstance(LogConfig().setLocation(dir.path), ec)
+    fun newLogInstance(dir: File, ec: EnvironmentConfig): Log =
+        newLogInstance(LogConfig().setLocation(dir.path), ec)
 
     @JvmStatic
     fun newLogInstance(config: LogConfig, ec: EnvironmentConfig): Log {
@@ -135,17 +143,38 @@ object Environments : KLogging() {
     @JvmStatic
     fun newLogInstance(config: LogConfig): Log = Log(
         config.also
-        { SharedOpenFilesCache.setSize(config.cacheOpenFilesCount) }, EnvironmentImpl.CURRENT_FORMAT_VERSION
+        { SharedOpenFilesCache.setSize(config.cacheOpenFilesCount) },
+        EnvironmentImpl.CURRENT_FORMAT_VERSION
     )
 
     private fun <T : EnvironmentImpl> prepare(envCreator: () -> T): T {
         var env = envCreator()
         val ec = env.environmentConfig
-        val needsToBeMigrated = !env.log.formatWithHashCodeIsUsed
 
+        if (env.environmentConfig.storesToRemoveBeforeCompaction != null) {
+            EnvironmentImpl.loggerInfo(
+                "Store(s) are(is) ${env.environmentConfig.storesToRemoveBeforeCompaction} " +
+                        "going to be removed from the database."
+            )
+
+
+            env.executeInTransaction { tx ->
+                val storesToRemove = env.environmentConfig.storesToRemoveBeforeCompaction.split(",")
+
+                for (storeToRemove in storesToRemove) {
+                    if (env.storeExists(storeToRemove, tx)) {
+                        env.removeStore(storeToRemove, tx)
+                        EnvironmentImpl.loggerInfo("$storeToRemove is removed from database")
+                    }
+                }
+            }
+        }
+
+        val needsToBeMigrated = !env.log.formatWithHashCodeIsUsed
         if (ec.logDataReaderWriterProvider == DataReaderWriterProvider.DEFAULT_READER_WRITER_PROVIDER &&
             (ec.envCompactOnOpen || ec.envCompactInSingleBatchOnOpen) && env.log.numberOfFiles > 1 || needsToBeMigrated
         ) {
+
             if (needsToBeMigrated) {
                 EnvironmentImpl.loggerInfo(
                     "Outdated binary format is used in environment ${env.log.location} " +
@@ -228,7 +257,9 @@ object Environments : KLogging() {
                     }
                 }
 
-                Files.deleteIfExists(Paths.get(tempDir.toURI()).resolve(LockingManager.LOCK_FILE_NAME))
+                Files.deleteIfExists(
+                    Paths.get(tempDir.toURI()).resolve(LockingManager.LOCK_FILE_NAME)
+                )
 
                 env = envCreator()
 

environment/src/main/kotlin/jetbrains/exodus/env/Unsafe.kt

@@ -15,9 +15,6 @@
  */
 package jetbrains.exodus.env
 
-import jetbrains.exodus.ExodusException
-import jetbrains.exodus.io.Block
-import jetbrains.exodus.log.*
 import kotlin.concurrent.withLock
 
 fun <T> EnvironmentImpl.executeInCommitLock(action: () -> T): T {
@@ -32,121 +29,3 @@ fun <T> EnvironmentImpl.executeInMetaWriteLock(action: () -> T): T {
     }
 }
 
-internal fun EnvironmentImpl.tryUpdate(): Boolean {
-    return executeInCommitLock {
-        tryUpdateUnsafe()
-    }
-}
-
-private fun EnvironmentImpl.tryUpdateUnsafe(): Boolean {
-    val prevHighAddress = log.highAddress
-    log.tryUpdate(prevHighAddress)?.let {
-        val root = it.first.rootAddress
-        val highAddress = it.second
-        val files = it.third
-
-        return executeInMetaWriteLock {
-            try {
-                log.updateBlockSetHighAddressUnsafe(prevHighAddress, highAddress, files)
-                loadMetaTree(root, highAddress)?.let { metaTree ->
-                    metaTreeInternal = MetaTreeImpl(metaTree, root).also {
-                        MetaTreeImpl.cloneTree(metaTree) // try to traverse meta tree
-                    }
-                    true
-                } ?: run {
-                    // Throwable class is used for exceptions to prevent stacktrace masking
-                    throwableOnCommit = Throwable("Cannot load updated meta tree")
-                    false
-                }
-            } catch (t: Throwable) {
-                throwableOnCommit = Throwable("Cannot read updated meta tree", t)
-                true
-            }
-        }
-    }
-    return false
-}
-
-// advance to some root loggable
-private fun Log.tryUpdate(highAddress: Long): Triple<DatabaseRoot, Long, BlockSet.Immutable>? {
-    val blockSet = mutableBlocksUnsafe()
-    val addedBlocks = config.reader.getBlocks(getFileAddress(highAddress))
-    val itr = addedBlocks.iterator()
-    if (!itr.hasNext()) {
-        return null
-    }
-    val lastBlock: Block
-    while (true) {
-        val block = itr.next()
-        blockSet.add(block.address, block)
-        if (!itr.hasNext()) {
-            lastBlock = block
-            break
-        }
-    }
-    // create loggable
-    // update "last page"
-    return tryUpdate(this, lastBlock, blockSet)
-}
-
-private fun tryUpdate(
-    log: Log,
-    lastBlock: Block,
-    blockSet: BlockSet.Mutable
-): Triple<DatabaseRoot, Long, BlockSet.Immutable>? {
-    val lastBlockAddress = lastBlock.address
-    val highAddress = lastBlockAddress + lastBlock.length()
-    val startAddress = maxOf(highAddress, lastBlockAddress)
-    if (startAddress > lastBlockAddress + log.fileLengthBound) {
-        throw IllegalStateException("Log truncated abnormally, aborting")
-    }
-    val dataIterator = BlockDataIterator(log, lastBlock, startAddress, false)
-    val loggables = LoggableIteratorUnsafe(log, dataIterator)
-    val rootType = DatabaseRoot.DATABASE_ROOT_TYPE
-    var lastRoot: DatabaseRoot? = null
-    try {
-        while (loggables.hasNext()) {
-            val loggable = loggables.next()
-            val loggableEnd = loggable.end()
-            if (loggableEnd > highAddress) {
-                break
-            }
-            if (loggable.type == rootType) {
-                lastRoot = DatabaseRoot(loggable, loggables.iterator)
-            } else if (!NullLoggable.isNullLoggable(loggable) && !HashCodeLoggable.isHashCodeLoggable(loggable)) {
-                // don't skip DatabaseRoot content
-                val expectedDataLength = loggable.dataLength.toLong()
-                if (loggables.iterator.skip(expectedDataLength) < expectedDataLength) {
-                    break
-                }
-            }
-            if (loggableEnd != dataIterator.address) {
-                break
-            }
-            if (loggableEnd == highAddress) {
-                break
-            }
-        }
-    } catch (e: ExodusException) {
-        Log.logger.info(e) { "Exception on Log recovery by tryUpdate() in ${Thread.currentThread().name}." }
-    }
-    if (lastRoot == null) {
-        return null
-    }
-    return Triple(lastRoot, highAddress, blockSet.endWrite())
-}
-
-internal class LoggableIteratorUnsafe(private val log: Log, internal val iterator: ByteIteratorWithAddress) :
-    Iterator<RandomAccessLoggable> {
-
-    fun getHighAddress() = iterator.address
-
-    override fun next(): RandomAccessLoggable {
-        if (!hasNext()) {
-            throw IllegalStateException()
-        }
-        return log.read(iterator)
-    }
-
-    override fun hasNext() = iterator.hasNext()
-}