Xodus to OrientDB migration 2: migrate classIds and localEntityIds if backward compatible EntityId is enabled

entity-store/src/main/kotlin/jetbrains/exodus/entitystore/orientdb/OVertexEntity.kt

@@ -41,6 +41,14 @@ class OVertexEntity(private var vertex: OVertex, private val store: PersistentEn
         fun blobHashProperty(propertyName: String) = "\$$propertyName$STRING_BLOB_HASH_PROPERTY_NAME_SUFFIX"
 
         const val STRING_BLOB_CLASS_NAME: String = "StringBlob"
+
+        // Backward compatible EntityId
+
+        const val CLASS_ID_CUSTOM_PROPERTY_NAME = "classId"
+        const val CLASS_ID_SEQUENCE_NAME = "sequence_classId"
+
+        const val BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME = "backwardCompatibleLocalEntityId"
+        fun localEntityIdSequenceName(className: String): String = "${className}_sequence_localEntityId"
     }
 
     private val activeSession get() = ODatabaseSession.getActiveSession()

query/src/main/kotlin/jetbrains/exodus/query/metadata/IndicesCreator.kt

@@ -73,7 +73,7 @@ data class DeferredIndex(
 ) {
     constructor(ownerVertexName: String, properties: List<IndexField>, unique: Boolean): this(
         ownerVertexName,
-        indexName = "${ownerVertexName}_${properties.joinToString("_") { it.name }}",
+        indexName = "${ownerVertexName}_${properties.joinToString("_") { it.name }}${if (unique) "_unique" else ""}",
         properties,
         unique = unique
     )

query/src/main/kotlin/jetbrains/exodus/query/metadata/OrientDbSchemaInitializer.kt

@@ -20,19 +20,25 @@ import com.orientechnologies.orient.core.db.ODatabaseSession
 import com.orientechnologies.orient.core.metadata.schema.OClass
 import com.orientechnologies.orient.core.metadata.schema.OProperty
 import com.orientechnologies.orient.core.metadata.schema.OType
+import com.orientechnologies.orient.core.metadata.sequence.OSequence
 import com.orientechnologies.orient.core.record.ODirection
 import com.orientechnologies.orient.core.record.OVertex
 import jetbrains.exodus.entitystore.orientdb.OVertexEntity
+import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME
+import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.CLASS_ID_CUSTOM_PROPERTY_NAME
+import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.CLASS_ID_SEQUENCE_NAME
+import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.localEntityIdSequenceName
 import mu.KotlinLogging
 
 private val log = KotlinLogging.logger {}
 
 fun ODatabaseSession.applySchema(
     model: ModelMetaDataImpl,
     indexForEverySimpleProperty: Boolean = false,
-    applyLinkCardinality: Boolean = true
+    applyLinkCardinality: Boolean = true,
+    backwardCompatibleEntityId: Boolean = false,
 ): Map<String, Set<DeferredIndex>> {
-    val initializer = OrientDbSchemaInitializer(model, this, indexForEverySimpleProperty, applyLinkCardinality)
+    val initializer = OrientDbSchemaInitializer(model, this, indexForEverySimpleProperty, applyLinkCardinality, backwardCompatibleEntityId)
     initializer.apply()
     return initializer.getIndices()
 }
@@ -41,7 +47,8 @@ internal class OrientDbSchemaInitializer(
     private val dnqModel: ModelMetaDataImpl,
     private val oSession: ODatabaseSession,
     private val indexForEverySimpleProperty: Boolean,
-    private val applyLinkCardinality: Boolean
+    private val applyLinkCardinality: Boolean,
+    private val backwardCompatibleEntityId: Boolean
 ) {
     private val paddedLogger = PaddedLogger(log)
 
@@ -52,10 +59,10 @@ internal class OrientDbSchemaInitializer(
     private fun appendLine(s: String = "") = paddedLogger.appendLine(s)
 
 
-    private val indices = HashMap<String, MutableSet<DeferredIndex>>()
+    private val indices = HashMap<String, MutableMap<String, DeferredIndex>>()
 
     private fun addIndex(index: DeferredIndex) {
-        indices.getOrPut(index.ownerVertexName) { HashSet() }.add(index)
+        indices.getOrPut(index.ownerVertexName) { HashMap() }[index.indexName] = index
     }
 
     private fun simplePropertyIndex(entityName: String, propertyName: String): DeferredIndex {
@@ -65,11 +72,14 @@ internal class OrientDbSchemaInitializer(
         return DeferredIndex(entityName, listOf(indexField), unique = false)
     }
 
-    fun getIndices(): Map<String, Set<DeferredIndex>> = indices
-
+    fun getIndices(): Map<String, Set<DeferredIndex>> = indices.map { it.key to it.value.values.toSet() }.toMap()
 
     fun apply() {
         try {
+            if (backwardCompatibleEntityId) {
+                createClassIdSequenceIfAbsent()
+            }
+
             appendLine("applying the DNQ schema to OrientDB")
             val sortedEntities = dnqModel.entitiesMetaData.sortedTopologically()
 
@@ -120,8 +130,8 @@ internal class OrientDbSchemaInitializer(
                 for ((indexOwner, indices) in indices) {
                     appendLine("$indexOwner:")
                     withPadding {
-                        for (index in indices) {
-                            appendLine(index.indexName)
+                        for ((indexName, _) in indices) {
+                            appendLine(indexName)
                         }
                     }
                 }
@@ -131,6 +141,20 @@ internal class OrientDbSchemaInitializer(
         }
     }
 
+    // ClassId
+
+    private fun createClassIdSequenceIfAbsent() {
+        createSequenceIfAbsent(CLASS_ID_SEQUENCE_NAME)
+    }
+
+    private fun OClass.setClassIdIfAbsent() {
+        if (getCustom(CLASS_ID_CUSTOM_PROPERTY_NAME) == null) {
+            val sequences = oSession.metadata.sequenceLibrary
+            val sequence: OSequence = sequences.getSequence(CLASS_ID_SEQUENCE_NAME) ?: throw IllegalStateException("$CLASS_ID_SEQUENCE_NAME not found")
+
+            setCustom(CLASS_ID_CUSTOM_PROPERTY_NAME, sequence.next().toString())
+        }
+    }
 
     // Vertices and Edges
 
@@ -140,6 +164,16 @@ internal class OrientDbSchemaInitializer(
         oClass.applySuperClass(dnqEntity.superType)
         appendLine()
 
+        if (backwardCompatibleEntityId) {
+            oClass.setClassIdIfAbsent()
+            createSequenceIfAbsent(localEntityIdSequenceName(dnqEntity.type))
+            /*
+            * We do not apply a unique index to the localEntityId property because indices in OrientDB are polymorphic.
+            * So, you can not have the same value in a property in an instance of a superclass and in an instance of its subclass.
+            * But it exactly what happens in the original Xodus.
+            * */
+        }
+
         /*
         * It is more efficient to create indices after the data migration.
         * So, we only remember indices here and let the user create them later.
@@ -302,6 +336,12 @@ internal class OrientDbSchemaInitializer(
                     oClass.applySimpleProperty(propertyMetaData, required || requiredBecauseOfIndex)
                 }
             }
+            if (backwardCompatibleEntityId) {
+                val prop = SimplePropertyMetaDataImpl(BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME, "long")
+                oClass.applySimpleProperty(prop, true)
+                // we need this index regardless what we have in indexForEverySimpleProperty
+                addIndex(simplePropertyIndex(dnqEntity.type, BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME))
+            }
         }
     }
 
@@ -515,6 +555,15 @@ internal class OrientDbSchemaInitializer(
         return oProperty
     }
 
+    private fun createSequenceIfAbsent(sequenceName: String) {
+        val sequences = oSession.metadata.sequenceLibrary
+        if (sequences.getSequence(sequenceName) == null) {
+            val params = OSequence.CreateParams()
+            params.start = 0L
+            sequences.createSequence(sequenceName, OSequence.SEQUENCE_TYPE.ORDERED, params)
+        }
+    }
+
     private fun getOType(jvmTypeName: String): OType {
         return when (jvmTypeName.lowercase()) {
             "boolean" -> OType.BOOLEAN

query/src/main/kotlin/jetbrains/exodus/query/metadata/XodusToOrientDataMigrator.kt

@@ -1,13 +1,31 @@
 package jetbrains.exodus.query.metadata
 
 import com.orientechnologies.orient.core.db.ODatabaseSession
+import com.orientechnologies.orient.core.metadata.schema.OType
+import com.orientechnologies.orient.core.metadata.sequence.OSequence
 import jetbrains.exodus.entitystore.EntityId
 import jetbrains.exodus.entitystore.PersistentEntityStore
 import jetbrains.exodus.entitystore.orientdb.OPersistentEntityStore
 import jetbrains.exodus.entitystore.orientdb.OVertexEntity
+import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME
 import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.BINARY_BLOB_CLASS_NAME
+import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.CLASS_ID_CUSTOM_PROPERTY_NAME
+import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.CLASS_ID_SEQUENCE_NAME
+import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.localEntityIdSequenceName
 import jetbrains.exodus.entitystore.orientdb.withSession
 
+fun migrateDataFromXodusToOrientDb(
+    xodus: PersistentEntityStore,
+    orient: OPersistentEntityStore,
+    /*
+    * How many entities should be copied in a single transaction
+    * */
+    entitiesPerTransaction: Int = 10,
+    backwardCompatibleEntityId: Boolean = false
+) {
+    val migrator = XodusToOrientDataMigrator(xodus, orient, entitiesPerTransaction, backwardCompatibleEntityId)
+    return migrator.migrate()
+}
 
 /**
  * This class is responsible for migrating data from Xodus to OrientDB.
@@ -16,13 +34,14 @@ import jetbrains.exodus.entitystore.orientdb.withSession
  * @param orient The OrientDB OPersistentEntityStore instance.
  * @param entitiesPerTransaction The number of entities to be copied in a single transaction.
  */
-class XodusToOrientDataMigrator(
+internal class XodusToOrientDataMigrator(
     private val xodus: PersistentEntityStore,
     private val orient: OPersistentEntityStore,
     /*
     * How many entities should be copied in a single transaction
     * */
-    private val entitiesPerTransaction: Int = 10
+    private val entitiesPerTransaction: Int = 10,
+    private val backwardCompatibleEntityId: Boolean = false
 ) {
     private val xEntityIdToOEntityId = HashMap<EntityId, EntityId>()
 
@@ -38,12 +57,33 @@ class XodusToOrientDataMigrator(
     private fun createVertexClassesIfAbsent(oSession: ODatabaseSession) {
         // make sure all the vertex classes are created in OrientDB
         // classes can not be created in a transaction, so we have to create them before copying the data
+        var maxClassId = 0
         xodus.withReadonlyTx { xTx ->
             for (type in xTx.entityTypes) {
-                oSession.getClass(type) ?: oSession.createVertexClass(type)
+                val oClass = oSession.getClass(type) ?: oSession.createVertexClass(type)
+                val classId = xodus.getEntityTypeId(type)
+
+                if (backwardCompatibleEntityId) {
+                    oClass.setCustom(CLASS_ID_CUSTOM_PROPERTY_NAME, classId.toString())
+                    maxClassId = maxOf(maxClassId, classId)
+
+                    // create localEntityId property if absent
+                    if (oClass.getProperty(BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME) == null) {
+                        oClass.createProperty(BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME, OType.LONG)
+                    }
+                }
             }
         }
 
+        if (backwardCompatibleEntityId) {
+            // create a sequence to generate classIds
+            val sequences = oSession.metadata.sequenceLibrary
+
+            require(sequences.getSequence(CLASS_ID_SEQUENCE_NAME) == null) { "$CLASS_ID_SEQUENCE_NAME is already created. It means that some data migration has happened to the target database before. Such a scenario is not supported." }
+
+            oSession.createSequenceIfAbsent(CLASS_ID_SEQUENCE_NAME, maxClassId.toLong())
+        }
+
         oSession.getClass(BINARY_BLOB_CLASS_NAME) ?: oSession.createClass(BINARY_BLOB_CLASS_NAME)
     }
 
@@ -58,6 +98,7 @@ class XodusToOrientDataMigrator(
         xodus.withReadonlyTx { xTx ->
             oSession.withCountingTx(entitiesPerTransaction) { countingTx ->
                 for (type in xTx.entityTypes) {
+                    var largestEntityId = 0L
                     for (xEntity in xTx.getAll(type)) {
                         val vertex = oSession.newVertex(type)
                         val oEntity = OVertexEntity(vertex, orient)
@@ -73,20 +114,42 @@ class XodusToOrientDataMigrator(
                         countingTx.increment()
 
                         edgeClassesToCreate.addAll(xEntity.linkNames)
+
+                        if (backwardCompatibleEntityId) {
+                            // copy localEntityId
+                            val localEntityId = xEntity.id.localId
+                            oEntity.setProperty(BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME, localEntityId)
+
+                            largestEntityId = maxOf(largestEntityId, localEntityId)
+                        }
+                    }
+                    if (backwardCompatibleEntityId) {
+                        // create a sequence to generate localEntityIds for the class
+                        oSession.createSequenceIfAbsent(localEntityIdSequenceName(type), largestEntityId)
                     }
                 }
             }
         }
         return edgeClassesToCreate
     }
 
+    private fun ODatabaseSession.createSequenceIfAbsent(sequenceName: String, startingFrom: Long) {
+        val sequences = metadata.sequenceLibrary
+        if (sequences.getSequence(sequenceName) == null) {
+            val params = OSequence.CreateParams()
+            params.start = startingFrom
+            sequences.createSequence(sequenceName, OSequence.SEQUENCE_TYPE.ORDERED, params)
+        }
+    }
+
     private fun copyLinks(oSession: ODatabaseSession) {
         xodus.withReadonlyTx { xTx ->
             oSession.withCountingTx(entitiesPerTransaction) { countingTx ->
                 for (type in xTx.entityTypes) {
                     for (xEntity in xTx.getAll(type)) {
                         val oEntityId = xEntityIdToOEntityId.getValue(xEntity.id)
                         val oEntity = orient.getEntity(oEntityId)
+
                         var copiedSomeLinks = false
                         for (linkName in xEntity.linkNames) {
                             for (xTargetEntity in xEntity.getLinks(linkName)) {