Skip to content

Commit

Permalink
XD-1005 copy localEntityId for each entity on the data migration step…
Browse files Browse the repository at this point in the history 
… if backward compatible EntityId is enabled
  • Loading branch information
@kirillvasilenko
kirillvasilenko committed Apr 12, 2024
1 parent 9b28607 commit 861f69f
Show file tree
Hide file tree
Showing 3 changed files with 116 additions and 11 deletions.
5 changes: 5 additions & 0 deletions entity-store/src/main/kotlin/jetbrains/exodus/entitystore/orientdb/OVertexEntity.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,13 @@ class OVertexEntity(private var vertex: OVertex, private val store: PersistentEn

const val STRING_BLOB_CLASS_NAME: String = "StringBlob"

// Backward compatible EntityId

const val CLASS_ID_CUSTOM_PROPERTY_NAME = "classId"
const val CLASS_ID_SEQUENCE_NAME = "sequence_classId"

const val BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME = "backwardCompatibleLocalEntityId"
fun localEntityIdSequenceName(className: String): String = "${className}_sequence_localEntityId"
}

private val activeSession get() = ODatabaseSession.getActiveSession()
Expand Down
39 changes: 31 additions & 8 deletions query/src/main/kotlin/jetbrains/exodus/query/metadata/XodusToOrientDataMigrator.kt
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
package jetbrains.exodus.query.metadata

import com.orientechnologies.orient.core.db.ODatabaseSession
import com.orientechnologies.orient.core.metadata.schema.OType
import com.orientechnologies.orient.core.metadata.sequence.OSequence
import jetbrains.exodus.entitystore.EntityId
import jetbrains.exodus.entitystore.PersistentEntityStore
import jetbrains.exodus.entitystore.orientdb.OPersistentEntityStore
import jetbrains.exodus.entitystore.orientdb.OVertexEntity
import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME
import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.BINARY_BLOB_CLASS_NAME
import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.CLASS_ID_CUSTOM_PROPERTY_NAME
import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.CLASS_ID_SEQUENCE_NAME
import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.localEntityIdSequenceName
import jetbrains.exodus.entitystore.orientdb.withSession

fun migrateDataFromXodusToOrientDb(
Expand Down Expand Up @@ -42,9 +45,6 @@ internal class XodusToOrientDataMigrator(
) {
private val xEntityIdToOEntityId = HashMap<EntityId, EntityId>()

// it is required for backward compatible EntityId
private val classNameToLargestEntityId = HashMap<String, Long>()

fun migrate() {
orient.databaseProvider.withSession { oSession ->
createVertexClassesIfAbsent(oSession)
Expand All @@ -66,18 +66,22 @@ internal class XodusToOrientDataMigrator(
if (backwardCompatibleEntityId) {
oClass.setCustom(CLASS_ID_CUSTOM_PROPERTY_NAME, classId.toString())
maxClassId = maxOf(maxClassId, classId)

// create localEntityId property if absent
if (oClass.getProperty(BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME) == null) {
oClass.createProperty(BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME, OType.LONG)
}
}
}
}

if (backwardCompatibleEntityId) {
// create a sequence to generate classIds
val sequences = oSession.metadata.sequenceLibrary

require(sequences.getSequence(CLASS_ID_SEQUENCE_NAME) == null) { "$CLASS_ID_SEQUENCE_NAME is already created. It means that some data migration has happened to the target database before. Such a scenario is not supported." }

val params = OSequence.CreateParams()
params.start = maxClassId.toLong()
sequences.createSequence(CLASS_ID_SEQUENCE_NAME, OSequence.SEQUENCE_TYPE.ORDERED, params)
oSession.createSequenceIfAbsent(CLASS_ID_SEQUENCE_NAME, maxClassId.toLong())
}

oSession.getClass(BINARY_BLOB_CLASS_NAME) ?: oSession.createClass(BINARY_BLOB_CLASS_NAME)
Expand Down Expand Up @@ -109,16 +113,35 @@ internal class XodusToOrientDataMigrator(
xEntityIdToOEntityId[xEntity.id] = oEntity.id
countingTx.increment()

largestEntityId = maxOf(largestEntityId, xEntity.id.localId)
edgeClassesToCreate.addAll(xEntity.linkNames)

if (backwardCompatibleEntityId) {
// copy localEntityId
val localEntityId = xEntity.id.localId
oEntity.setProperty(BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME, localEntityId)

largestEntityId = maxOf(largestEntityId, localEntityId)
}
}
if (backwardCompatibleEntityId) {
// create a sequence to generate localEntityIds for the class
oSession.createSequenceIfAbsent(localEntityIdSequenceName(type), largestEntityId)
}
classNameToLargestEntityId[type] = largestEntityId
}
}
}
return edgeClassesToCreate
}

private fun ODatabaseSession.createSequenceIfAbsent(sequenceName: String, startingFrom: Long) {
val sequences = metadata.sequenceLibrary
if (sequences.getSequence(sequenceName) == null) {
val params = OSequence.CreateParams()
params.start = startingFrom
sequences.createSequence(sequenceName, OSequence.SEQUENCE_TYPE.ORDERED, params)
}
}

private fun copyLinks(oSession: ODatabaseSession) {
xodus.withReadonlyTx { xTx ->
oSession.withCountingTx(entitiesPerTransaction) { countingTx ->
Expand Down
83 changes: 80 additions & 3 deletions query/src/test/kotlin/jetbrains/exodus/query/metadata/MigrateDataTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@ import com.orientechnologies.orient.core.record.impl.OVertexDocument
import jetbrains.exodus.entitystore.StoreTransaction
import jetbrains.exodus.entitystore.XodusTestDB
import jetbrains.exodus.entitystore.orientdb.OVertexEntity
import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME
import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.CLASS_ID_CUSTOM_PROPERTY_NAME
import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.CLASS_ID_SEQUENCE_NAME
import jetbrains.exodus.entitystore.orientdb.OVertexEntity.Companion.localEntityIdSequenceName
import jetbrains.exodus.entitystore.orientdb.testutil.InMemoryOrientDB
import junit.framework.TestCase.assertNull
import org.junit.Assert
import org.junit.Rule
import org.junit.Test
Expand Down Expand Up @@ -166,10 +169,84 @@ class MigrateDataTest {
}
}

@Test
fun `if backward compatible EntityId enabled, copy localEntityId for every entity and create a sequence for every class to generate new localEntityIds`() {
val entities = pileOfEntities(
eProps("type1", 1),
eProps("type1", 2),
eProps("type1", 3),

eProps("type2", 2),
eProps("type2", 4),
eProps("type2", 5),
)
xodus.withTx { tx ->
tx.createEntities(entities)
}

migrateDataFromXodusToOrientDb(xodus.store, orientDb.store, backwardCompatibleEntityId = true)

xodus.withTx { xTx ->
orientDb.withSession { oSession ->
for (type in xTx.entityTypes) {
val xTestIdToLocalEntityId = HashMap<Int, Long>()
val oTestIdToLocalEntityId = HashMap<Int, Long>()
var maxLocalEntityId = 0L
for (xEntity in xTx.getAll(type)) {
val testId = xEntity.getProperty("id") as Int
val localEntityId = xEntity.id.localId
xTestIdToLocalEntityId[testId] = localEntityId
maxLocalEntityId = maxOf(maxLocalEntityId, localEntityId)
}

for (oEntity in oSession.browseClass(type)) {
val testId = oEntity.getTestId()
val localEntityId = oEntity.getProperty<Long>(BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME)
oTestIdToLocalEntityId[testId] = localEntityId
}

assertTrue(maxLocalEntityId > 0)
val nextGeneratedLocalEntityId = oSession.metadata.sequenceLibrary.getSequence(localEntityIdSequenceName(type)).next()
assertEquals(maxLocalEntityId + 1, nextGeneratedLocalEntityId)

assertEquals(xTestIdToLocalEntityId, oTestIdToLocalEntityId)
}
}
}
}

@Test
fun `if backward compatible EntityId disabled, ignore localEntityId`() {
val entities = pileOfEntities(
eProps("type1", 1),
eProps("type1", 2),
eProps("type1", 3),

eProps("type2", 2),
eProps("type2", 4),
eProps("type2", 5),
)
xodus.withTx { tx ->
tx.createEntities(entities)
}

migrateDataFromXodusToOrientDb(xodus.store, orientDb.store)

xodus.withTx { xTx ->
orientDb.withSession { oSession ->
for (type in xTx.entityTypes) {
val oClass = oSession.getClass(type)
assertNull(oClass.getProperty(BACKWARD_COMPATIBLE_LOCAL_ENTITY_ID_PROPERTY_NAME))
assertNull(oSession.metadata.sequenceLibrary.getSequence(localEntityIdSequenceName(type)))
}
}
}
}

private fun ODatabaseSession.assertOrientContainsAllTheEntities(pile: PileOfEntities) {
for (type in pile.types) {
for (record in this.browseClass(type)) {
val entity = pile.getEntity(type, record.getId())
val entity = pile.getEntity(type, record.getTestId())
record.assertEquals(entity)
}
}
Expand All @@ -179,7 +256,7 @@ class MigrateDataTest {
val actualDocument = this
val actual = OVertexEntity(actualDocument as OVertexDocument, orientDb.store)

Assert.assertEquals(expected.id, actualDocument.getId())
Assert.assertEquals(expected.id, actualDocument.getTestId())
for ((propName, propValue) in expected.props) {
Assert.assertEquals(propValue, actual.getProperty(propName))
}
Expand All @@ -195,7 +272,7 @@ class MigrateDataTest {
}
}

private fun ODocument.getId(): Int = getProperty("id")
private fun ODocument.getTestId(): Int = getProperty("id")

private fun StoreTransaction.createEntities(pile: PileOfEntities) {
for (type in pile.types) {
Expand Down

0 comments on commit 861f69f

Please sign in to comment.