Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/release-3.0' into release-3.1
Browse files Browse the repository at this point in the history
  • Loading branch information
@vladimir-zatsepin
vladimir-zatsepin committed Apr 2, 2024
2 parents 9919ccd + 9654946 commit fff23c0
Show file tree
Hide file tree
Showing 6 changed files with 142 additions and 40 deletions.
6 changes: 2 additions & 4 deletions build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ subprojects {
}

afterEvaluate {
if (shouldDeploy(this) && !(isDailyBuild && name == "xodus-tools")) {
if (shouldDeploy(this)) {
configure<PublishingExtension> {
repositories {
maven {
Expand Down Expand Up @@ -262,9 +262,7 @@ subprojects {
}
}
}
if (this.name == "xodus-tools") {
artifact(tasks.getByName("shadowJar"))
}

}
}
}
Expand Down
27 changes: 1 addition & 26 deletions entity-store/src/main/java/jetbrains/exodus/entitystore/FileSystemBlobVault.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,32 +67,7 @@ public File getBlobLocation(long blobHandle, boolean readonly) {
return result;
}

public long getBlobHandleByFile(@NotNull final File file) {
final String name = file.getName();
final String blobExtension = getBlobExtension();
final int blobExtensionStart = name.indexOf(blobExtension);
if (name.endsWith(blobExtension) && (blobExtensionStart == 2 || blobExtensionStart == 1)) {
try {
long result = Integer.parseInt(name.substring(0, blobExtensionStart), 16);
File f = file;
int shift = 0;
while (true) {
f = f.getParentFile();
if (f == null) {
break;
}
if (f.equals(getVaultLocation())) {
return result;
}
shift += Byte.SIZE;
result = result + (((long) Integer.parseInt(f.getName(), 16)) << shift);
}
} catch (NumberFormatException nfe) {
throw new EntityStoreException("Not a file of filesystem blob vault: " + file, nfe);
}
}
throw new EntityStoreException("Not a file of filesystem blob vault: " + file);
}


public static long getBlobHandleByFile(@NotNull File file, @NotNull final String blobExtension,
@Nullable File vaultLocation) {
Expand Down
79 changes: 79 additions & 0 deletions entity-store/src/main/java/jetbrains/exodus/entitystore/FileSystemBlobVaultOld.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -423,6 +423,85 @@ public void remove() {
};
}

public Iterator<Long> storedBlobHandles() {
return new Iterator<>() {
private final Deque<File> stack =
new ArrayDeque<>(Collections.singletonList(location.toFile()));
private final File[] EMPTY = new File[0];
private File[] files = EMPTY;
private int i = 0;

@Override
public boolean hasNext() {
while (true) {
if (i < files.length) {
final File file = files[i++];
if (file.isDirectory()) {
stack.push(file);
files = EMPTY;
i = 0;
} else {
final String name = file.getName();
if (name.endsWith(blobExtension)) {
return true;
}
}
} else {
if (stack.isEmpty()) {
return false;
}
final File dir = stack.pop();
files = dir.listFiles();
if (files == null) {
files = EMPTY;
}
i = 0;
}
}
}

@Override
public Long next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
final File file = files[i - 1];
return getBlobHandleByFile(file);
}

@Override
public void remove() {
throw new UnsupportedOperationException();
}
};
}
public long getBlobHandleByFile(@NotNull final File file) {
final String name = file.getName();
final String blobExtension = getBlobExtension();
final int blobExtensionStart = name.indexOf(blobExtension);
if (name.endsWith(blobExtension) && (blobExtensionStart == 2 || blobExtensionStart == 1)) {
try {
long result = Integer.parseInt(name.substring(0, blobExtensionStart), 16);
File f = file;
int shift = 0;
while (true) {
f = f.getParentFile();
if (f == null) {
break;
}
if (f.equals(getVaultLocation())) {
return result;
}
shift += Byte.SIZE;
result = result + (((long) Integer.parseInt(f.getName(), 16)) << shift);
}
} catch (NumberFormatException nfe) {
throw new EntityStoreException("Not a file of filesystem blob vault: " + file, nfe);
}
}
throw new EntityStoreException("Not a file of filesystem blob vault: " + file);
}

@NotNull
public File getBlobLocation(long blobHandle) {
return getBlobLocation(blobHandle, true);
Expand Down
52 changes: 47 additions & 5 deletions entity-store/src/main/java/jetbrains/exodus/entitystore/PersistentEntityStoreImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,38 @@ private void clearBrokenBlobs(PersistentStoreTransaction txn) {
}
}

private void clearNotRegisteredBlobs(PersistentStoreTransaction txn) {
logger.warn("Database: " + getLocation() +
" . Clearing blobs not registered inside database.");

var envTx = txn.getEnvironmentTransaction();
var blobsToRemove = new ArrayList<Long>();

if (blobVault instanceof FileSystemBlobVaultOld) {
var blobHandleIterator = ((FileSystemBlobVaultOld) blobVault).storedBlobHandles();
while (blobHandleIterator.hasNext()) {
var blobHandle = blobHandleIterator.next();
var blobLen = getBlobFileLength(blobHandle, envTx);
if (blobLen == null) {
blobsToRemove.add(blobHandle);
}
}

if (!blobsToRemove.isEmpty()) {
logger.warn("Database: " + getLocation() + ". " + blobsToRemove.size() +
" not registered BLOBs were found. Removing them.");

blobsToRemove.forEach(blob -> {
var blobLocation = blobVault.getBlobLocation(blob);
blobVault.delete(blob);
logger.warn("BLOB at " + blobLocation + " was removed.");
});
}
}

logger.warn("Database: " + getLocation() + ". Blobs not registered inside database were cleared.");
}

private void initBasicStores(Transaction envTxn) {
sequences = environment.openStore(SEQUENCES_STORE, StoreConfig.WITHOUT_DUPLICATES, envTxn);
blobFileLengths = environment.openStore(namingRulez.getBlobFileLengthsTable(),
Expand Down Expand Up @@ -326,31 +358,40 @@ private void applyRefactorings(final boolean fromScratch) {
if (config.getRefactoringClearBrokenBlobs()) {
executeInTransaction(txn -> clearBrokenBlobs((PersistentStoreTransaction) txn));
}
if (!useVersion1Format() && (fromScratch || Settings.get(internalSettings, "refactorBlobsForVersion2Format() applied") == null)) {
if (config.getRefactoringClearNotRegisteredBlobs()) {
executeInTransaction(txn -> clearNotRegisteredBlobs((PersistentStoreTransaction) txn));
}
if (!useVersion1Format() &&
(fromScratch ||
Settings.get(internalSettings, "refactorBlobsForVersion2Format() applied") == null)) {
if (!fromScratch) {
refactorings.refactorBlobsToVersion2Format(internalSettings);
}
Settings.set(internalSettings, "refactorBlobsForVersion2Format() applied", "y");
}
if (!useVersion1Format() && (fromScratch || Settings.get(internalSettings, "refactorEntitiesTablesToBitmap() applied") == null)) {
if (!useVersion1Format() && (fromScratch ||
Settings.get(internalSettings, "refactorEntitiesTablesToBitmap() applied") == null)) {
if (!fromScratch) {
refactorings.refactorEntitiesTablesToBitmap(internalSettings);
}
Settings.set(internalSettings, "refactorEntitiesTablesToBitmap() applied", "y");
}
if (!useVersion1Format() && useIntForLocalId() && (fromScratch || Settings.get(internalSettings, "refactorAllPropsIndexToBitmap() applied") == null)) {
if (!useVersion1Format() && useIntForLocalId() && (fromScratch ||
Settings.get(internalSettings, "refactorAllPropsIndexToBitmap() applied") == null)) {
if (!fromScratch) {
refactorings.refactorAllPropsIndexToBitmap();
}
Settings.set(internalSettings, "refactorAllPropsIndexToBitmap() applied", "y");
}
if (!useVersion1Format() && useIntForLocalId() && (fromScratch || Settings.get(internalSettings, "refactorAllLinksIndexToBitmap() applied") == null)) {
if (!useVersion1Format() && useIntForLocalId() && (fromScratch ||
Settings.get(internalSettings, "refactorAllLinksIndexToBitmap() applied") == null)) {
if (!fromScratch) {
refactorings.refactorAllLinksIndexToBitmap();
}
Settings.set(internalSettings, "refactorAllLinksIndexToBitmap() applied", "y");
}
if (!useVersion1Format() && useIntForLocalId() && (fromScratch || Settings.get(internalSettings, "refactorAllBlobsIndexToBitmap() applied") == null)) {
if (!useVersion1Format() && useIntForLocalId() && (fromScratch ||
Settings.get(internalSettings, "refactorAllBlobsIndexToBitmap() applied") == null)) {
if (!fromScratch) {
refactorings.refactorAllBlobsIndexToBitmap();
}
Expand Down Expand Up @@ -2092,6 +2133,7 @@ public void remove() { // don't give a damn
}
);
}

public void deleteEntityType(@NotNull final String entityTypeName) {
final PersistentStoreTransaction txn = getAndCheckCurrentTransaction();
final int entityTypeId = entityTypes.delete(txn, entityTypeName);
Expand Down
9 changes: 9 additions & 0 deletions openAPI/src/main/java/jetbrains/exodus/entitystore/PersistentEntityStoreConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ public PersistentEntityStoreConfig setMutable(boolean isMutable) {
* <p>Mutable at runtime: no
*/
public static final String REFACTORING_CLEAR_BROKEN_BLOBS = "exodus.entityStore.refactoring.clearBrokenBlobs";
public static final String REFACTORING_CLEAR_NOT_REGISTERED_BLOBS = "exodus.entityStore.refactoring.clearNotRegisteredBlobs";

/**
* Not for public use, for debugging and troubleshooting purposes. Default value is {@code 30}.
Expand Down Expand Up @@ -468,6 +469,7 @@ public PersistentEntityStoreConfig(@NotNull final ConfigurationStrategy strategy
new Pair(REFACTORING_HEAVY_PROPS, false),
new Pair(REFACTORING_DELETE_REDUNDANT_BLOBS, false),
new Pair(REFACTORING_CLEAR_BROKEN_BLOBS, false),
new Pair(REFACTORING_CLEAR_NOT_REGISTERED_BLOBS, false),
new Pair(REFACTORING_DEDUPLICATE_BLOBS_EVERY, 30),
new Pair(REFACTORING_DEDUPLICATE_BLOBS_MIN_SIZE, 10),
new Pair(MAX_IN_PLACE_BLOB_SIZE, 10000),
Expand Down Expand Up @@ -591,6 +593,13 @@ public boolean getRefactoringClearBrokenBlobs() {
return getRefactoringForceAll() || (Boolean) (getSetting(REFACTORING_CLEAR_BROKEN_BLOBS));
}

public PersistentEntityStoreConfig setRefactoringClearNotRegisteredBlobs(final boolean clearBrokenBlobs) {
return setSetting(REFACTORING_CLEAR_NOT_REGISTERED_BLOBS, clearBrokenBlobs);
}

public boolean getRefactoringClearNotRegisteredBlobs() {
return getRefactoringForceAll() || (Boolean) (getSetting(REFACTORING_CLEAR_NOT_REGISTERED_BLOBS));
}

public int getRefactoringDeduplicateBlobsEvery() {
return getRefactoringForceAll() ? 0 : (Integer) getSetting(REFACTORING_DEDUPLICATE_BLOBS_EVERY);
Expand Down
9 changes: 4 additions & 5 deletions tools/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@ val testArtifacts: Configuration by configurations.creating

tasks {
shadowJar {
mustRunAfter(jar)
archiveFileName.set(jar.get().archiveFileName)
manifest {
attributes["Main-Class"] = "jetbrains.exodus.MainKt"
}
}


jar {
enabled = false
finalizedBy(shadowJar)
}

val jarTest by creating(Jar::class) {
Expand All @@ -34,8 +37,4 @@ tasks {
add("default", shadowJar)
add("testArtifacts", jarTest)
}

build {
dependsOn(shadowJar)
}
}

0 comments on commit fff23c0

Please sign in to comment.