Skip to content

Commit

Permalink
MCR-3126 rebase main
Browse files Browse the repository at this point in the history
  • Loading branch information
@Mewel
Mewel committed Jul 4, 2024
1 parent c50df19 commit 4ee8872
Show file tree
Hide file tree
Showing 68 changed files with 7,309 additions and 193 deletions.
12 changes: 6 additions & 6 deletions ...-ocfl/src/main/java/org/mycore/ocfl/classification/MCROCFLClassificationEventHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@

package org.mycore.ocfl.classification;

import static org.mycore.ocfl.MCROCFLPersistenceTransaction.addClassficationEvent;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.mycore.common.MCRException;
Expand All @@ -28,6 +26,8 @@
import org.mycore.datamodel.classifications2.MCRCategory;
import org.mycore.datamodel.common.MCRAbstractMetadataVersion;

import static org.mycore.ocfl.classification.MCROCFLClassificationTransaction.addClassificationEvent;

/**
* @author Tobias Lenhardt [Hammer1279]
*/
Expand All @@ -41,15 +41,15 @@ public void doHandleEvent(MCREvent evt) throws MCRException {
MCRCategory mcrCg = (MCRCategory) evt.get(MCREvent.CLASS_KEY);
LOGGER.debug("{} handling {} {}", getClass().getName(), mcrCg.getId(), evt.getEventType());
switch (evt.getEventType()) {
case CREATE -> addClassficationEvent(mcrCg.getRoot().getId(), MCRAbstractMetadataVersion.CREATED);
case UPDATE -> addClassficationEvent(mcrCg.getRoot().getId(), MCRAbstractMetadataVersion.UPDATED);
case CREATE -> addClassificationEvent(mcrCg.getRoot().getId(), MCRAbstractMetadataVersion.CREATED);
case UPDATE -> addClassificationEvent(mcrCg.getRoot().getId(), MCRAbstractMetadataVersion.UPDATED);
case DELETE -> {
if (mcrCg.getId().isRootID()) {
// delete complete classification
addClassficationEvent(mcrCg.getRoot().getId(), MCRAbstractMetadataVersion.DELETED);
addClassificationEvent(mcrCg.getRoot().getId(), MCRAbstractMetadataVersion.DELETED);
} else {
// update classification to new version
addClassficationEvent(mcrCg.getRoot().getId(), MCRAbstractMetadataVersion.UPDATED);
addClassificationEvent(mcrCg.getRoot().getId(), MCRAbstractMetadataVersion.UPDATED);
}
}
default -> LOGGER.error("No Method available for {}", evt.getEventType());
Expand Down
21 changes: 10 additions & 11 deletions ...e/ocfl/MCROCFLPersistenceTransaction.java → ...ion/MCROCFLClassificationTransaction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,7 @@
* along with MyCoRe. If not, see <http://www.gnu.org/licenses/>.
*/

package org.mycore.ocfl;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
package org.mycore.ocfl.classification;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand All @@ -37,13 +31,18 @@
import org.mycore.datamodel.classifications2.MCRCategoryID;
import org.mycore.datamodel.classifications2.utils.MCRCategoryTransformer;
import org.mycore.datamodel.common.MCRAbstractMetadataVersion;
import org.mycore.ocfl.classification.MCROCFLXMLClassificationManager;

import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;

/**
* @author Tobias Lenhardt [Hammer1279]
* @author Thomas Scheffler (yagee)
*/
public class MCROCFLPersistenceTransaction implements MCRPersistenceTransaction {
public class MCROCFLClassificationTransaction implements MCRPersistenceTransaction {

private static final Logger LOGGER = LogManager.getLogger();

Expand All @@ -59,7 +58,7 @@ public class MCROCFLPersistenceTransaction implements MCRPersistenceTransaction

private boolean active;

public MCROCFLPersistenceTransaction() {
public MCROCFLClassificationTransaction() {
rollbackOnly = false;
active = false;
}
Expand Down Expand Up @@ -172,7 +171,7 @@ public boolean isActive() {
* @param id The ID of the Classification
* @param type 'A' for created, 'M' for modified, 'D' deleted
*/
public static void addClassficationEvent(MCRCategoryID id, char type) {
public static void addClassificationEvent(MCRCategoryID id, char type) {
if (!Objects.requireNonNull(id).isRootID()) {
throw new IllegalArgumentException("Only root category ids are allowed: " + id);
}
Expand Down
25 changes: 12 additions & 13 deletions ...re-ocfl/src/main/java/org/mycore/ocfl/classification/MCROCFLXMLClassificationManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,12 @@

package org.mycore.ocfl.classification;

import java.io.IOException;
import java.io.InputStream;
import java.time.ZoneOffset;
import java.util.Date;
import java.util.Map;
import java.util.Objects;

import io.ocfl.api.OcflOption;
import io.ocfl.api.OcflRepository;
import io.ocfl.api.exception.NotFoundException;
import io.ocfl.api.exception.ObjectOutOfSyncException;
import io.ocfl.api.model.ObjectVersionId;
import io.ocfl.api.model.VersionInfo;
import org.jdom2.Document;
import org.jdom2.JDOMException;
import org.mycore.common.MCRPersistenceException;
Expand All @@ -40,12 +39,12 @@
import org.mycore.ocfl.util.MCROCFLMetadataVersion;
import org.mycore.ocfl.util.MCROCFLObjectIDPrefixHelper;

import io.ocfl.api.OcflOption;
import io.ocfl.api.OcflRepository;
import io.ocfl.api.exception.NotFoundException;
import io.ocfl.api.exception.ObjectOutOfSyncException;
import io.ocfl.api.model.ObjectVersionId;
import io.ocfl.api.model.VersionInfo;
import java.io.IOException;
import java.io.InputStream;
import java.time.ZoneOffset;
import java.util.Date;
import java.util.Map;
import java.util.Objects;

/**
* OCFL File Manager for MyCoRe Classifications
Expand Down
145 changes: 138 additions & 7 deletions mycore-ocfl/src/main/java/org/mycore/ocfl/commands/MCROCFLCommands.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,20 @@

package org.mycore.ocfl.commands;

import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
Expand All @@ -34,7 +44,9 @@
import org.jdom2.JDOMException;
import org.mycore.common.MCRUsageException;
import org.mycore.common.config.MCRConfiguration2;
import org.mycore.common.config.MCRConfigurationDir;
import org.mycore.common.content.MCRContent;
import org.mycore.common.digest.MCRDigest;
import org.mycore.datamodel.classifications2.MCRCategory;
import org.mycore.datamodel.classifications2.MCRCategoryDAO;
import org.mycore.datamodel.classifications2.MCRCategoryDAOFactory;
Expand All @@ -44,20 +56,28 @@
import org.mycore.datamodel.common.MCRAbstractMetadataVersion;
import org.mycore.datamodel.common.MCRXMLMetadataManager;
import org.mycore.datamodel.metadata.MCRObjectID;
import org.mycore.datamodel.niofs.MCRFileAttributes;
import org.mycore.datamodel.niofs.MCRPath;
import org.mycore.datamodel.niofs.MCRVersionedPath;
import org.mycore.datamodel.niofs.utils.MCRTreeCopier;
import org.mycore.frontend.cli.annotation.MCRCommand;
import org.mycore.frontend.cli.annotation.MCRCommandGroup;
import org.mycore.ocfl.MCROCFLPersistenceTransaction;
import org.mycore.ocfl.classification.MCROCFLClassificationTransaction;
import org.mycore.ocfl.classification.MCROCFLXMLClassificationManager;
import org.mycore.ocfl.metadata.MCROCFLXMLMetadataManager;
import org.mycore.ocfl.metadata.migration.MCROCFLMigration;
import org.mycore.ocfl.metadata.migration.MCROCFLRevisionPruner;
import org.mycore.ocfl.niofs.MCROCFLFileSystemProvider;
import org.mycore.ocfl.repository.MCROCFLRepositoryProvider;
import org.mycore.ocfl.user.MCROCFLXMLUserManager;
import org.mycore.ocfl.util.MCROCFLObjectIDPrefixHelper;
import org.mycore.user2.MCRUser;
import org.mycore.user2.MCRUserManager;
import org.xml.sax.SAXException;

import com.google.common.collect.MapDifference;
import com.google.common.collect.Maps;

import io.ocfl.api.OcflRepository;

@SuppressWarnings("JavaUtilDate")
Expand All @@ -78,7 +98,6 @@ public class MCROCFLCommands {

private static boolean confirmPurgeMarked = false;


protected static void migrateWithPrunersAndRepositoryKeyOrMetadataManager(String repository,
String metadataManagerConfigKey,
String prunersStringList) throws Exception {
Expand All @@ -98,7 +117,7 @@ protected static void migrateWithPrunersAndRepositoryKeyOrMetadataManager(String
= MCRConfiguration2.getInstanceOf(MCROCFLXMLMetadataManager.class, metadataManagerConfigKey)
.orElseThrow(() -> MCRConfiguration2.createConfigurationException(metadataManagerConfigKey));
migration = new MCROCFLMigration(null, prunerList, metadataManager);
} else if(repository != null && !repository.isEmpty()) {
} else if (repository != null && !repository.isEmpty()) {
migration = new MCROCFLMigration(repository, prunerList);
} else {
throw new MCRUsageException("Either a repository or a metadata manager must be specified");
Expand Down Expand Up @@ -170,14 +189,14 @@ public static List<String> updateOCFLClassifications() {
help = "Update classification {0} in the OCFL Store from database")
public static void updateOCFLClassification(String classId) {
final MCRCategoryID rootID = MCRCategoryID.rootID(classId);
MCROCFLPersistenceTransaction.addClassficationEvent(rootID, MCRAbstractMetadataVersion.UPDATED);
MCROCFLClassificationTransaction.addClassificationEvent(rootID, MCRAbstractMetadataVersion.UPDATED);
}

@MCRCommand(syntax = "delete ocfl classification {0}",
help = "Delete classification {0} in the OCFL Store")
public static void deleteOCFLClassification(String classId) {
final MCRCategoryID rootID = MCRCategoryID.rootID(classId);
MCROCFLPersistenceTransaction.addClassficationEvent(rootID, MCRAbstractMetadataVersion.DELETED);
MCROCFLClassificationTransaction.addClassificationEvent(rootID, MCRAbstractMetadataVersion.DELETED);
}

@MCRCommand(syntax = "sync ocfl classifications",
Expand Down Expand Up @@ -409,11 +428,95 @@ public static void purgeMarkedUsers() throws IOException {
confirmPurgeMarked = false;
}

@MCRCommand(syntax = "migrate derivates to ocfl", help = "migrates all ifs2 derivates to ocfl")
public static List<String> migrateDerivates() {
List<String> derivateIds = MCRXMLMetadataManager.instance().listIDsOfType("derivate");
return derivateIds.stream().map(derivateId -> {
return "migrate derivate " + derivateId + " to ocfl";
}).toList();
}

@MCRCommand(syntax = "migrate derivate {0} to ocfl", help = "migrate an ifs2 derivate to ocfl")
public static void migrateDerivate(String derivateId) throws IOException {
MCRPath source = MCRPath.getPath(derivateId, "/");
LOGGER.info("migrating {} to ocfl...", derivateId);
if (!Files.exists(source)) {
throw new NoSuchFileException(source + " does not exist");
}
MCROCFLFileSystemProvider ocflFileSystemProvider = MCROCFLFileSystemProvider.get();
MCRVersionedPath target = ocflFileSystemProvider.getPath(derivateId, "/");
if (Files.exists(target)) {
throw new FileAlreadyExistsException(target + " already exists");
}
ocflFileSystemProvider.getFileSystem().createRoot(derivateId);
Files.walkFileTree(source, new MCRTreeCopier(source, target));
}

@MCRCommand(syntax = "validate ocfl derivates", help = "checks if all derivates are synchron in ifs2 and ocfl")
public static List<String> validateDerivates() throws IOException {
Path errorFilePath = getDerivateMigrationErrorReportPath();
Files.deleteIfExists(errorFilePath);
LOGGER.info("Validation errors will be written to: '{}'. If this file does not exists, all derivates "
+ "are successfully migrated to ocfl and can be removed from ifs2.", errorFilePath);
List<String> derivateIds = MCRXMLMetadataManager.instance().listIDsOfType("derivate");
return derivateIds.stream().map(derivateId -> {
return "validate ocfl derivate " + derivateId;
}).toList();
}

@MCRCommand(syntax = "validate ocfl derivate {0}",
help = "checks if the derivate has the same digests in ifs2 and ocfl")
public static void validateDerivate(String derivateId) throws IOException {
Map<String, MCRDigest> ifs2Map = new HashMap<>();
Map<String, MCRDigest> ocflMap = new HashMap<>();

// collect from ifs2
MCRPath source = MCRPath.getPath(derivateId, "/");
Files.walkFileTree(source, new DigestFileVisitor(ifs2Map));

// collect from ocfl
MCROCFLFileSystemProvider ocflFileSystemProvider = MCROCFLFileSystemProvider.get();
MCRVersionedPath target = ocflFileSystemProvider.getPath(derivateId, "/");
Files.walkFileTree(target, new DigestFileVisitor(ocflMap));

// validate
Map<String, String> errorMap = new HashMap<>();
MapDifference<String, MCRDigest> difference = Maps.difference(ifs2Map, ocflMap);
difference.entriesDiffering().forEach((path, diff) -> {
errorMap.put(path, "ifs digest '" + diff.leftValue() + "' vs ocfl digest '" + diff.rightValue() + "'");
});
difference.entriesOnlyOnLeft().forEach((path, diff) -> {
errorMap.put(path, "exist in ifs2 but not in ocfl");
});
difference.entriesOnlyOnRight().forEach((path, diff) -> {
errorMap.put(path, "exist in ocfl but not in ifs2");
});

if (!errorMap.isEmpty()) {
Path errorFilePath = getDerivateMigrationErrorReportPath();
// Convert the map entries to a list of strings
List<String> errorLines = errorMap.entrySet().stream()
.map(entry -> entry.getKey() + ": " + entry.getValue())
.toList();
Files.write(errorFilePath, errorLines, StandardOpenOption.CREATE, StandardOpenOption.APPEND);
LOGGER.info("Validation error in '{}'. See '{}' for details.", derivateId, errorFilePath.getFileName());
}
}

private static Path getDerivateMigrationErrorReportPath() throws IOException {
File configurationDirectory = MCRConfigurationDir.getConfigurationDirectory();
if (configurationDirectory == null) {
throw new IOException("Configuration directory not set!");
}
Path confPath = configurationDirectory.toPath();
return confPath.resolve("ocfl-derivate-migration-errors");
}

private static List<String> getStaleOCFLClassificationIDs() {
String repositoryKey = MCRConfiguration2.getStringOrThrow("MCR.Classification.Manager.Repository");
List<String> classDAOList = new MCRCategoryDAOImpl().getRootCategoryIDs().stream()
.map(MCRCategoryID::toString)
.collect(Collectors.toList());
.toList();
OcflRepository repository = MCROCFLRepositoryProvider.getRepository(repositoryKey);
return repository.listObjectIds()
.filter(obj -> obj.startsWith(MCROCFLObjectIDPrefixHelper.CLASSIFICATION))
Expand All @@ -429,7 +532,7 @@ private static List<String> getStaleOCFLUserIDs() {
String repositoryKey = MCRConfiguration2.getStringOrThrow("MCR.Users.Manager.Repository");
List<String> userEMList = MCRUserManager.listUsers("*", null, null, null).stream()
.map(MCRUser::getUserID)
.collect(Collectors.toList());
.toList();
OcflRepository repository = MCROCFLRepositoryProvider.getRepository(repositoryKey);
return repository.listObjectIds()
.filter(obj -> obj.startsWith(MCROCFLObjectIDPrefixHelper.USER))
Expand All @@ -440,4 +543,32 @@ private static List<String> getStaleOCFLUserIDs() {
.filter(Predicate.not(userEMList::contains))
.collect(Collectors.toList());
}

private static class DigestFileVisitor extends SimpleFileVisitor<Path> {

private final Map<String, MCRDigest> map;

DigestFileVisitor(Map<String, MCRDigest> map) {
this.map = map;
}

@Override
public FileVisitResult visitFile(Path path, BasicFileAttributes basicFileAttributes) throws IOException {
String relativePath = MCRPath.toMCRPath(path).getOwnerRelativePath();
if (basicFileAttributes instanceof MCRFileAttributes<?> mcrFileAttributes) {
map.put(relativePath, mcrFileAttributes.digest());
} else {
throw new IOException("Path '" + path + "' should have MCRFileAttributes.");
}
return FileVisitResult.CONTINUE;
}

@Override
public FileVisitResult postVisitDirectory(Path path, IOException e) {
map.put(MCRPath.toMCRPath(path).getOwnerRelativePath(), null);
return FileVisitResult.CONTINUE;
}

}

}
15 changes: 7 additions & 8 deletions mycore-ocfl/src/main/java/org/mycore/ocfl/commands/MCROCFLRegexCommands.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,7 @@

package org.mycore.ocfl.commands;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;

import io.ocfl.api.OcflRepository;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.mycore.common.config.MCRConfiguration2;
Expand All @@ -37,7 +31,12 @@
import org.mycore.ocfl.user.MCROCFLXMLUserManager;
import org.mycore.ocfl.util.MCROCFLObjectIDPrefixHelper;

import io.ocfl.api.OcflRepository;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;

/**
* All OCFL commands utilizing RegEx for bulk operations
Expand Down
Loading

0 comments on commit 4ee8872

Please sign in to comment.