From d6c8495c101fe72452455b5e8b6aa5b92c756127 Mon Sep 17 00:00:00 2001 From: Matthias Eichner Date: Tue, 3 Sep 2024 14:27:57 +0200 Subject: [PATCH] MCR-3126 update OCFL * update ocfl-java version * support file size * support filekey --- .../ocfl/niofs/MCROCFLFileAttributes.java | 1 + .../ocfl/niofs/MCROCFLFileSystemProvider.java | 28 +- .../ocfl/niofs/MCROCFLLocalVirtualObject.java | 48 +-- .../niofs/MCROCFLRemoteVirtualObject.java | 39 +-- .../ocfl/niofs/MCROCFLVirtualObject.java | 308 +++++++++++------- .../niofs/storage/MCROCFLHybridStorage.java | 6 +- .../niofs/MCROCFLBasicFileAttributesTest.java | 18 +- .../MCROCFLEmptyDirectoryTrackerTest.java | 4 + .../ocfl/niofs/MCROCFLFileStoreTest.java | 4 + .../niofs/MCROCFLFileSystemProviderTest.java | 38 ++- .../ocfl/niofs/MCROCFLFileSystemTest.java | 4 + .../MCROCFLFileSystemTransactionTest.java | 4 + .../niofs/MCROCFLFileTypeDetectorTest.java | 4 + .../mycore/ocfl/niofs/MCROCFLTestCase.java | 78 ++++- .../MCROCFLVirtualObjectProviderTest.java | 4 + .../ocfl/niofs/MCRVirtualObjectTest.java | 52 ++- ...faultTransactionalTempFileStorageTest.java | 8 +- .../storage/MCROCFLHybridStorageTest.java | 14 +- .../MCROCFLRollingCacheStorageTest.java | 4 + .../niofs/storage/MCROCFLStorageTestCase.java | 4 + .../repository/MCROCFLRepositoryTest.java | 7 +- .../src/test/resources/mycore.properties | 4 +- pom.xml | 2 +- 23 files changed, 464 insertions(+), 219 deletions(-) diff --git a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLFileAttributes.java b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLFileAttributes.java index 5b04c9d864..511dedcf09 100644 --- a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLFileAttributes.java +++ b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLFileAttributes.java @@ -25,6 +25,7 @@ import org.mycore.datamodel.niofs.MCRFileAttributes; import org.mycore.datamodel.niofs.MCRVersionedPath; +// TODO javadoc public class MCROCFLFileAttributes implements MCRFileAttributes { private final FileTime creationTime; diff --git a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLFileSystemProvider.java b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLFileSystemProvider.java index 3597c02d44..ccbfa16dd0 100644 --- a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLFileSystemProvider.java +++ b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLFileSystemProvider.java @@ -23,6 +23,7 @@ import java.nio.channels.SeekableByteChannel; import java.nio.file.AccessMode; import java.nio.file.CopyOption; +import java.nio.file.DirectoryNotEmptyException; import java.nio.file.DirectoryStream; import java.nio.file.DirectoryStream.Filter; import java.nio.file.FileAlreadyExistsException; @@ -97,7 +98,8 @@ void init() throws IOException { .getSingleInstanceOfOrThrow(MCROCFLTransactionalTempFileStorage.class, tempFileClassProperty); Files.createDirectories(this.localStorage.getRoot()); boolean remote = MCRConfiguration2.getBoolean(configurationPrefix + "FS.Remote").orElseThrow(); - this.virtualObjectProvider = new MCROCFLVirtualObjectProvider(getRepository(), localStorage, remote); + this.virtualObjectProvider + = new MCROCFLVirtualObjectProvider(getRepository(), localStorage, remote); } catch (Exception exception) { throw new IOException("Unable to create MCROCFLFileSystem.", exception); } @@ -245,7 +247,7 @@ public void copy(MCRVersionedPath source, MCRVersionedPath target, CopyOption... throw new NoSuchFileException(source.toString()); } if (virtualSource.isDirectory(source)) { - createDirectory(target); + copyDirectory(target, options); } else { copyFile(source, target, options); } @@ -291,9 +293,6 @@ public void move(MCRVersionedPath source, MCRVersionedPath target, CopyOption... delete(source); } - /** - * {@inheritDoc} - */ private void copyFile(MCRVersionedPath source, MCRVersionedPath target, CopyOption... options) throws IOException { MCROCFLVirtualObject virtualSource = virtualObjectProvider().get(source); MCROCFLVirtualObject virtualTarget = virtualObjectProvider().getWritable(target); @@ -304,11 +303,26 @@ private void copyFile(MCRVersionedPath source, MCRVersionedPath target, CopyOpti } if (virtualSource.equals(virtualTarget)) { // same virtual object - virtualSource.copyFile(source, target, options); + virtualSource.copy(source, target, options); } else { // different virtual object - virtualSource.copyFileToVirtualObject(virtualTarget, source, target, options); + virtualSource.externalCopy(virtualTarget, source, target, options); + } + } + + private void copyDirectory(MCRVersionedPath target, CopyOption... options) + throws IOException { + MCROCFLVirtualObject virtualTarget = virtualObjectProvider().getWritable(target); + boolean targetExists = virtualTarget.exists(target); + boolean replaceExisting = Arrays.asList(options).contains(StandardCopyOption.REPLACE_EXISTING); + if (targetExists && replaceExisting) { + boolean targetIsDirectory = virtualTarget.isDirectory(target); + boolean targetDirectoryIsEmpty = virtualTarget.isDirectoryEmpty(target); + if (targetIsDirectory && !targetDirectoryIsEmpty) { + throw new DirectoryNotEmptyException(target.toString()); + } } + createDirectory(target); } /** diff --git a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLLocalVirtualObject.java b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLLocalVirtualObject.java index a1540521b1..e9c335712d 100644 --- a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLLocalVirtualObject.java +++ b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLLocalVirtualObject.java @@ -32,17 +32,21 @@ import java.util.HashSet; import java.util.Set; +import org.mycore.common.config.MCRConfiguration2; import org.mycore.common.digest.MCRDigest; import org.mycore.common.events.MCREvent; import org.mycore.datamodel.niofs.MCRVersionedPath; import org.mycore.ocfl.niofs.storage.MCROCFLTempFileStorage; import org.mycore.ocfl.repository.MCROCFLRepository; +import io.ocfl.api.model.FileChangeHistory; import io.ocfl.api.model.ObjectVersionId; import io.ocfl.api.model.OcflObjectVersion; /** - * Represents a virtual object stored locally in an OCFL repository. + * Represents a virtual object that is stored on the same drive as the OCFL repository. This provides the implementation + * direct access to the files of the OCFL repository when needed. For example a file can be accessed directly for read + * operations without copying it first to the local storage. *

* This class extends {@link MCROCFLVirtualObject} and provides implementations specific to local storage. * It handles file operations such as copying, moving, and deleting files within the local file system, @@ -90,7 +94,8 @@ public MCROCFLLocalVirtualObject(MCROCFLRepository repository, OcflObjectVersion * @param directoryTracker the directory tracker. */ protected MCROCFLLocalVirtualObject(MCROCFLRepository repository, ObjectVersionId versionId, - OcflObjectVersion objectVersion, MCROCFLTempFileStorage localStorage, boolean readonly, + OcflObjectVersion objectVersion, MCROCFLTempFileStorage localStorage, + boolean readonly, MCROCFLFileTracker fileTracker, MCROCFLEmptyDirectoryTracker directoryTracker) { super(repository, versionId, objectVersion, localStorage, readonly, fileTracker, directoryTracker); @@ -100,7 +105,7 @@ protected MCROCFLLocalVirtualObject(MCROCFLRepository repository, ObjectVersionI * {@inheritDoc} */ @Override - public void copyFile(MCRVersionedPath source, MCRVersionedPath target, CopyOption... options) throws IOException { + public void copy(MCRVersionedPath source, MCRVersionedPath target, CopyOption... options) throws IOException { checkPurged(source); checkReadOnly(); boolean targetExists = exists(target); @@ -119,7 +124,7 @@ public void copyFile(MCRVersionedPath source, MCRVersionedPath target, CopyOptio * {@inheritDoc} */ @Override - public void copyFileToVirtualObject(MCROCFLVirtualObject virtualTarget, MCRVersionedPath source, + public void externalCopy(MCROCFLVirtualObject virtualTarget, MCRVersionedPath source, MCRVersionedPath target, CopyOption... options) throws IOException { checkPurged(source); virtualTarget.checkReadOnly(); @@ -172,7 +177,6 @@ protected SeekableByteChannel readOrWriteByteChannel(MCRVersionedPath path, Set< @Override public FileTime getModifiedTime(MCRVersionedPath path) throws IOException { - checkPurged(path); checkExists(path); Path physicalPath = toPhysicalPath(path); return Files.readAttributes(physicalPath, BasicFileAttributes.class).lastModifiedTime(); @@ -180,31 +184,33 @@ public FileTime getModifiedTime(MCRVersionedPath path) throws IOException { @Override public FileTime getAccessTime(MCRVersionedPath path) throws IOException { - checkPurged(path); checkExists(path); Path physicalPath = toPhysicalPath(path); return Files.readAttributes(physicalPath, BasicFileAttributes.class).lastAccessTime(); } - @Override - public long getSize(MCRVersionedPath path) throws IOException { - checkPurged(path); + /** + * {@inheritDoc} + */ + public Path toPhysicalPath(MCRVersionedPath path) throws IOException { checkExists(path); - if (isDirectory(path)) { - return 0; + if (this.localStorage.exists(path)) { + return this.localStorage.toPhysicalPath(path); } - Path physicalPath = toPhysicalPath(path); - return Files.size(physicalPath); + FileChangeHistory changeHistory = getChangeHistory(path); + String storageRelativePath = changeHistory.getMostRecent().getStorageRelativePath(); + return getLocalRepositoryPath().resolve(storageRelativePath); } - @Override - public Object getFileKey(MCRVersionedPath path) throws IOException { - checkPurged(path); - checkExists(path); - // TODO the fileKey between the localstorage and the ocfl repository should always be the same - // this implementation is just a hack for testing - Path physicalPath = toPhysicalPath(path); - return Files.readAttributes(physicalPath, BasicFileAttributes.class).fileKey(); + /** + * Returns the local OCFL repository path. + * + * @return the local repository path. + */ + protected Path getLocalRepositoryPath() { + return Path.of(MCRConfiguration2 + .getString("MCR.OCFL.Repository." + repository.getId() + ".RepositoryRoot") + .orElseThrow()); } /** diff --git a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLRemoteVirtualObject.java b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLRemoteVirtualObject.java index 6ce931f74b..6cee17f59b 100644 --- a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLRemoteVirtualObject.java +++ b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLRemoteVirtualObject.java @@ -118,6 +118,9 @@ protected SeekableByteChannel readOrWriteByteChannel(MCRVersionedPath path, Set< return seekableByteChannel; } + /** + * {@inheritDoc} + */ @Override public FileTime getModifiedTime(MCRVersionedPath path) throws IOException { checkExists(path); @@ -129,9 +132,13 @@ public FileTime getModifiedTime(MCRVersionedPath path) throws IOException { return FileTime.from(changeHistory.getMostRecent().getTimestamp().toInstant()); } + /** + * {@inheritDoc} + */ @Override public FileTime getAccessTime(MCRVersionedPath path) throws IOException { checkExists(path); + // TODO is the localStorage check required? Or does the localVirtualObject take care of that if (this.localStorage.exists(path)) { Path physicalPath = this.localStorage.toPhysicalPath(path); return Files.readAttributes(physicalPath, BasicFileAttributes.class).lastAccessTime(); @@ -140,37 +147,11 @@ public FileTime getAccessTime(MCRVersionedPath path) throws IOException { return FileTime.from(changeHistory.getMostRecent().getTimestamp().toInstant()); } - @Override - public long getSize(MCRVersionedPath path) throws IOException { - checkExists(path); - if (isDirectory(path)) { - return 0; - } - // TODO right now we just copy and deliver from local storage - // in future versions we should use the database to get the size - localCopy(path); - //if(this.localStorage.exists(path)) { - Path physicalPath = this.localStorage.toPhysicalPath(path); - return Files.size(physicalPath); - //} - } - - @Override - public Object getFileKey(MCRVersionedPath path) throws IOException { - checkExists(path); - // TODO rm this - localCopy(path); - // TODO the fileKey between the localstorage and the ocfl repository should always be the same - // this implementation is just a hack for testing - Path physicalPath = toPhysicalPath(path); - return Files.readAttributes(physicalPath, BasicFileAttributes.class).fileKey(); - } - /** * {@inheritDoc} */ @Override - public void copyFile(MCRVersionedPath source, MCRVersionedPath target, CopyOption... options) throws IOException { + public void copy(MCRVersionedPath source, MCRVersionedPath target, CopyOption... options) throws IOException { checkPurged(source); checkReadOnly(); boolean targetExists = exists(target); @@ -183,7 +164,7 @@ public void copyFile(MCRVersionedPath source, MCRVersionedPath target, CopyOptio * {@inheritDoc} */ @Override - public void copyFileToVirtualObject(MCROCFLVirtualObject virtualTarget, MCRVersionedPath source, + public void externalCopy(MCROCFLVirtualObject virtualTarget, MCRVersionedPath source, MCRVersionedPath target, CopyOption... options) throws IOException { checkPurged(source); virtualTarget.checkReadOnly(); @@ -202,7 +183,7 @@ public void copyFileToVirtualObject(MCROCFLVirtualObject virtualTarget, MCRVersi public Path toPhysicalPath(MCRVersionedPath path) throws IOException { checkExists(path); localCopy(path); - return super.toPhysicalPath(path); + return this.localStorage.toPhysicalPath(path); } /** diff --git a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLVirtualObject.java b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLVirtualObject.java index 04d29de20e..393236ac55 100644 --- a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLVirtualObject.java +++ b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/MCROCFLVirtualObject.java @@ -51,7 +51,6 @@ import java.util.stream.Stream; import org.mycore.common.MCRUtils; -import org.mycore.common.config.MCRConfiguration2; import org.mycore.common.digest.MCRDigest; import org.mycore.common.digest.MCRSHA512Digest; import org.mycore.common.events.MCREvent; @@ -63,14 +62,16 @@ import org.mycore.ocfl.niofs.storage.MCROCFLTempFileStorage; import org.mycore.ocfl.repository.MCROCFLRepository; +import io.ocfl.api.DigestAlgorithmRegistry; import io.ocfl.api.OcflObjectUpdater; import io.ocfl.api.OcflOption; import io.ocfl.api.io.FixityCheckInputStream; -import io.ocfl.api.model.DigestAlgorithm; +import io.ocfl.api.model.FileChange; import io.ocfl.api.model.FileChangeHistory; import io.ocfl.api.model.ObjectVersionId; import io.ocfl.api.model.OcflObjectVersion; import io.ocfl.api.model.OcflObjectVersionFile; +import io.ocfl.api.model.SizeDigestAlgorithm; import io.ocfl.api.model.VersionInfo; import io.ocfl.api.model.VersionNum; @@ -144,8 +145,8 @@ public MCROCFLVirtualObject(MCROCFLRepository repository, OcflObjectVersion obje * @param readonly whether the object is read-only. */ protected MCROCFLVirtualObject(MCROCFLRepository repository, ObjectVersionId versionId, - OcflObjectVersion objectVersion, - MCROCFLTempFileStorage localStorage, boolean readonly) { + OcflObjectVersion objectVersion, MCROCFLTempFileStorage localStorage, + boolean readonly) { this(repository, versionId, objectVersion, localStorage, readonly, null, null); initTrackers(); } @@ -162,8 +163,8 @@ protected MCROCFLVirtualObject(MCROCFLRepository repository, ObjectVersionId ver * @param directoryTracker the directory tracker. */ protected MCROCFLVirtualObject(MCROCFLRepository repository, ObjectVersionId versionId, - OcflObjectVersion objectVersion, MCROCFLTempFileStorage localStorage, boolean readonly, - MCROCFLFileTracker fileTracker, + OcflObjectVersion objectVersion, MCROCFLTempFileStorage localStorage, + boolean readonly, MCROCFLFileTracker fileTracker, MCROCFLEmptyDirectoryTracker directoryTracker) { Objects.requireNonNull(repository); Objects.requireNonNull(versionId); @@ -196,7 +197,7 @@ protected void initTrackers() { MCRVersionedPath filePath = toMCRPath(file.getPath()); boolean hasKeepFile = filePath.getFileName().toString().equals(KEEP_FILE); if (!hasKeepFile) { - String sha512Digest = file.getFixity().get(DigestAlgorithm.sha512); + String sha512Digest = file.getFixity().get(DigestAlgorithmRegistry.sha512); filePaths.put(filePath, new MCRSHA512Digest(sha512Digest)); } directoryPaths.put(filePath.getParent(), hasKeepFile); @@ -256,8 +257,10 @@ public boolean exists(MCRVersionedPath path) { * * @param path the path to check. * @return {@code true} if the path is stored locally, {@code false} otherwise. + * @throws NoSuchFileException if the path does not exist. */ - public boolean isLocal(MCRVersionedPath path) { + public boolean isLocal(MCRVersionedPath path) throws NoSuchFileException { + checkExists(path); return !this.markForPurge && this.localStorage.exists(path); } @@ -266,11 +269,25 @@ public boolean isLocal(MCRVersionedPath path) { * * @param path the path to check. * @return {@code true} if the path is a directory, {@code false} otherwise. + * @throws NoSuchFileException if the path does not exist. */ - public boolean isDirectory(MCRVersionedPath path) { + public boolean isDirectory(MCRVersionedPath path) throws NoSuchFileException { + checkExists(path); return !this.markForPurge && this.emptyDirectoryTracker.exists(path); } + /** + * Checks if the specified directory is empty. + * + * @param directoryPath the path to the directory. + * @return {@code true} if the directory is empty, {@code false} otherwise. + * @throws NoSuchFileException if the path does not exist. + */ + public boolean isDirectoryEmpty(MCRVersionedPath directoryPath) throws NoSuchFileException { + checkExists(directoryPath); + return this.emptyDirectoryTracker.isEmpty(directoryPath); + } + /** * Checks if the specified path is a file. * @@ -297,6 +314,7 @@ public SeekableByteChannel newByteChannel(MCRVersionedPath path, Set { - trackFileWrite(path, fireCreateEvent ? MCREvent.EventType.CREATE : MCREvent.EventType.UPDATE); + if (!isKeepFile) { + trackFileWrite(path, fireCreateEvent ? MCREvent.EventType.CREATE : MCREvent.EventType.UPDATE); + } else { + trackEmptyDirectory(path.getParent()); + } }); } return readOrWriteByteChannel(path, options, fileAttributes); @@ -324,19 +346,14 @@ protected abstract SeekableByteChannel readOrWriteByteChannel(MCRVersionedPath p FileAttribute... fileAttributes) throws IOException; /** - * Returns the local OCFL repository path. - * - * @return the local repository path. - */ - protected Path getLocalRepositoryPath() { - return Path.of(MCRConfiguration2 - .getString("MCR.OCFL.Repository." + repository.getId() + ".RepositoryRoot") - .orElseThrow()); - } - - /** + *

* Copies a file from the OCFL repository to the local storage. This should be called whenever it is necessary to * work on a copy rather than the OCFL file directly. + *

+ *

+ * If the requested path is a directory and it does not exist yet, an empty directory is created in the local + * storage. This guarantees that the given path is always accessible. + *

* * @param path the path to the file. * @throws IOException if an I/O error occurs. @@ -346,6 +363,7 @@ public void localCopy(MCRVersionedPath path) throws IOException { return; } if (!isFile(path)) { + this.localStorage.createDirectories(path); return; } OcflObjectVersionFile ocflFile = fromOcfl(path); @@ -402,13 +420,17 @@ public void delete(MCRVersionedPath path) throws IOException { /** * Copies a file from the source path to the target path. + *

This copy operation only works if both the source and + * the target are within the same {@link MCROCFLVirtualObject}. Use + * {@link #externalCopy(MCROCFLVirtualObject, MCRVersionedPath, MCRVersionedPath, CopyOption...)} if that is not + * the case.

* * @param source the source path. * @param target the target path. * @param options the options specifying how the copy is performed. * @throws IOException if an I/O error occurs. */ - public abstract void copyFile(MCRVersionedPath source, MCRVersionedPath target, CopyOption... options) + public abstract void copy(MCRVersionedPath source, MCRVersionedPath target, CopyOption... options) throws IOException; /** @@ -420,7 +442,7 @@ public abstract void copyFile(MCRVersionedPath source, MCRVersionedPath target, * @param options the options specifying how the copy is performed. * @throws IOException if an I/O error occurs. */ - public abstract void copyFileToVirtualObject(MCROCFLVirtualObject virtualTarget, MCRVersionedPath source, + public abstract void externalCopy(MCROCFLVirtualObject virtualTarget, MCRVersionedPath source, MCRVersionedPath target, CopyOption... options) throws IOException; /** @@ -545,7 +567,13 @@ public MCRDigest getDigest(MCRVersionedPath path) throws IOException { return this.fileTracker.getDigest(path); } - // TODO javadoc + /** + * Returns a file's creation time. + * + * @param path versioned path + * @return time of file creation + * @throws IOException if an I/O exception occurs + */ public FileTime getCreationTime(MCRVersionedPath path) throws IOException { checkExists(path); boolean added = this.isAdded(path); @@ -558,17 +586,77 @@ public FileTime getCreationTime(MCRVersionedPath path) throws IOException { return FileTime.from(changeHistory.getOldest().getTimestamp().toInstant()); } - // TODO javadoc + /** + * Returns a file's last modified time. + * + * @param path versioned path + * @return last modified time + * @throws IOException if an I/O exception occurs + */ public abstract FileTime getModifiedTime(MCRVersionedPath path) throws IOException; - // TODO javadoc + /** + * Returns a file's last access time. + * + * @param path versioned path + * @return last access time + * @throws IOException if an I/O exception occurs + */ public abstract FileTime getAccessTime(MCRVersionedPath path) throws IOException; - // TODO javadoc - public abstract long getSize(MCRVersionedPath path) throws IOException; + /** + * Returns the size of the file (in bytes). The size may differ from the actual size on the file system due to + * compression, support for sparse files, or other reasons. + * + * @param path the path + * @return size in bytes + * @throws IOException if an I/O exception occurs + */ + public long getSize(MCRVersionedPath path) throws IOException { + checkExists(path); + if (isDirectory(path)) { + return 0; + } + if (this.localStorage.exists(path)) { + Path physicalPath = this.localStorage.toPhysicalPath(path); + return Files.size(physicalPath); + } + OcflObjectVersionFile ocflObjectVersionFile = fromOcfl(path); + String sizeAsString = ocflObjectVersionFile.getFixity().get(new SizeDigestAlgorithm()); + return Long.parseLong(sizeAsString); + } - // TODO javadoc & implementation - public abstract Object getFileKey(MCRVersionedPath path) throws IOException; + /** + * Returns the `filekey` of the given path. + *

+ * Because OCFL is a version file system and the mycore implementation uses transactions this `filekey` + * implementation differs from a Unix like system. + *

+ *

+ * The Unix `filekey` is typically derived from the inode and device ID, ensuring uniqueness within the + * filesystem. When a file is modified (written, moved...), the Unix `filekey` remains unchanged as long as the + * inode remains the same. + *

+ *

+ * In contrast this implementation returns a new `filekey` as soon as a file is written or moved. The `filekey` + * then remains constant as long as the transaction is open. After the transaction is committed the `filekey` + * may change again. + *

+ *

+ * Implementation detail: Be aware that this method calls {@link #toPhysicalPath(MCRVersionedPath)}. For remote + * virtual objects this means that the whole file is copied to the local storage. Because the fileKey is not + * accessed frequently this should be acceptable. If this assumption proves to be wrong a special implementation + * for remote virtual objects is required! + *

+ * + * @param path versioned path + * @return fileKey + * @throws IOException if an I/O error occurs. + */ + public Object getFileKey(MCRVersionedPath path) throws IOException { + Path physicalPath = toPhysicalPath(path); + return Files.readAttributes(physicalPath, BasicFileAttributes.class).fileKey(); + } /** * Converts the specified versioned path to a local file system path. The path can either point at the local @@ -580,41 +668,59 @@ public FileTime getCreationTime(MCRVersionedPath path) throws IOException { * @return the physical path. * @throws IOException if an I/O error occurs. */ - public Path toPhysicalPath(MCRVersionedPath path) throws IOException { - checkExists(path); - if (this.localStorage.exists(path)) { - return this.localStorage.toPhysicalPath(path); - } - FileChangeHistory changeHistory = getChangeHistory(path); - String storageRelativePath = changeHistory.getMostRecent().getStorageRelativePath(); - return getLocalRepositoryPath().resolve(storageRelativePath); - } + public abstract Path toPhysicalPath(MCRVersionedPath path) throws IOException; - protected FileChangeHistory getChangeHistory(MCRVersionedPath path) { + /** + * Returns the change history of a path. + * + * @param path the path + * @return file change history + */ + protected FileChangeHistory getChangeHistory(MCRVersionedPath path) throws NoSuchFileException { boolean isDirectory = isDirectory(path); MCRVersionedPath resolvedPath = isDirectory ? path : this.fileTracker.findPath(path); - FileChangeHistory changeHistory = this.changeHistoryCache.get(resolvedPath); - if (changeHistory == null) { - changeHistory = isDirectory + FileChangeHistory fullChangeHistory = this.changeHistoryCache.get(resolvedPath); + if (fullChangeHistory == null) { + fullChangeHistory = isDirectory ? getRepository().directoryChangeHistory(getOwner(), resolvedPath.toRelativePath()) : getRepository().fileChangeHistory(getOwner(), resolvedPath.toRelativePath()); - this.changeHistoryCache.put(resolvedPath, changeHistory); + this.changeHistoryCache.put(resolvedPath, fullChangeHistory); + } + String requestedVersion = path.getVersion() == null ? getVersion() : path.getVersion(); + if (fullChangeHistory.getMostRecent().getVersionNum().toString().equals(requestedVersion)) { + return fullChangeHistory; + } + FileChangeHistory changeHistory = new FileChangeHistory(); + changeHistory.setPath(fullChangeHistory.getPath()); + changeHistory.setFileChanges(new ArrayList<>()); + Iterator forwardChangeIterator = fullChangeHistory.getForwardChangeIterator(); + while (forwardChangeIterator.hasNext()) { + FileChange change = forwardChangeIterator.next(); + changeHistory.getFileChanges().add(change); + if (change.getVersionNum().toString().equals(requestedVersion)) { + break; + } } return changeHistory; } /** - * Marks this virtual object for creation. + * Explicitly marks this virtual object for creation. A virtual object can only be marked for creation + * if it does not exist yet in the repository or if it was marked for purge (removes the purge status). + *

+ * Implementation detail: A virtual object instance can exist without having a ocfl repository version and + * without being created. + *

* * @throws MCRReadOnlyIOException if the object is read-only. + * @throws FileAlreadyExistsException if the object already exist */ - public void create() throws MCRReadOnlyIOException { + public void create() throws MCRReadOnlyIOException, FileAlreadyExistsException { if (this.readonly) { throw new MCRReadOnlyIOException("Cannot create read-only object: " + this); } - if (this.objectVersion != null && this.markForPurge) { - this.markForPurge = false; - return; + if (this.objectVersion != null && !this.markForPurge) { + throw new FileAlreadyExistsException("Cannot create already existing object: " + this); } this.markForPurge = false; this.markForCreate = true; @@ -652,29 +758,35 @@ public boolean isModified() { // TODO javadoc & junit public boolean isAdded(MCRVersionedPath path) { - if (this.readonly || this.markForPurge) { + if (this.readonly || this.markForPurge || !this.exists(path)) { return false; } - if (this.markForCreate) { - return true; + try { + return this.isDirectory(path) ? this.emptyDirectoryTracker.isAdded(path) : this.fileTracker.isAdded(path); + } catch (NoSuchFileException noSuchFileException) { + return false; } - return this.isDirectory(path) ? this.emptyDirectoryTracker.isAdded(path) : this.fileTracker.isAdded(path); } - // todo javadoc + /** + * Checks whether the path was newly added or modified (written) in any way. + *

The method will return false if the path does not exist anymore!

+ * + * @param path the versioned path to check + * @return true if the path was added or modified, otherwise false + * @throws IOException if an I/O exception occurred + */ public boolean isAddedOrModified(MCRVersionedPath path) throws IOException { - if (this.readonly) { + if (this.readonly || this.markForPurge || !this.exists(path)) { return false; } - if (this.markForPurge || this.markForCreate) { - return true; - } return this.isDirectory(path) ? isDirectoryModified(path) : isFileModified(path); } + @SuppressWarnings("PMD") protected boolean isFileModified(MCRVersionedPath file) throws IOException { if (!this.fileTracker.exists(file)) { - throw new NoSuchFileException(file.toString()); + return false; } return fileTracker.isAddedOrModified(file); } @@ -722,27 +834,17 @@ public boolean persist() throws IOException { repository.purgeObject(objectVersionId.getObjectId()); return true; } - // create object - if (this.markForCreate) { - String owner = getOwner(); - String version = getVersion(); - Path root = this.localStorage.toPhysicalPath(owner, version); - if (!Files.exists(root)) { - Files.createDirectories(root); - } - createKeepFiles(root); - repository.updateObject(objectVersionId, new VersionInfo().setMessage("create"), (updater) -> { - updater.addPath(root); - persistDirectoryChanges(updater); - }); - return true; - } - // update object + // persist + String type = this.objectVersion == null ? "create" : "update"; AtomicBoolean updatedFiles = new AtomicBoolean(false); AtomicBoolean updatedDirectories = new AtomicBoolean(false); - repository.updateObject(objectVersionId, new VersionInfo().setMessage("update"), (updater) -> { - updatedFiles.set(persistFileChanges(updater)); - updatedDirectories.set(persistDirectoryChanges(updater)); + repository.updateObject(objectVersionId, new VersionInfo().setMessage(type), (updater) -> { + try { + updatedFiles.set(persistFileChanges(updater)); + updatedDirectories.set(persistDirectoryChanges(updater)); + } catch (IOException ioException) { + throw new UncheckedIOException("Unable to " + type + " " + objectVersionId, ioException); + } }); return updatedFiles.get() || updatedDirectories.get(); } @@ -753,14 +855,17 @@ public boolean persist() throws IOException { * @param updater the OCFL object updater. * @return {@code true} if changes were persisted, {@code false} otherwise. */ - protected boolean persistFileChanges(OcflObjectUpdater updater) { + protected boolean persistFileChanges(OcflObjectUpdater updater) throws IOException { List> changes = this.fileTracker.changes(); for (MCROCFLFileTracker.Change change : changes) { String ocflSourcePath = change.source().toRelativePath(); switch (change.type()) { case ADDED_OR_MODIFIED -> { Path localSourcePath = this.localStorage.toPhysicalPath(change.source()); - updater.addPath(localSourcePath, ocflSourcePath, OcflOption.OVERWRITE); + long size = Files.size(localSourcePath); + updater + .addPath(localSourcePath, ocflSourcePath, OcflOption.OVERWRITE) + .addFileFixity(ocflSourcePath, new SizeDigestAlgorithm(), String.valueOf(size)); } case DELETED -> updater.removeFile(ocflSourcePath); case RENAMED -> { @@ -783,36 +888,19 @@ protected boolean persistDirectoryChanges(OcflObjectUpdater updater) { for (MCROCFLEmptyDirectoryTracker.Change change : changes) { String ocflKeepFile = change.keepFile().toRelativePath(); switch (change.type()) { - case ADD_KEEP -> updater.writeFile(InputStream.nullInputStream(), ocflKeepFile); - case REMOVE_KEEP -> updater.removeFile(ocflKeepFile); + case ADD_KEEP -> { + updater + .writeFile(InputStream.nullInputStream(), ocflKeepFile) + .addFileFixity(ocflKeepFile, new SizeDigestAlgorithm(), "0"); + } + case REMOVE_KEEP -> { + updater.removeFile(ocflKeepFile); + } } } return !changes.isEmpty(); } - /** - * Creates keep files for empty directories recursively. - * - * @param directory the target directory. - * @throws IOException if an I/O error occurs. - */ - protected void createKeepFiles(Path directory) throws IOException { - // keep for empty directory - try (Stream directoryStream = Files.list(directory)) { - if (directoryStream.findFirst().isEmpty()) { - Files.write(directory.resolve(KEEP_FILE), new byte[] {}); - return; - } - } - // run recursive through subdirectories - try (Stream directoryStream = Files.list(directory)) { - List subdirectories = directoryStream.filter(Files::isDirectory).toList(); - for (Path subdirectory : subdirectories) { - createKeepFiles(subdirectory); - } - } - } - /** * Checks if the specified path exists. * @@ -849,16 +937,6 @@ protected void checkReadOnly() throws MCRReadOnlyIOException { } } - /** - * Checks if the specified directory is empty. - * - * @param directoryPath the path to the directory. - * @return {@code true} if the directory is empty, {@code false} otherwise. - */ - protected boolean isDirectoryEmpty(MCRVersionedPath directoryPath) { - return this.emptyDirectoryTracker.isEmpty(directoryPath); - } - /** * Tracks a file write operation. * diff --git a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/storage/MCROCFLHybridStorage.java b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/storage/MCROCFLHybridStorage.java index 4e00f104fb..c8f960756c 100644 --- a/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/storage/MCROCFLHybridStorage.java +++ b/mycore-ocfl/src/main/java/org/mycore/ocfl/niofs/storage/MCROCFLHybridStorage.java @@ -43,9 +43,9 @@ */ public class MCROCFLHybridStorage implements MCROCFLTransactionalTempFileStorage { - private final static String TRANSACTION_DIRECTORY = "transaction"; + public final static String TRANSACTION_DIRECTORY = "transaction"; - private final static String ROLLING_DIRECTORY = "rolling"; + public final static String ROLLING_DIRECTORY = "rolling"; private MCROCFLDefaultTransactionalTempFileStorage transactionalStorage; @@ -56,7 +56,7 @@ public class MCROCFLHybridStorage implements MCROCFLTransactionalTempFileStorage @MCRProperty(name = "Path") public String rootPathProperty; - @MCRInstance(name = "EvictionStrategy", valueClass = MCROCFLEvictionStrategy.class, required = false) + @MCRInstance(name = "EvictionStrategy", valueClass = MCROCFLEvictionStrategy.class) public MCROCFLEvictionStrategy evictionStrategy; /** diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLBasicFileAttributesTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLBasicFileAttributesTest.java index 67674e1792..aa54a8eaf7 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLBasicFileAttributesTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLBasicFileAttributesTest.java @@ -7,7 +7,6 @@ import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; @@ -20,21 +19,14 @@ import org.mycore.datamodel.niofs.MCRFileAttributes; import org.mycore.datamodel.niofs.MCRPath; -import io.ocfl.api.model.ObjectVersionId; -import io.ocfl.api.model.VersionInfo; - public class MCROCFLBasicFileAttributesTest extends MCROCFLTestCase { + public MCROCFLBasicFileAttributesTest(boolean remote) { + super(remote); + } + @Test public void testDirectoryAttributes() throws IOException, InterruptedException { - // add some versions - repository.updateObject(ObjectVersionId.head(DERIVATE_1), new VersionInfo(), updater -> { - updater.writeFile(new ByteArrayInputStream(new byte[] { 1, 3, 3, 7 }), "file1"); - }); - repository.updateObject(ObjectVersionId.head(DERIVATE_1), new VersionInfo(), updater -> { - updater.writeFile(new ByteArrayInputStream(new byte[] { 1, 3, 3, 7 }), "file2"); - }); - MCRPath root = MCRPath.getPath(DERIVATE_1, "/"); MCRFileAttributes rootAttributes = Files.readAttributes(root, MCRFileAttributes.class); assertTrue("root should be a directory", rootAttributes.isDirectory()); @@ -42,7 +34,7 @@ public void testDirectoryAttributes() throws IOException, InterruptedException { Thread.sleep(1); MCRTransactionHelper.requireTransaction(MCROCFLFileSystemTransaction.class); - Files.write(MCRPath.getPath(DERIVATE_1, "file3"), new byte[] { 1, 3, 3, 7 }); + Files.write(MCRPath.getPath(DERIVATE_1, "file1"), new byte[] { 1, 3, 3, 7 }); MCRFileAttributes rootAttributes2 = Files.readAttributes(root, MCRFileAttributes.class); assertEquals("should have the same creation time after writing a file", rootAttributes.creationTime(), rootAttributes2.creationTime()); diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLEmptyDirectoryTrackerTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLEmptyDirectoryTrackerTest.java index 062f87f863..ec6988cba5 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLEmptyDirectoryTrackerTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLEmptyDirectoryTrackerTest.java @@ -14,6 +14,10 @@ public class MCROCFLEmptyDirectoryTrackerTest extends MCROCFLTestCase { private MCROCFLEmptyDirectoryTracker directoryTracker; + public MCROCFLEmptyDirectoryTrackerTest(boolean remote) { + super(remote); + } + @Before public void setUp() throws Exception { super.setUp(); diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileStoreTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileStoreTest.java index 4af2832b9d..783c11c0d5 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileStoreTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileStoreTest.java @@ -10,6 +10,10 @@ public class MCROCFLFileStoreTest extends MCROCFLTestCase { + public MCROCFLFileStoreTest(boolean remote) { + super(remote); + } + @Test public void name() { assertNotNull("file store should have a name", fs().name()); diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemProviderTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemProviderTest.java index ec371e8010..b1c27c5341 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemProviderTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemProviderTest.java @@ -12,6 +12,7 @@ import java.net.URL; import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; +import java.nio.file.DirectoryNotEmptyException; import java.nio.file.DirectoryStream; import java.nio.file.FileAlreadyExistsException; import java.nio.file.Files; @@ -35,6 +36,10 @@ public class MCROCFLFileSystemProviderTest extends MCROCFLTestCase { + public MCROCFLFileSystemProviderTest(boolean remote) { + super(remote); + } + @Test public void checkAccess() { // check files @@ -139,10 +144,17 @@ public void move() throws IOException { } @Test - public void copy() throws IOException, URISyntaxException { + public void copyFiles() throws IOException, URISyntaxException { + Path whiteV1 = MCRVersionedPath.head(DERIVATE_1, "white.png"); + Path copiedV2 = MCRVersionedPath.head(DERIVATE_1, "copied.png"); + + // copy non-existing file + MCRTransactionHelper.beginTransaction(); + assertThrows(NoSuchFileException.class, () -> Files.copy(copiedV2, whiteV1)); + MCRTransactionHelper.commitTransaction(); + + // copy white to copied MCRTransactionHelper.beginTransaction(); - Path whiteV1 = MCRVersionedPath.getPath(DERIVATE_1, "v1", "white.png"); - Path copiedV2 = MCRPath.getPath(DERIVATE_1, "copied.png"); Files.copy(whiteV1, copiedV2); assertTrue("'white.png' should exist", Files.exists(whiteV1)); assertTrue("'copied.png' should exist", Files.exists(copiedV2)); @@ -181,6 +193,26 @@ public void copy() throws IOException, URISyntaxException { assertTrue("'copied3.png' should exist after committing", Files.exists(ocflTarget)); } + @Test + public void copyDirectories() throws IOException { + Path emptyDirectory = MCRVersionedPath.head(DERIVATE_1, "empty"); + Path copiedEmptyDirectory = MCRVersionedPath.head(DERIVATE_1, "emptyCopy"); + + // copy empty directory + MCRTransactionHelper.beginTransaction(); + Files.copy(emptyDirectory, copiedEmptyDirectory); + assertTrue("'empty' directory should exist", Files.exists(emptyDirectory)); + assertTrue("'emptyCopy' directory should exist", Files.exists(copiedEmptyDirectory)); + MCRTransactionHelper.commitTransaction(); + + // copy to non-empty directory + MCRTransactionHelper.beginTransaction(); + Files.write(emptyDirectory.resolve("file"), new byte[] { 1, 3, 3, 7 }); + assertThrows(DirectoryNotEmptyException.class, + () -> Files.copy(copiedEmptyDirectory, emptyDirectory, StandardCopyOption.REPLACE_EXISTING)); + MCRTransactionHelper.commitTransaction(); + } + @Test public void delete() throws IOException { Path whitePng = MCRPath.getPath(DERIVATE_1, "white.png"); diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemTest.java index d367cc1bba..c2a3c262c1 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemTest.java @@ -18,6 +18,10 @@ public class MCROCFLFileSystemTest extends MCROCFLTestCase { + public MCROCFLFileSystemTest(boolean remote) { + super(remote); + } + @Test public void createRoot() throws Exception { MCROCFLFileSystem fs = MCROCFLFileSystemProvider.getMCROCFLFileSystem(); diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemTransactionTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemTransactionTest.java index 71dd924f5c..d7218e562f 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemTransactionTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileSystemTransactionTest.java @@ -16,6 +16,10 @@ public class MCROCFLFileSystemTransactionTest extends MCROCFLTestCase { + public MCROCFLFileSystemTransactionTest(boolean remote) { + super(remote); + } + @Test public void commit() throws IOException { MCRVersionedPath whitePng = MCRVersionedPath.head(DERIVATE_1, "white.png"); diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileTypeDetectorTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileTypeDetectorTest.java index f5cd5abbc2..821b752df7 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileTypeDetectorTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLFileTypeDetectorTest.java @@ -10,6 +10,10 @@ public class MCROCFLFileTypeDetectorTest extends MCROCFLTestCase { + public MCROCFLFileTypeDetectorTest(boolean remote) { + super(remote); + } + @Test public void probeContentType() throws IOException { String contentType = Files.probeContentType(MCRPath.getPath(DERIVATE_1, "white.png")); diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLTestCase.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLTestCase.java index 7da72ee0e2..fdcd158d07 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLTestCase.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLTestCase.java @@ -1,19 +1,27 @@ package org.mycore.ocfl.niofs; +import java.io.IOException; import java.net.URISyntaxException; import java.net.URL; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; import java.nio.file.Path; -import java.time.OffsetDateTime; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; import org.mycore.common.MCRTestCase; import org.mycore.common.MCRTransactionHelper; import org.mycore.common.config.MCRConfiguration2; +import org.mycore.datamodel.niofs.MCRVersionedPath; import org.mycore.ocfl.repository.MCROCFLRepository; import org.mycore.ocfl.repository.MCROCFLRepositoryProvider; -import io.ocfl.api.model.ObjectVersionId; -import io.ocfl.api.model.VersionInfo; - +@RunWith(Parameterized.class) public abstract class MCROCFLTestCase extends MCRTestCase { /** @@ -28,7 +36,23 @@ public abstract class MCROCFLTestCase extends MCRTestCase { protected MCROCFLRepository repository; - protected ObjectVersionId derivateVersionId; + private final boolean remote; + + @Parameterized.Parameters(name = "remote: {0}") + public static Iterable data() { + return Arrays.asList(new Object[][] { { false }, { true } }); + } + + public MCROCFLTestCase(boolean remote) { + this.remote = remote; + } + + @Override + protected Map getTestProperties() { + HashMap properties = new HashMap<>(); + properties.put("MCR.OCFL.Repository.Test.FS.Remote", remote ? "true" : "false"); + return properties; + } @Override public void setUp() throws Exception { @@ -40,7 +64,7 @@ public void setUp() throws Exception { MCROCFLFileSystemProvider.get().init(); - this.derivateVersionId = loadObject(DERIVATE_1); + loadObject(DERIVATE_1); } @Override @@ -54,18 +78,44 @@ public void tearDown() throws Exception { } } - protected ObjectVersionId loadObject(String id) throws URISyntaxException { + protected void loadObject(String id) throws URISyntaxException, IOException { URL derivateURL = getClass().getClassLoader().getResource(id); if (derivateURL == null) { throw new NullPointerException("Unable to locate '" + id + "' folder in resources."); } - return repository.putObject( - ObjectVersionId.head(id), - Path.of(derivateURL.toURI()), - new VersionInfo() - .setMessage("created") - .setCreated(OffsetDateTime.now()) - .setUser("junit", "")); + final Path sourcePath = Path.of(derivateURL.toURI()); + final MCRVersionedPath targetPath = MCRVersionedPath.head(id, "/"); + MCRTransactionHelper.beginTransaction(); + Files.walkFileTree(sourcePath, new CopyFileVisitor(targetPath)); + MCRTransactionHelper.commitTransaction(); + } + + private static class CopyFileVisitor extends SimpleFileVisitor { + + private final Path targetPath; + + private Path sourcePath = null; + + public CopyFileVisitor(Path targetPath) { + this.targetPath = targetPath; + } + + @Override + public FileVisitResult preVisitDirectory(final Path dir, final BasicFileAttributes attrs) throws IOException { + if (sourcePath == null) { + sourcePath = dir; + } else { + Files.createDirectories(targetPath.resolve(sourcePath.relativize(dir))); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(final Path file, final BasicFileAttributes attrs) throws IOException { + Files.copy(file, targetPath.resolve(sourcePath.relativize(file))); + return FileVisitResult.CONTINUE; + } + } } diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLVirtualObjectProviderTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLVirtualObjectProviderTest.java index 5c6a71f17e..caed625a3f 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLVirtualObjectProviderTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCROCFLVirtualObjectProviderTest.java @@ -12,6 +12,10 @@ public class MCROCFLVirtualObjectProviderTest extends MCROCFLTestCase { + public MCROCFLVirtualObjectProviderTest(boolean remote) { + super(remote); + } + @Test public void exists() throws FileSystemException { MCROCFLVirtualObjectProvider virtualObjectProvider = MCROCFLFileSystemProvider.get().virtualObjectProvider(); diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCRVirtualObjectTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCRVirtualObjectTest.java index 82f7d30090..531b6f4fbf 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCRVirtualObjectTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/MCRVirtualObjectTest.java @@ -14,6 +14,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributes; import java.util.HashSet; import java.util.Set; @@ -26,6 +27,10 @@ public class MCRVirtualObjectTest extends MCROCFLTestCase { + public MCRVirtualObjectTest(boolean remote) { + super(remote); + } + @Test public void toPhysicalPath() throws IOException { MCRVersionedPath source = MCRVersionedPath.head(DERIVATE_1, "white.png"); @@ -227,7 +232,7 @@ public void isFileAddedOrModified() throws IOException { MCRTransactionHelper.beginTransaction(); Files.move(whitePng, movedPng); assertTrue("Renamed file should be marked as modified", getVirtualObject().isAddedOrModified(movedPng)); - assertThrows(NoSuchFileException.class, () -> getVirtualObject().isAddedOrModified(whitePng)); + assertFalse(getVirtualObject().isAddedOrModified(whitePng)); MCRTransactionHelper.commitTransaction(); // Add a new file (recreate white.png) @@ -246,7 +251,7 @@ public void isFileAddedOrModified() throws IOException { // Delete the file MCRTransactionHelper.beginTransaction(); Files.delete(whitePng); - assertThrows(NoSuchFileException.class, () -> getVirtualObject().isAddedOrModified(whitePng)); + assertFalse(getVirtualObject().isAddedOrModified(whitePng)); MCRTransactionHelper.commitTransaction(); // Check no modification scenario (recreate white.png with same initial content) @@ -282,7 +287,7 @@ public void isDirectoryAddedOrModified() throws IOException { MCRTransactionHelper.beginTransaction(); Files.move(testDir, movedDir); assertTrue("movedDir should be marked as modified", getVirtualObject().isAddedOrModified(movedDir)); - assertThrows(NoSuchFileException.class, () -> getVirtualObject().isAddedOrModified(testDir)); + assertFalse(getVirtualObject().isAddedOrModified(testDir)); assertTrue("Renaming directory should mark root as modified", getVirtualObject().isAddedOrModified(rootDir)); MCRTransactionHelper.commitTransaction(); @@ -312,11 +317,50 @@ public void isDirectoryAddedOrModified() throws IOException { // Delete the directory MCRTransactionHelper.beginTransaction(); Files.delete(movedDir); - assertThrows(NoSuchFileException.class, () -> getVirtualObject().isAddedOrModified(movedDir)); + assertFalse(getVirtualObject().isAddedOrModified(movedDir)); assertTrue("Deleting directory should mark root as modified", getVirtualObject().isAddedOrModified(rootDir)); MCRTransactionHelper.commitTransaction(); } + @Test + public void getSize() throws IOException { + MCRVersionedPath headWhitePng = MCRVersionedPath.head(DERIVATE_1, "white.png"); + assertEquals("original white.png should have 554 bytes", 554, Files.size(headWhitePng)); + + // write 4 bytes + MCRTransactionHelper.beginTransaction(); + Files.write(headWhitePng, new byte[] { 5, 6, 7, 8 }); + assertEquals("written white.png should have 4 bytes", 4, Files.size(headWhitePng)); + MCRTransactionHelper.commitTransaction(); + + // check v1 + MCRVersionedPath v1WhitePng = MCRVersionedPath.getPath(DERIVATE_1, "v1", "white.png"); + assertEquals("v1 white.png should have 554 bytes", 554, Files.size(v1WhitePng)); + + // check v2 + MCRVersionedPath v2WhitePng = MCRVersionedPath.getPath(DERIVATE_1, "v2", "white.png"); + assertEquals("v2 white.png should have 4 bytes", 4, Files.size(v2WhitePng)); + } + + @Test + public void getFileKey() throws IOException { + MCRVersionedPath v1WhitePng = MCRVersionedPath.head(DERIVATE_1, "white.png"); + MCRVersionedPath v1BlackPng = MCRVersionedPath.head(DERIVATE_1, "black.png"); + MCRVersionedPath notFoundPng = MCRVersionedPath.head(DERIVATE_1, "notFound.png"); + + Object v1WhitePngFileKey = getFileKey(v1WhitePng); + Object v1BlackPngFileKey = getFileKey(v1BlackPng); + + assertNotNull("fileKey of original white.png should not be null", v1WhitePngFileKey); + assertNotNull("fileKey of original black.png should not be null", v1BlackPngFileKey); + assertThrows("fileKey of notFound.png should not exist yet", NoSuchFileException.class, + () -> getFileKey(notFoundPng)); + } + + private static Object getFileKey(MCRVersionedPath path) throws IOException { + return Files.readAttributes(path, BasicFileAttributes.class).fileKey(); + } + private static MCROCFLVirtualObject getVirtualObject() { return MCROCFLFileSystemProvider.get().virtualObjectProvider().get(DERIVATE_1); } diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLDefaultTransactionalTempFileStorageTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLDefaultTransactionalTempFileStorageTest.java index a359dee4e7..da49232d49 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLDefaultTransactionalTempFileStorageTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLDefaultTransactionalTempFileStorageTest.java @@ -14,6 +14,10 @@ public class MCROCFLDefaultTransactionalTempFileStorageTest extends MCROCFLStora private MCROCFLDefaultTransactionalTempFileStorage storage; + public MCROCFLDefaultTransactionalTempFileStorageTest(boolean remote) { + super(remote); + } + @Override public void setUp() throws Exception { super.setUp(); @@ -37,10 +41,10 @@ public void exists() throws IOException { @Test public void toPhysicalPath() { MCRTransactionHelper.beginTransaction(); - assertTrue(storage.toPhysicalPath(path1).endsWith("1/owner1/v0/file1")); + assertTrue(storage.toPhysicalPath(path1).endsWith("2/owner1/v0/file1")); MCRTransactionHelper.commitTransaction(); MCRTransactionHelper.beginTransaction(); - assertTrue(storage.toPhysicalPath(path1).endsWith("2/owner1/v0/file1")); + assertTrue(storage.toPhysicalPath(path1).endsWith("3/owner1/v0/file1")); MCRTransactionHelper.commitTransaction(); } diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLHybridStorageTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLHybridStorageTest.java index 94460f799e..cfb52392cd 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLHybridStorageTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLHybridStorageTest.java @@ -15,6 +15,10 @@ public class MCROCFLHybridStorageTest extends MCROCFLStorageTestCase { private MCROCFLHybridStorage storage; + public MCROCFLHybridStorageTest(boolean remote) { + super(remote); + } + @Override public void setUp() throws Exception { super.setUp(); @@ -39,15 +43,17 @@ public void exists() throws IOException { @Test public void toPhysicalPath() { - assertTrue("should access rolling store", storage.toPhysicalPath(path1).endsWith("rolling/owner1/v0/file1")); + String rollingDirectory = "/" + MCROCFLHybridStorage.ROLLING_DIRECTORY + "/"; + String transactionDirectory = "/" + MCROCFLHybridStorage.TRANSACTION_DIRECTORY + "/"; + assertTrue("should access rolling store", storage.toPhysicalPath(path1).toString().contains(rollingDirectory)); MCRTransactionHelper.beginTransaction(); assertTrue("should access transactional store", - storage.toPhysicalPath(path1).endsWith("transaction/1/owner1/v0/file1")); + storage.toPhysicalPath(path1).toString().contains(transactionDirectory)); MCRTransactionHelper.commitTransaction(); - assertTrue("should access rolling store", storage.toPhysicalPath(path1).endsWith("rolling/owner1/v0/file1")); + assertTrue("should access rolling store", storage.toPhysicalPath(path1).toString().contains(rollingDirectory)); MCRTransactionHelper.beginTransaction(); assertTrue("should access transactional store", - storage.toPhysicalPath(path1).endsWith("transaction/2/owner1/v0/file1")); + storage.toPhysicalPath(path1).toString().contains(transactionDirectory)); MCRTransactionHelper.commitTransaction(); } diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLRollingCacheStorageTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLRollingCacheStorageTest.java index 486aabb5d1..c937993704 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLRollingCacheStorageTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLRollingCacheStorageTest.java @@ -12,6 +12,10 @@ public class MCROCFLRollingCacheStorageTest extends MCROCFLStorageTestCase { private MCROCFLRollingCacheStorage storage; + public MCROCFLRollingCacheStorageTest(boolean remote) { + super(remote); + } + @Override public void setUp() throws Exception { super.setUp(); diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLStorageTestCase.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLStorageTestCase.java index cafba99934..367d24edd3 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLStorageTestCase.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/niofs/storage/MCROCFLStorageTestCase.java @@ -27,6 +27,10 @@ public abstract class MCROCFLStorageTestCase extends MCROCFLTestCase { protected MCRVersionedPath path3; + public MCROCFLStorageTestCase(boolean remote) { + super(remote); + } + @Override public void setUp() throws Exception { super.setUp(); diff --git a/mycore-ocfl/src/test/java/org/mycore/ocfl/repository/MCROCFLRepositoryTest.java b/mycore-ocfl/src/test/java/org/mycore/ocfl/repository/MCROCFLRepositoryTest.java index a74dccddb7..02b5f5a986 100644 --- a/mycore-ocfl/src/test/java/org/mycore/ocfl/repository/MCROCFLRepositoryTest.java +++ b/mycore-ocfl/src/test/java/org/mycore/ocfl/repository/MCROCFLRepositoryTest.java @@ -10,6 +10,7 @@ import org.mycore.ocfl.niofs.MCROCFLTestCase; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.net.URISyntaxException; import static org.junit.Assert.assertEquals; @@ -18,8 +19,12 @@ public class MCROCFLRepositoryTest extends MCROCFLTestCase { + public MCROCFLRepositoryTest(boolean remote) { + super(remote); + } + @Test - public void directoryChangeHistory() throws URISyntaxException { + public void directoryChangeHistory() throws URISyntaxException, IOException { assertThrows(NotFoundException.class, () -> { repository.directoryChangeHistory(DERIVATE_2, "/"); }); diff --git a/mycore-ocfl/src/test/resources/mycore.properties b/mycore-ocfl/src/test/resources/mycore.properties index 3ddc22e3e6..a07e7082cf 100644 --- a/mycore-ocfl/src/test/resources/mycore.properties +++ b/mycore-ocfl/src/test/resources/mycore.properties @@ -9,8 +9,8 @@ MCR.OCFL.Repository.Test.WorkDir=%MCR.basedir%/ocfl-temp # MCR.OCFL.Repository.Test.FS.TempStorage=org.mycore.ocfl.niofs.storage.MCROCFLHybridStorage # MCR.OCFL.Repository.Test.FS.TempStorage.EvictionStrategy=org.mycore.ocfl.niofs.storage.MCROCFLNeverEvictStrategy -MCR.OCFL.Repository.Test.FS.Remote=false -MCR.OCFL.Repository.Test.FS.TempStorage=org.mycore.ocfl.niofs.storage.MCROCFLDefaultTransactionalTempFileStorage +MCR.OCFL.Repository.Test.FS.TempStorage=org.mycore.ocfl.niofs.storage.MCROCFLHybridStorage +MCR.OCFL.Repository.Test.FS.TempStorage.EvictionStrategy=org.mycore.ocfl.niofs.storage.MCROCFLNeverEvictStrategy MCR.OCFL.Repository.Test.FS.TempStorage.Path=%MCR.basedir%/ocfl-temp-storage diff --git a/pom.xml b/pom.xml index 079591866e..8323c47d7f 100644 --- a/pom.xml +++ b/pom.xml @@ -1954,7 +1954,7 @@ Applications based on MyCoRe use a common core, which provides the functionality org.apache.httpcomponents httpclient