Merge pull request #845 from DataVault/DS-LUC070-147-20241129

DS-LUC070-147-20241129
DataVault · Dec 11, 2024 · 57e4576 · 57e4576
2 parents 592ebad + 909a246
commit 57e4576
Show file tree

Hide file tree

Showing 9 changed files with 65 additions and 38 deletions.
diff --git a/datavault-broker/src/main/java/org/datavaultplatform/broker/queue/EventListener.java b/datavault-broker/src/main/java/org/datavaultplatform/broker/queue/EventListener.java
@@ -5,12 +5,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.nio.charset.StandardCharsets;
 import java.time.Clock;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeSet;
+import java.util.*;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 
@@ -747,6 +742,10 @@ void process13DeleteStart(DeleteStart event, Deposit deposit) {
   void process14DeleteComplete(DeleteComplete event, Deposit deposit) {
     long depositSizeBeforeDelete = deposit.getSize();
     deposit.setStatus(Deposit.Status.DELETED);
+    deposit.setDepositPaths(Collections.emptyList());
+    depositsService.deleteDepositPaths(deposit.getID());
+    deposit.setDepositChunks(Collections.emptyList());
+    depositsService.deleteDepositChunks(deposit.getID());
     deposit.setSize(0);
     depositsService.updateDeposit(deposit);
 

diff --git a/datavault-broker/src/main/java/org/datavaultplatform/broker/services/DepositsService.java b/datavault-broker/src/main/java/org/datavaultplatform/broker/services/DepositsService.java
@@ -4,10 +4,7 @@
 
 import org.datavaultplatform.common.event.Event;
 import org.datavaultplatform.common.model.*;
-import org.datavaultplatform.common.model.dao.AuditChunkStatusDAO;
-import org.datavaultplatform.common.model.dao.DepositChunkDAO;
-import org.datavaultplatform.common.model.dao.DepositDAO;
-import org.datavaultplatform.common.model.dao.EventDAO;
+import org.datavaultplatform.common.model.dao.*;
 import org.datavaultplatform.common.util.RetrievedChunks;
 import org.datavaultplatform.common.util.StoredChunks;
 import org.slf4j.Logger;
@@ -33,6 +30,7 @@ public class DepositsService {
 
     private final DepositDAO depositDAO;
     private final DepositChunkDAO depositChunkDAO;
+    private final DepositPathDAO depositPathDAO;
     private final AuditChunkStatusDAO auditChunkStatusDAO;
     private final EventDAO eventDAO;
 
@@ -52,6 +50,7 @@ public class DepositsService {
     public DepositsService(
         DepositDAO depositDAO,
         DepositChunkDAO depositChunkDAO,
+        DepositPathDAO depositPathDAO,
         AuditChunkStatusDAO auditChunkStatusDAO,
         EventDAO eventDAO,
         @Value("${audit.period.minutes:0}") int auditPeriodMinutes,
@@ -63,6 +62,7 @@ public DepositsService(
         @Value("${audit.maxTotalChunks:2000}") int auditMaxTotalChunks){
         this.depositDAO = depositDAO;
         this.depositChunkDAO = depositChunkDAO;
+        this.depositPathDAO = depositPathDAO;
         this.auditChunkStatusDAO = auditChunkStatusDAO;
         this.eventDAO = eventDAO;
         this.auditPeriodMinutes = auditPeriodMinutes;
@@ -313,5 +313,13 @@ public String getDepositArchive(String depositId, ArchiveStore archiveStore) thr
         }
         return archiveID;
     }
+
+    public void deleteDepositChunks(String depositId) {
+        depositChunkDAO.deleteAllByDepositId(depositId);
+    }
+
+    public void deleteDepositPaths(String depositId) {
+        depositPathDAO.deleteAllByDepositId(depositId);
+    }
 }
 
diff --git a/...roker/src/test/java/org/datavaultplatform/broker/controllers/BaseGenerateMessageTest.java b/...roker/src/test/java/org/datavaultplatform/broker/controllers/BaseGenerateMessageTest.java
@@ -11,10 +11,7 @@
 import org.datavaultplatform.broker.queue.Sender;
 import org.datavaultplatform.broker.services.*;
 import org.datavaultplatform.common.model.ArchiveStore;
-import org.datavaultplatform.common.model.dao.AuditChunkStatusDAO;
-import org.datavaultplatform.common.model.dao.DepositChunkDAO;
-import org.datavaultplatform.common.model.dao.DepositDAO;
-import org.datavaultplatform.common.model.dao.EventDAO;
+import org.datavaultplatform.common.model.dao.*;
 import org.datavaultplatform.common.storage.impl.LocalFileSystem;
 import org.datavaultplatform.common.util.StorageClassNameResolver;
 import org.junit.jupiter.api.AfterEach;
@@ -53,6 +50,8 @@ public abstract class BaseGenerateMessageTest {
   @Mock
   DepositChunkDAO depositChunkDao;
   @Mock
+  DepositPathDAO depositPathDao;
+  @Mock
   AuditChunkStatusDAO auditChunkStatusDAO;
   @Mock
   RetrievesService retrievesService;
@@ -98,7 +97,7 @@ void setup() {
   }
 
   protected DepositsController getDepositController() {
-    DepositsService depositsService = new DepositsService(depositDao, depositChunkDao,
+    DepositsService depositsService = new DepositsService(depositDao, depositChunkDao, depositPathDao,
         auditChunkStatusDAO, eventDAO,0, 0, 0, 0, 0, 0, 0);
     DepositsController dc = new DepositsController(vaultsService,
         depositsService,

diff --git a/...vault-broker/src/test/java/org/datavaultplatform/broker/services/DepositsServiceTest.java b/...vault-broker/src/test/java/org/datavaultplatform/broker/services/DepositsServiceTest.java
@@ -1,10 +1,7 @@
 package org.datavaultplatform.broker.services;
 
 import org.datavaultplatform.common.event.Event;
-import org.datavaultplatform.common.model.dao.AuditChunkStatusDAO;
-import org.datavaultplatform.common.model.dao.DepositChunkDAO;
-import org.datavaultplatform.common.model.dao.DepositDAO;
-import org.datavaultplatform.common.model.dao.EventDAO;
+import org.datavaultplatform.common.model.dao.*;
 import org.datavaultplatform.common.util.RetrievedChunks;
 import org.datavaultplatform.common.util.StoredChunks;
 import org.junit.jupiter.api.BeforeEach;
@@ -31,6 +28,10 @@ class DepositsServiceTest {
     EventDAO mEventDAO;
     @Mock
     DepositChunkDAO mDepositChunkDAO;
+
+    @Mock
+    DepositPathDAO mDepositPathDAO;
+
     @Mock
     AuditChunkStatusDAO mAuditChunkStatusDAO;
 
@@ -49,6 +50,7 @@ void setup() {
         depositsService = new DepositsService(
                 mDepositDAO,
                 mDepositChunkDAO,
+                mDepositPathDAO,
                 mAuditChunkStatusDAO,
                 mEventDAO,
                 auditPeriodMinutes,

diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/model/dao/DepositChunkDAO.java b/datavault-common/src/main/java/org/datavaultplatform/common/model/dao/DepositChunkDAO.java
@@ -18,4 +18,7 @@ public interface DepositChunkDAO extends BaseDAO<DepositChunk>, DepositChunkCust
   @Override
   @EntityGraph(DepositChunk.EG_DEPOSIT_CHUNK)
   List<DepositChunk> findAll();
+
+  @EntityGraph(DepositChunk.EG_DEPOSIT_CHUNK)
+  void deleteAllByDepositId(String string);
 }
diff --git a/datavault-common/src/main/java/org/datavaultplatform/common/model/dao/DepositPathDAO.java b/datavault-common/src/main/java/org/datavaultplatform/common/model/dao/DepositPathDAO.java
@@ -1,11 +1,15 @@
 package org.datavaultplatform.common.model.dao;
 
 import org.datavaultplatform.common.model.DepositPath;
+import org.springframework.data.jpa.repository.EntityGraph;
 import org.springframework.stereotype.Repository;
 import org.springframework.transaction.annotation.Transactional;
 
 @Repository
 @Transactional
 public interface DepositPathDAO extends BaseDAO<DepositPath> {
 
+    @EntityGraph(DepositPath.EG_DEPOSIT_PATH)
+    void deleteAllByDepositId(String depositId);
+
 }
diff --git a/...-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java b/...-common/src/main/java/org/datavaultplatform/common/storage/impl/TivoliStorageManager.java
@@ -208,16 +208,26 @@ public Verify.Method getVerifyMethod() {
     public void delete(String depositId, File working, Progress progress, String optFilePath) throws Exception {
 		Path depositDirectoryPath = getDepositDirectoryPath(depositId);
 		Path tsmFilePath = depositDirectoryPath.resolve(working.getName());
-		log.info("TSM Delete [{}] skipping",tsmFilePath);
-		//ProcessHelper.ProcessInfo info = getProcessInfo("tsmDelete",
-		//		"dsmc", "delete", "archive", tsmFilePath.toString(), "-noprompt", "-optfile=" + optFilePath);
-		//if (info.wasFailure()) {
-		//	String errMessage = String.format("Delete of [%s] failed.", tsmFilePath);
-		//	logProcessOutput(info, errMessage);
-		//	throw new Exception(errMessage);
-		//} else {
-		//	log.info("Delete of [{}] was Successful.", tsmFilePath);
-		//}
+		log.info("TSM Delete [{}] ",tsmFilePath);
+
+		boolean deleted = false;
+		for (int r = 0; r < maxRetries && !deleted; r++) {
+			ProcessHelper.ProcessInfo info = getProcessInfo("tsmDelete",
+					"dsmc", "delete", "archive", tsmFilePath.toString(), "-noprompt", "-optfile=" + optFilePath);
+			String attemptCtx = String.format("attempt[%s/%s]", r+1, maxRetries);
+			if (info.wasFailure()) {
+				boolean lastAttempt = r == (maxRetries -1);
+				String errMessage = String.format("Delete of [%s] failed using location[%s] %s", tsmFilePath, optFilePath, attemptCtx);
+				logProcessOutput(info, errMessage);
+				if (lastAttempt) {
+					throw new Exception(errMessage);
+				}
+				log.info("{} Retrying in {} mins", errMessage, retryTimeSeconds);
+				TimeUnit.SECONDS.sleep(retryTimeSeconds);
+			} else {
+				log.info("Delete of [{}] was Successful.", tsmFilePath);
+			}
+		}
     }
 
 	/*

diff --git a/...mon/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java b/...mon/src/test/java/org/datavaultplatform/common/storage/impl/TivoliStorageManagerTest.java
@@ -306,12 +306,12 @@ void setup() throws IOException {
             Map<String, String> props = new HashMap<>();
             props.put(PropNames.TEMP_DIR, tsmTemp.toString());
             props.put(PropNames.OPTIONS_DIR, "/tmp/opt");
+            props.put(PropNames.TSM_MAX_RETRIES, "5");
+            props.put(PropNames.TSM_RETRY_TIME, "1");
             tsm = Mockito.spy(new TivoliStorageManager("testTSM", props));
         }
 
-        // Removed as TSM deletes can never succeed for TSM since the move to write once tapes
-        // resulted in us not deleting them
-        /*@Test
+        @Test
         void testDeleteSucceeds() throws Exception {
 
             File fileToDelete = Files.createTempFile("test",".txt").toFile();
@@ -333,11 +333,9 @@ void testDeleteSucceeds() throws Exception {
 
             //Check that the local file has not been deleted. We are trying to delete file on TSM ONLY
             assertThat(fileToDelete).exists();
-        }*/
+        }
 
-        // Removed as TSM deletes can never fail for TSM since the move to write once tapes
-        // resulted in us not deleting them
-        /*@Test
+        @Test
         void testDeleteFails() throws Exception {
 
             File fileToDelete = Files.createTempFile("test",".txt").toFile();
@@ -356,7 +354,7 @@ void testDeleteFails() throws Exception {
                 tsm.delete("testDepositId", fileToDelete, progress, "specificLocation");
             });
             String expectedTsmFile = tsmTemp.resolve("testDepositId").resolve(fileToDelete.getName()).toString();
-            String expectedErrorMessage = String.format("Delete of [%s] failed.",expectedTsmFile);
+            String expectedErrorMessage = String.format("Delete of [%s] failed using location[specificLocation] attempt[5/5]",expectedTsmFile);
             assertThat(ex).hasMessage(expectedErrorMessage);
 
             assertThat(argDesc.getValue()).isEqualTo("tsmDelete");
@@ -365,7 +363,7 @@ void testDeleteFails() throws Exception {
 
             //Check that the local file has not been deleted. We are trying to delete file on TSM ONLY
             assertThat(fileToDelete).exists();
-        }*/
+        }
 
     }
 

diff --git a/datavault-webapp/src/main/webapp/WEB-INF/templates/deposits/deposit.html b/datavault-webapp/src/main/webapp/WEB-INF/templates/deposits/deposit.html
@@ -109,6 +109,10 @@ <h2>View the Deposit: <small>[[${deposit.name}]]</small></h2>
                                     <span class="glyphicon glyphicon-ok" aria-hidden="true"></span>
                                     Complete
                                 </div>
+                                <div th:case="DELETED" class="text-success">
+                                    <span class="glyphicon glyphicon-ok" aria-hidden="true"></span>
+                                    Deleted
+                                </div>
                                 <div th:case="FAILED" class="text-danger">
                                     <span class="glyphicon glyphicon-remove" aria-hidden="true"></span>
                                     Failed