Fixed clean module (#280)

* services.json modified in order to properly delete fitrimmed fastq les in assembly annotation * self.service_samples modified in order to properly get list of service samples * assembly files to delete modified in services.json * services.json updated with mtbseq service info * Modified purge_files function for avoiding repeated files in files_to_delete list * Modified purge_files function for avoiding repeated files in files_to_delete list * Relocated self.rename() method usage for _DEL renaming of purged folders. Now it is called when self.option = rename_nocopy * delete_rename() function renamed to just delete() * self.option rename_nocopy renamed to rename * Reverted lasts commits * Reverted some weird changes... * Relocated self.rename() method usage for _DEL renaming of purged folders * Renamed self.delete_rename() method. Now is self.delete() * rename_nocopy option modified in just rename * files to delete added to services.json for rnaseq service * Updated CHANGELOG * fixed linting * Fixed linting
BU-ISCIII · Jun 7, 2024 · f67600e · f67600e
1 parent ba47621
commit f67600e
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 24 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -56,6 +56,7 @@ Code contributions to the new version:
 #### Fixes
 
 - Fixed archive module. Updated correct header for scout tsv [#258](https://github.com/BU-ISCIII/buisciii-tools/pull/258).
+- Fixed clean module. Corrected purge_files function. Renaming stage moved from clean to rename_nocopy option. Updated services.json file with correct paths for some services. [#280](https://github.com/BU-ISCIII/buisciii-tools/pull/280)
 
 #### Changed
 

diff --git a/bu_isciii/__main__.py b/bu_isciii/__main__.py
@@ -307,7 +307,7 @@ def scratch(ctx, resolution, path, tmp_dir, direction, ask_path):
     type=click.Choice(
         [
             "full_clean",
-            "rename_nocopy",
+            "rename",
             "clean",
             "revert_renaming",
             "show_removable",
@@ -317,7 +317,7 @@ def scratch(ctx, resolution, path, tmp_dir, direction, ask_path):
     multiple=False,
     help=(
         "Select what to do inside the cleanning step: full_clean: delete files and folders to clean,"
-        " rename no copy and deleted folders, rename_nocopy: just rename no copy folders, clean: "
+        " rename no copy and deleted folders, rename: just rename folders, clean: "
         "delete files and folders to clean,"
         "revert_renaming: remove no_copy and delete tags,"
         "show_removable: list folders and files to remove "
@@ -447,7 +447,7 @@ def finish(ctx, resolution, path, ask_path, sftp_folder, tmp_dir):
         resolution,
         path,
         ask_path,
-        "rename_nocopy",
+        "rename",
         ctx.obj["api_user"],
         ctx.obj["api_password"],
         ctx.obj["conf"],

diff --git a/bu_isciii/clean.py b/bu_isciii/clean.py
@@ -55,7 +55,9 @@ def __init__(
         self.services_requested = self.resolution_info["resolutions"][0][
             "available_services"
         ]
-        self.service_samples = self.resolution_info["samples"]
+        self.service_samples = [
+            sample_id["sample_name"] for sample_id in self.resolution_info["samples"]
+        ]
 
         if ask_path and path is None:
             stderr.print(
@@ -94,14 +96,13 @@ def __init__(
         self.delete_files = self.get_clean_items(self.services_to_clean, type="files")
         # self.delete_list = [item for item in self.delete_list if item]
         self.nocopy = self.get_clean_items(self.services_to_clean, type="no_copy")
-        self.service_samples = self.resolution_info.get("Samples", None)
 
         if option is None:
             self.option = bu_isciii.utils.prompt_selection(
                 "Options",
                 [
                     "full_clean",
-                    "rename_nocopy",
+                    "rename",
                     "clean",
                     "revert_renaming",
                     "show_removable",
@@ -312,10 +313,9 @@ def purge_files(self):
             files_to_delete = []
             for sample_info in self.service_samples:
                 for file in self.delete_files:
-                    file_to_delete = file.replace(
-                        "sample_name", sample_info["sample_name"]
-                    )
-                    files_to_delete.append(file_to_delete)
+                    file_to_delete = file.replace("sample_name", sample_info)
+                    if file_to_delete not in files_to_delete:
+                        files_to_delete.append(file_to_delete)
             path_content = self.scan_dirs(to_find=files_to_delete)
             for file in path_content:
                 os.remove(file)
@@ -371,7 +371,7 @@ def delete_work(self):
         else:
             stderr.print("There is no work folder here")
 
-    def delete_rename(self, verbose=True, sacredtexts=["lablog", "logs"], add="_DEL"):
+    def delete(self, verbose=True, sacredtexts=["lablog", "logs"], add="_DEL"):
         """
         Description:
             Remove both files and purge folders defined for the service, and rename to tag.
@@ -392,10 +392,8 @@ def delete_rename(self, verbose=True, sacredtexts=["lablog", "logs"], add="_DEL"
         # Purge folders
         if self.delete_folders != "":
             self.purge_folders(sacredtexts=sacredtexts, add=add, verbose=verbose)
-            # Rename to tag.
-            self.rename(add=add, to_find=self.delete_folders, verbose=verbose)
         else:
-            stderr.print("No folders to remove or rename")
+            stderr.print("No folders to remove")
         # Purge work
         self.delete_work()
         # Delete files
@@ -432,8 +430,10 @@ def full_clean(self):
         Perform and handle the whole cleaning of the service
         """
 
-        self.delete_rename()
+        self.delete()
         self.rename(to_find=self.nocopy, add="_NC", verbose=True)
+        if self.delete_folders != "":
+            self.rename(add="_DEL", to_find=self.delete_folders, verbose=True)
 
     def handle_clean(self):
         """
@@ -445,9 +445,11 @@ def handle_clean(self):
             self.show_nocopy()
         if self.option == "full_clean":
             self.full_clean()
-        if self.option == "rename_nocopy":
+        if self.option == "rename":
             self.rename(to_find=self.nocopy, add="_NC", verbose=True)
+            if self.delete_folders != "":
+                self.rename(add="_DEL", to_find=self.delete_folders, verbose=True)
         if self.option == "clean":
-            self.delete_rename()
+            self.delete()
         if self.option == "revert_renaming":
             self.revert_renaming()
diff --git a/bu_isciii/templates/services.json b/bu_isciii/templates/services.json
@@ -8,8 +8,8 @@
         "end": "",
         "description": "nf-core/bacass:  Simple bacterial assembly and annotation pipeline",
         "clean": {
-          "folders":["01-preprocessing/trimmed_sequences"],
-          "files":[]
+          "folders":[],
+          "files":["01-processing/fastp/sample_name_1.fastp.fastq.gz", "01-processing/fastp/sample_name_2.fastp.fastq.gz"]
         },
         "no_copy": ["RAW", "TMP", "latest"],
         "last_folder":"REFERENCES",
@@ -25,8 +25,8 @@
         "url": "https://github.com/ngs-fzb/MTBseq_source",
         "description": "Mycobacterium tuberculosis mapping, variant calling and detection of resistance using MTBseq",
         "clean": {
-          "folders":["01-preprocessing/trimmed_sequences", "Bam", "Mpileup"],
-          "files":[]
+          "folders":["Bam", "Mpileup"],
+          "files":["01-processing/fastp/sample_name_1.fastp.fastq.gz", "01-processing/fastp/sample_name_2.fastp.fastq.gz"]
         },
         "no_copy": ["RAW", "TMP"],
         "last_folder":"REFERENCES",
@@ -42,8 +42,8 @@
         "url": "https://github.com/ngs-fzb/MTBseq_source",
         "description": "Mycobacterium tuberculosis mapping, variant calling and detection of resistance using MTBseq",
         "clean": {
-          "folders":["01-preprocessing", "Bam", "Mpileup"],
-          "files":[]
+          "folders":["Bam", "Mpileup"],
+          "files":["01-processing/fastp/sample_name_1.fastp.fastq.gz", "01-processing/fastp/sample_name_2.fastp.fastq.gz"]
         },
         "no_copy": ["RAW", "TMP"],
         "last_folder":"REFERENCES",
@@ -146,7 +146,7 @@
         "description": "RNA-seq analysis",
         "clean": {
           "folders":[],
-          "files":[]
+          "files":["star_salmon/sample_name.Aligned.out.bam", "star_salmon/sample_name.Aligned.toTranscriptome.out.bam"]
         },
         "no_copy": ["RAW", "TMP"],
         "last_folder":"RESULTS",