Merge pull request #7 from rabix/feature/improved_publishDir_search

Feature/improved publish dir search

wrabbit/parser/nextflow.py

@@ -17,6 +17,7 @@
     get_entrypoint,
     get_docs_file,
     get_sample_sheet_schema,
+    find_publish_params,
 )
 
 from wrabbit.parser.constants import (
@@ -46,6 +47,9 @@
 from wrabbit.specification.hints import (
     NextflowExecutionMode,
 )
+from wrabbit.specification.sbg import (
+    ExecutorVersion
+)
 
 from wrabbit.exceptions import (
     SampleSheetError, MalformedConfigException, ErrorMessages,
@@ -173,10 +177,36 @@ def generate_sb_inputs(self):
                     ))
 
         # Remap publishDir to string type
+        found_publish_dir = False
         if temp := self.sb_wrapper.get_input(NFCORE_OUTPUT_DIRECTORY_ID):
             print(f"Detected publishDir input --{temp.id_}. Remapping to "
                   f"string type input.")
             temp.set_property('type', 'string')
+            found_publish_dir = True
+
+        if not found_publish_dir:
+            # Look for publish dir in config files
+            pub_dir_params = set()
+            for config_file in self.nf_config_files:
+                pub_dir_params = pub_dir_params.union(
+                    find_publish_params(config_file)
+                )
+
+            pub_dir_params = list(pub_dir_params)
+
+            if pub_dir_params:
+                if len(pub_dir_params) == 1:
+                    id_ = pub_dir_params.pop()
+                    print(f"Detected publishDir input --{id_}. Remapping to "
+                          f"string type input.")
+                    if temp := self.sb_wrapper.get_input(id_):
+                        temp.set_property('type', 'string')
+                    else:
+                        print(f"Input --{id_} not found.")
+
+                if len(pub_dir_params) > 1:
+                    print(f"Detected multiple possible publishDir inputs. "
+                          f"Skipping remapping.")
 
         # Add the generic file array input - auxiliary files
         self.sb_wrapper.safe_add_input(GENERIC_FILE_ARRAY_INPUT)
@@ -206,12 +236,15 @@ def generate_app_data(self):
             manifest_data = parse_manifest(file)
 
             self.entrypoint = self.entrypoint or manifest_data.get(
-                'mainScript', None) or get_entrypoint(self.workflow_path)
+                'mainScript', None)
 
-            executor_version = manifest_data.get('nextflowVersion', None)
-            if executor_version:
-                executor_version = get_executor_version(executor_version)
-            self.executor_version = self.executor_version or executor_version
+            if not self.executor_version:
+                executor_version = manifest_data.get('nextflowVersion', None)
+                if executor_version:
+                    executor_version = get_executor_version(executor_version)
+                self.executor_version = executor_version
+            else:
+                print(f"Using provided version {self.executor_version}")
 
             tk_author = manifest_data.get('author', None)
             if not self.sb_wrapper.toolkit_author:
@@ -229,10 +262,14 @@ def generate_app_data(self):
                 # Stop searching if manifest is found
                 break
 
+        if not self.entrypoint:
+            self.entrypoint = get_entrypoint(self.workflow_path)
+
         if not self.executor_version and self.sb_doc:
             self.executor_version = get_executor_version(self.sb_doc)
 
-        if self.executor_version:
+        if self.executor_version and \
+                isinstance(self.executor_version, ExecutorVersion):
             # Confirm if the executor version is valid.
             # If it is below 22.10.1 it is not supported.
             self.executor_version.correct_version()
@@ -246,6 +283,12 @@ def generate_app_data(self):
         # },
         # manifest.homePage
 
+    def nf_schema_build(self):
+        """
+        To be replaced before used.
+        """
+        pass
+
     def generate_sb_app(
             self, sb_entrypoint='main.nf',
             executor_version: Optional[str] = None,
@@ -261,9 +304,10 @@ def generate_sb_app(
         self.sb_wrapper.cwl_version = 'None'
         self.sb_wrapper.class_ = 'nextflow'
 
+        self.generate_app_data()
+        self.nf_schema_build()
         self.generate_sb_inputs()
         self.generate_sb_outputs()
-        self.generate_app_data()
 
         if sample_sheet_schema or self.sb_samplesheet_schema:
             self.parse_sample_sheet_schema(open(
@@ -411,7 +455,7 @@ def parse_sample_sheet_schema(self, path):
         self.sb_wrapper.add_requirement(INLINE_JS_REQUIREMENT)
         self.sb_wrapper.add_requirement(LOAD_LISTING_REQUIREMENT)
 
-    def make_output_type(self, key, output_dict, is_record=False):
+    def make_output_type(self, key, output_dict, is_record=False) -> dict:
         """
         This creates an output of specific type based on information provided
         through output_dict.

wrabbit/parser/utils.py

@@ -272,7 +272,7 @@ def get_executor_version(string: str):
         return verify_version(*result.pop(0))
 
     result = re.findall(
-        r"((?:[!><=]+|))([0-9.]+)((?:\+|))",
+        r"((?:[!><=]+|))(\d{2}\.\d+\.\d+)((?:\+|))",
         string
     )
 
@@ -299,7 +299,28 @@ def get_sample_sheet_schema(path):
         return None
 
 
-def get_config_files(path):
+def get_config_files(path: str) -> list:
+    """
+    Auto find config files.
+    """
+    paths = list_config_files(path)
+    all_paths = paths
+    include_pattern = re.compile(
+        r'^includeConfig\s+[\'\"]([a-zA-Z_.\\/]+)[\'\"]'
+    )
+
+    for config_path in paths:
+        with open(config_path, 'r') as config_file:
+            for line in config_file.readlines():
+                if include_path := re.findall(include_pattern, line):
+                    chk = os.path.join(path, include_path[0])
+                    if os.path.exists(chk):
+                        all_paths.append(chk)
+
+    return all_paths
+
+
+def list_config_files(path: str) -> list:
     """
     Auto find config files.
     """
@@ -336,6 +357,46 @@ def find_config_section(file_path: str, section: str) -> str:
     return section_text
 
 
+def find_publish_params(file_path: str) -> set:
+    sections = []
+    section_text = ""
+    brackets = 0
+    found_section = False
+
+    with open(file_path, 'r') as file:
+        for line in file.readlines():
+            if found_section:
+                section_text += line
+                brackets += line.count("[") - line.count("]")
+
+            if brackets < 0:
+                sections.append(section_text)
+                brackets = 0
+                section_text = ""
+                found_section = False
+
+            if re.findall(r'publishDir(?:\s|)=(?:\s+|)\[', line):
+                section_text += "[\n"
+                found_section = True
+
+                # Handle "section []"
+                if line.count("[") == line.count("]"):
+                    section_text += line[line.index("[")+1:line.index("]")]
+                    section_text += "]"
+                    sections.append(section_text)
+                    brackets = 0
+                    section_text = ""
+                    found_section = False
+
+    params = set()
+
+    for s in sections:
+        if param := re.findall(r"path:[^$]+\$\{(?:\s+|)params\.(\w+)", s):
+            params = params.union(set(param))
+
+    return params
+
+
 def parse_manifest(file_path: str) -> dict:
     manifest_text = find_config_section(file_path, 'manifest')
 

wrabbit/specification/sbg.py

@@ -60,9 +60,7 @@ def __init__(
 
     def correct_version(self):
         if not self.from_version:
-            raise ValueError(
-                "Executor Version was not set."
-            )
+            return
         if self._from_version >= Version(MINIMUM_SUPPORTED_NF_VERSION):
             return
         if self.from_sign in ['<', '=', '<=']:

wrabbit/version.py

@@ -1 +1 @@
-__version__ = "0.2.0"
+__version__ = "0.2.1"

wrabbit/wrapper/wrapper.py

@@ -139,13 +139,14 @@ def set_app_content(
     ):
         payload = dict()
 
+        if hasattr(executor_version, 'serialize'):
+            executor_version = executor_version.serialize()
+
         if code_package:
             payload['code_package'] = code_package
         if entrypoint:
             payload['entrypoint'] = entrypoint
         if executor_version:
-            if hasattr(executor_version, 'serialize'):
-                executor_version = executor_version.serialize()
             payload['executor_version'] = executor_version
 
         payload.update(kwargs)