🧑‍💻 Move YAML comment updater to pre-commit hook

.pre-commit-config.yaml

@@ -23,3 +23,14 @@ repos:
         entry: .github/scripts/autoversioning.sh
         language: script
         files: '.*Dockerfile$|.*\.yaml$|^CPAC/info\.py$'
+      - id: update-yaml-comments
+        name: Update YAML comments
+        entry: CPAC/utils/configuration/yaml_template.py
+        language: python
+        files: '^CPAC/resources/configs/pipeline_config_.*\.ya?ml'
+        additional_dependencies:
+          - "click"
+          - "nipype"
+          - "pathvalidate"
+          - "pyyaml"
+          - "voluptuous"

CPAC/pipeline/schema.py

@@ -21,6 +21,7 @@
 from itertools import chain, permutations
 import numpy as np
 from pathvalidate import sanitize_filename
+from subprocess import CalledProcessError
 from voluptuous import All, ALLOW_EXTRA, Any, BooleanInvalid, Capitalize, \
                        Coerce, CoerceInvalid, ExclusiveInvalid, In, Length, \
                        LengthInvalid, Lower, Match, Maybe, MultipleInvalid, \
@@ -355,6 +356,7 @@ def sanitize(filename):
 
 latest_schema = Schema({
     'FROM': Maybe(str),
+    'skip env check': Maybe(bool),  # flag for skipping an environment check
     'pipeline_setup': {
         'pipeline_name': All(str, Length(min=1), sanitize),
         'output_directory': {
@@ -1182,13 +1184,14 @@ def schema(config_dict):
     except KeyError:
         pass
     try:
-        if 'unet' in [using.lower() for using in
-                      partially_validated['anatomical_preproc'][
-                          'brain_extraction']['using']]:
+        if not partially_validated.get("skip env check"
+                                       ) and 'unet' in [using.lower() for using in
+                partially_validated['anatomical_preproc'][
+                    'brain_extraction']['using']]:
             try:
                 from importlib import import_module
                 import_module('CPAC.unet')
-            except (ImportError, ModuleNotFoundError, OSError) as error:
+            except (CalledProcessError, ImportError, ModuleNotFoundError, OSError) as error:
                 import site
                 raise OSError(
                     'U-Net brain extraction requires torch to be installed, '

CPAC/resources/configs/pipeline_config_regtest-4.yml

@@ -99,7 +99,6 @@ anatomical_preproc:
       # Perform final surface smoothing after all iterations. Default is 20.
       smooth_final: 22
 
-
   # Non-local means filtering via ANTs DenoiseImage
   non_local_means_filtering:
 

CPAC/utils/configuration/configuration.py

@@ -99,20 +99,32 @@ class Configuration:
     >>> slack_420349_preconfig['pipeline_setup', 'pipeline_name']
     'slack_420349_preconfig'
     """
-    def __init__(self, config_map=None):
+    def __init__(self, config_map: Optional[dict] = None,
+                 skip_env_check: bool = False) -> None:
+        """Initialize a Configuration instance.
+
+        Parameters
+        ----------
+        config_map : dict, optional
+
+        skip_env_check : bool, optional
+        """
         from CPAC.pipeline.schema import schema
         from CPAC.utils.utils import lookup_nested_value, update_nested_dict
 
         if config_map is None:
             config_map = {}
+        if skip_env_check:
+            config_map['skip env check'] = True
 
         base_config = config_map.pop('FROM', None)
         if base_config:
             if base_config.lower() in ['default', 'default_pipeline']:
                 base_config = 'default'
             # import another config (specified with 'FROM' key)
             try:
-                base_config = Preconfiguration(base_config)
+                base_config = Preconfiguration(base_config,
+                                               skip_env_check=skip_env_check)
             except BadParameter:
                 base_config = configuration_from_file(base_config)
             config_map = update_nested_dict(base_config.dict(), config_map)
@@ -140,20 +152,26 @@ def __init__(self, config_map=None):
 
         config_map = schema(config_map)
 
+        # remove 'skip env check' now that the config is validated
+        if "skip env check" in config_map:
+            del config_map["skip env check"]
         # remove 'FROM' before setting attributes now that it's imported
         if 'FROM' in config_map:
             del config_map['FROM']
 
-        # set FSLDIR to the environment $FSLDIR if the user sets it to
-        # 'FSLDIR' in the pipeline config file
-        _FSLDIR = config_map.get('FSLDIR')
-        if _FSLDIR and bool(re.match(r'^[\$\{]{0,2}?FSLDIR[\}]?$', _FSLDIR)):
-            config_map['FSLDIR'] = os.environ['FSLDIR']
-
-        for key in config_map:
-            # set attribute
-            setattr(self, key, set_from_ENV(config_map[key]))
-
+        if skip_env_check:
+            for key in config_map:
+                # set attribute
+                setattr(self, key, self.set_without_ENV(config_map[key]))
+        else:
+            # set FSLDIR to the environment $FSLDIR if the user sets it to
+            # 'FSLDIR' in the pipeline config file
+            _FSLDIR = config_map.get('FSLDIR')
+            if _FSLDIR and bool(re.match(r'^[\$\{]{0,2}?FSLDIR[\}]?$', _FSLDIR)):
+                config_map['FSLDIR'] = os.environ['FSLDIR']
+            for key in config_map:
+                # set attribute
+                setattr(self, key, self.set_from_ENV(config_map[key]))
         self._update_attr()
 
         # set working directory as an environment variable
@@ -271,6 +289,78 @@ def return_config_elements(self):
         ]
         return attributes
 
+    def set_from_ENV(self, conf):  # pylint: disable=invalid-name
+        '''Replace strings like $VAR and ${VAR} with environment variable values.
+
+        Parameters
+        ----------
+        conf : any
+
+        Returns
+        -------
+        conf : any
+
+        Examples
+        --------
+        >>> import os
+        >>> os.environ['SAMPLE_VALUE_SFE'] = '/example/path'
+        >>> c = Configuration()
+        >>> c.set_from_ENV({'key': {'nested_list': [
+        ...     1, '1', '$SAMPLE_VALUE_SFE/extended']}})
+        {'key': {'nested_list': [1, '1', '/example/path/extended']}}
+        >>> c.set_from_ENV(['${SAMPLE_VALUE_SFE}', 'SAMPLE_VALUE_SFE'])
+        ['/example/path', 'SAMPLE_VALUE_SFE']
+        >>> del os.environ['SAMPLE_VALUE_SFE']
+        '''
+        if isinstance(conf, list):
+            return [self.set_from_ENV(item) for item in conf]
+        if isinstance(conf, dict):
+            return {key: self.set_from_ENV(conf[key]) for key in conf}
+        if isinstance(conf, str):
+            # set any specified environment variables
+            # (only matching all-caps plus `-` and `_`)
+            # like `${VAR}`
+            _pattern1 = r'\${[A-Z\-_]*}'
+            # like `$VAR`
+            _pattern2 = r'\$[A-Z\-_]*(?=/|$)'
+            # replace with environment variables if they exist
+            for _pattern in [_pattern1, _pattern2]:
+                _match = re.search(_pattern, conf)
+                if _match:
+                    _match = _match.group().lstrip('${').rstrip('}')
+                    conf = re.sub(
+                        _pattern, os.environ.get(_match, f'${_match}'), conf)
+        return conf
+
+    def set_without_ENV(self, conf):  # pylint: disable=invalid-name
+        '''Retain strings like $VAR and ${VAR} when setting attributes.
+
+        Parameters
+        ----------
+        conf : any
+
+        Returns
+        -------
+        conf : any
+
+        Examples
+        --------
+        >>> import os
+        >>> os.environ['SAMPLE_VALUE_SFE'] = '/example/path'
+        >>> c = Configuration()
+        >>> c.set_without_ENV({'key': {'nested_list': [
+        ...     1, '1', '$SAMPLE_VALUE_SFE/extended']}})
+        {'key': {'nested_list': [1, '1', '$SAMPLE_VALUE_SFE/extended']}}
+        >>> c.set_without_ENV(['${SAMPLE_VALUE_SFE}', 'SAMPLE_VALUE_SFE'])
+        ['${SAMPLE_VALUE_SFE}', 'SAMPLE_VALUE_SFE']
+        >>> del os.environ['SAMPLE_VALUE_SFE']
+        '''
+        if isinstance(conf, list):
+            return [self.set_without_ENV(item) for item in conf]
+        if isinstance(conf, dict):
+            return {key: self.set_without_ENV(conf[key]) for key in conf}
+        return conf
+
     def sub_pattern(self, pattern, orig_key):
         return orig_key.replace(pattern, self[pattern[2:-1].split('.')])
 
@@ -659,52 +749,9 @@ class Preconfiguration(Configuration):
     preconfig : str
         The canonical name of the preconfig to load
     """
-    def __init__(self, preconfig):
-        super().__init__(config_map=preconfig_yaml(preconfig, True))
-
-
-def set_from_ENV(conf):  # pylint: disable=invalid-name
-    '''Function to replace strings like $VAR and ${VAR} with
-    environment variable values
-
-    Parameters
-    ----------
-    conf : any
-
-    Returns
-    -------
-    conf : any
-
-    Examples
-    --------
-    >>> import os
-    >>> os.environ['SAMPLE_VALUE_SFE'] = '/example/path'
-    >>> set_from_ENV({'key': {'nested_list': [
-    ...     1, '1', '$SAMPLE_VALUE_SFE/extended']}})
-    {'key': {'nested_list': [1, '1', '/example/path/extended']}}
-    >>> set_from_ENV(['${SAMPLE_VALUE_SFE}', 'SAMPLE_VALUE_SFE'])
-    ['/example/path', 'SAMPLE_VALUE_SFE']
-    >>> del os.environ['SAMPLE_VALUE_SFE']
-    '''
-    if isinstance(conf, list):
-        return [set_from_ENV(item) for item in conf]
-    if isinstance(conf, dict):
-        return {key: set_from_ENV(conf[key]) for key in conf}
-    if isinstance(conf, str):
-        # set any specified environment variables
-        # (only matching all-caps plus `-` and `_`)
-        # like `${VAR}`
-        _pattern1 = r'\${[A-Z\-_]*}'
-        # like `$VAR`
-        _pattern2 = r'\$[A-Z\-_]*(?=/|$)'
-        # replace with environment variables if they exist
-        for _pattern in [_pattern1, _pattern2]:
-            _match = re.search(_pattern, conf)
-            if _match:
-                _match = _match.group().lstrip('${').rstrip('}')
-                conf = re.sub(
-                    _pattern, os.environ.get(_match, f'${_match}'), conf)
-    return conf
+    def __init__(self, preconfig, skip_env_check=False):
+        super().__init__(config_map=preconfig_yaml(preconfig, True),
+                         skip_env_check=skip_env_check)
 
 
 def set_subject(sub_dict: dict, pipe_config: 'Configuration',

CPAC/utils/configuration/yaml_template.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 # Copyright (C) 2022  C-PAC Developers
 
 # This file is part of C-PAC.
@@ -18,6 +19,7 @@
 from copy import deepcopy
 import os
 import re
+from typing import Optional, Union
 from datetime import datetime
 from hashlib import sha1
 from click import BadParameter
@@ -226,8 +228,10 @@ def _count_indent(line):
     return (len(line) - len(line.lstrip())) // 2
 
 
-def create_yaml_from_template(d,  # pylint: disable=invalid-name
-                              template='default', import_from=None):
+def create_yaml_from_template(
+        d: Union[Configuration, dict],  # pylint: disable=invalid-name
+        template: str = 'default', import_from: Optional[str] = None,
+        skip_env_check: Optional[bool] = False) -> str:
     """Save dictionary to a YAML file, keeping the structure
     (such as first level comments and ordering) from the template
 
@@ -244,6 +248,9 @@ def create_yaml_from_template(d,  # pylint: disable=invalid-name
     import_from : str, optional
         name of a preconfig. Full config is generated if omitted
 
+    skip_env_check : bool, optional
+        skip environment check (for validating a config without running)
+
     Examples
     --------
     >>> import yaml
@@ -284,7 +291,7 @@ def create_yaml_from_template(d,  # pylint: disable=invalid-name
         base_config = None
     else:  # config based on preconfig
         d = Configuration(d) if not isinstance(d, Configuration) else d
-        base_config = Preconfiguration(import_from)
+        base_config = Preconfiguration(import_from, skip_env_check=skip_env_check)
         d = (d - base_config).left
         d.update({'FROM': import_from})
     yaml_template = YamlTemplate(template, base_config)
@@ -466,8 +473,9 @@ def update_a_preconfig(preconfig, import_from):
     """
     import sys
     print(f'Updating {preconfig} preconfig…', file=sys.stderr)
-    updated = create_yaml_from_template(Preconfiguration(preconfig),
-                                        import_from=import_from)
+    updated = create_yaml_from_template(Preconfiguration(preconfig,
+                                                         skip_env_check=True),
+                                        import_from=import_from, skip_env_check=True)
     with open(preconfig_yaml(preconfig), 'w', encoding='utf-8') as _f:
         _f.write(updated)