🚚 Move default pipeline config into pipeline config directory

FCP-INDI · Aug 2, 2022 · 9fbe3bd · 9fbe3bd
1 parent b2fea07
commit 9fbe3bd
Show file tree

Hide file tree

Showing 16 changed files with 26 additions and 48 deletions.
diff --git a/.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile b/.github/Dockerfiles/C-PAC.develop-ABCD-HCP-bionic.Dockerfile
@@ -3,7 +3,6 @@ LABEL org.opencontainers.image.description "Full C-PAC image with software depen
 USER root
 
 # install C-PAC
-COPY dev/docker_data/default_pipeline.yml /cpac_resources/default_pipeline.yml
 COPY dev/circleci_data/pipe-test_ci.yml /cpac_resources/pipe-test_ci.yml
 COPY . /code
 RUN pip install -e /code

diff --git a/.github/Dockerfiles/C-PAC.develop-bionic.Dockerfile b/.github/Dockerfiles/C-PAC.develop-bionic.Dockerfile
@@ -3,7 +3,6 @@ LABEL org.opencontainers.image.description "Full C-PAC image"
 USER root
 
 # install C-PAC
-COPY dev/docker_data/default_pipeline.yml /cpac_resources/default_pipeline.yml
 COPY dev/circleci_data/pipe-test_ci.yml /cpac_resources/pipe-test_ci.yml
 COPY . /code
 RUN pip install -e /code

diff --git a/.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile b/.github/Dockerfiles/C-PAC.develop-fMRIPrep-LTS-xenial.Dockerfile
@@ -3,7 +3,6 @@ LABEL org.opencontainers.image.description "Full C-PAC image with software depen
 USER root
 
 # install C-PAC & set up runscript
-COPY dev/docker_data/default_pipeline.yml /cpac_resources/default_pipeline.yml
 COPY dev/circleci_data/pipe-test_ci.yml /cpac_resources/pipe-test_ci.yml
 COPY . /code
 RUN pip install -e /code

diff --git a/.github/workflows/check_updated_version.yml b/.github/workflows/check_updated_version.yml
@@ -44,9 +44,8 @@ jobs:
             VERSION=$(python -c "from info import __version__; print('.'.join(('.'.join(__version__[::-1].split('-')[1].split('.')[1:])[::-1], __version__.split('-')[1])) if '-' in __version__ else __version__)")
             cd ..
             echo "v${VERSION}" > version
-            sed -i -r "s/^(# [Vv]ersion ).*$/# Version ${VERSION}/g" dev/docker_data/default_pipeline.yml
             find ./CPAC/resources/configs -name "*.yml" -exec sed -i -r "s/^(# [Vv]ersion ).*$/# Version ${VERSION}/g" {} \;
-            git add version CPAC/resources/configs dev/docker_data/default_pipeline.yml
+            git add version CPAC/resources/configs
             if [[ ! -z $(git diff origin/${GITHUB_BRANCH}) ]]
             then
               git commit -m ":bookmark: Update version to ${VERSION} ($COMMIT_MESSAGE)"

diff --git a/.gitignore b/.gitignore
@@ -17,9 +17,6 @@ develop-eggs
 .installed.cfg
 .env*
 
-# Copy from packaging
-CPAC/resources/configs/default_pipeline.yml
-
 # Installer logs
 pip-log.txt
 

diff --git a/CPAC/pipeline/test/test_cpac_runner.py b/CPAC/pipeline/test/test_cpac_runner.py
@@ -5,6 +5,7 @@
 from CPAC.pipeline.cpac_runner import run_T1w_longitudinal
 from CPAC.pipeline.cpac_pipeline import load_cpac_pipe_config
 from CPAC.utils.bids_utils import create_cpac_data_config
+from CPAC.utils.configuration import DEFAULT_PIPELINE_FILE
 
 
 @pytest.mark.skip(reason='not a pytest test')
@@ -23,7 +24,7 @@ def test_run_T1w_longitudinal(bids_dir, cfg, test_dir, part_id):
     run_T1w_longitudinal(sub_data_list, cfg)
 
 
-cfg = "/code/default_pipeline.yml"
+cfg = DEFAULT_PIPELINE_FILE
 bids_dir = "/Users/steven.giavasis/data/neurodata_hnu"
 test_dir = "/test_dir"
 part_id = "0025427"

diff --git a/CPAC/pipeline/test/test_engine.py b/CPAC/pipeline/test/test_engine.py
@@ -1,10 +1,15 @@
 import os
 import pytest
 from CPAC.pipeline.cpac_pipeline import initialize_nipype_wf, \
-    load_cpac_pipe_config, connect_pipeline, build_anat_preproc_stack, build_workflow
-from CPAC.pipeline.engine import ResourcePool, ingress_raw_anat_data, ingress_raw_func_data, \
-    ingress_pipeconfig_paths, initiate_rpool
+                                        load_cpac_pipe_config, \
+                                        connect_pipeline, \
+                                        build_anat_preproc_stack, \
+                                        build_workflow
+from CPAC.pipeline.engine import ResourcePool, ingress_raw_anat_data, \
+                                 ingress_raw_func_data, \
+                                 ingress_pipeconfig_paths, initiate_rpool
 from CPAC.utils.bids_utils import create_cpac_data_config
+# from CPAC.utils.configuration import DEFAULT_PIPELINE_FILE
 
 
 @pytest.mark.skip(reason='not a pytest test')
@@ -149,7 +154,7 @@ def test_build_workflow(pipe_config, bids_dir, test_dir):
     rpool.gather_pipes(wf, cfg)
 
     wf.run()
-# cfg = "/code/default_pipeline.yml"
+# cfg = DEFAULT_PIPELINE_FILE
 # bids_dir = "/Users/steven.giavasis/data/HBN-SI_dataset/rawdata"
 # test_dir = "/test_dir"
 

diff --git a/dev/docker_data/default_pipeline.yml → ...urces/configs/pipeline_config_default.yml b/dev/docker_data/default_pipeline.yml → ...urces/configs/pipeline_config_default.yml
diff --git a/CPAC/utils/build_data_config.py b/CPAC/utils/build_data_config.py
@@ -1652,22 +1652,20 @@ def get_nonBIDS_data(anat_template, func_template, file_list=None,
 
 def util_copy_template(template_type=None):
     """Copy the data settings YAML file template to the current directory."""
-
     import os
     import shutil
     import pkg_resources as p
+    from CPAC.utils.configuration import DEFAULT_PIPELINE_FILE
 
     if not template_type:
-        type = 'data_settings'
+        type = "data_settings"
     else:
         type = template_type
 
-    settings_template = "/cpac_resources/default_pipeline.yml" if (
+    settings_template = DEFAULT_PIPELINE_FILE if (
         type == "pipeline_config"
-    ) else p.resource_filename("CPAC",
-        os.path.join("resources",
-                     "configs",
-                     "{0}_template.yml".format(type)))
+    ) else p.resource_filename("CPAC", os.path.join(
+        "resources", "configs", "{0}_template.yml".format(type)))
 
     settings_file = os.path.join(os.getcwd(), "{0}.yml".format(type))
 

diff --git a/CPAC/utils/configuration/configuration.py b/CPAC/utils/configuration/configuration.py
@@ -31,23 +31,11 @@
 SPECIAL_REPLACEMENT_STRINGS = {r'${resolution_for_anat}',
                                r'${func_resolution}'}
 
-# Find default config
-# in-container location
-DEFAULT_PIPELINE_FILE = '/cpac_resources/default_pipeline.yml'
-if not os.path.exists(DEFAULT_PIPELINE_FILE):
-    CPAC_DIRECTORY = os.path.abspath(os.path.join(
+DEFAULT_PIPELINE_FILE = os.path.join(
+    os.path.abspath(os.path.join(
         __file__,
-        *repeat(os.path.pardir, 4)))
-    # package location
-    DEFAULT_PIPELINE_FILE = os.path.join(
-        CPAC_DIRECTORY,
-        'CPAC/resources/configs/default_pipeline.yml')
-    # source code (developer) location
-    if not os.path.exists(DEFAULT_PIPELINE_FILE):
-        DEFAULT_PIPELINE_FILE = os.path.join(
-            CPAC_DIRECTORY,
-            'dev/docker_data/default_pipeline.yml')
-    del CPAC_DIRECTORY
+        *repeat(os.path.pardir, 4))),
+    'CPAC/resources/configs/pipeline_config_default.yml')
 
 with open(DEFAULT_PIPELINE_FILE, 'r') as dp_fp:
     default_config = yaml.safe_load(dp_fp)

diff --git a/CPAC/utils/tests/test_yaml.py b/CPAC/utils/tests/test_yaml.py
@@ -1,8 +1,8 @@
 import pytest
 import tempfile
 import yaml
-import yamlordereddictloader
 
+from CPAC.utils.configuration import DEFAULT_PIPELINE_FILE
 from CPAC.utils.yaml_template import create_yaml_from_template
 
 
@@ -11,10 +11,11 @@ def test_yaml_template():
 
     config_file = tempfile.mkstemp(suffix='test_yaml_template')[1]
 
-    # Create a new YAML configuration file based on the default_pipeline.yml file.
-    config = yaml.safe_load(open('./data/default_pipeline.yml', 'r'))
+    # Create a new YAML configuration file based on the default pipeline
+    # YAML file.
+    config = yaml.safe_load(open(DEFAULT_PIPELINE_FILE, 'r'))
 
-    new_config = create_yaml_from_template(config, './data/default_pipeline.yml')
+    new_config = create_yaml_from_template(config, DEFAULT_PIPELINE_FILE)
 
     with open(config_file, 'wb') as f:
         f.write(new_config)

diff --git a/Dockerfile b/Dockerfile
@@ -3,7 +3,6 @@ LABEL org.opencontainers.image.description "Full C-PAC image"
 USER root
 
 # install C-PAC
-COPY dev/docker_data/default_pipeline.yml /cpac_resources/default_pipeline.yml
 COPY dev/circleci_data/pipe-test_ci.yml /cpac_resources/pipe-test_ci.yml
 COPY . /code
 RUN pip install -e /code

diff --git a/dev/circleci_data/override_version_tag_list b/dev/circleci_data/override_version_tag_list
@@ -47,7 +47,6 @@ def overwrite_list(file_diff_stats):
         fd[0] for fd in file_diff_stats if ((
             re.match(config_pattern, fd[0]) or
             fd[0] in {
-                'dev/docker_data/default_pipeline.yml',
                 'version'
             }
         ) and fd[1] == '2 +-')

diff --git a/setup.py b/setup.py
@@ -62,10 +62,6 @@ def main(**extra_args):
         REQUIREMENTS,
     )
 
-    # copy default pipeline into package
-    open('CPAC/resources/configs/default_pipeline.yml', 'w').write(
-        open('dev/docker_data/default_pipeline.yml', 'r').read())
-
     setup(
         name=NAME,
         maintainer=MAINTAINER,

diff --git a/variant-ABCD-HCP.Dockerfile b/variant-ABCD-HCP.Dockerfile
@@ -3,7 +3,6 @@ LABEL org.opencontainers.image.description "Full C-PAC image with software depen
 USER root
 
 # install C-PAC
-COPY dev/docker_data/default_pipeline.yml /cpac_resources/default_pipeline.yml
 COPY dev/circleci_data/pipe-test_ci.yml /cpac_resources/pipe-test_ci.yml
 COPY . /code
 RUN pip install -e /code

diff --git a/variant-fMRIPrep-LTS.Dockerfile b/variant-fMRIPrep-LTS.Dockerfile
@@ -3,7 +3,6 @@ LABEL org.opencontainers.image.description "Full C-PAC image with software depen
 USER root
 
 # install C-PAC & set up runscript
-COPY dev/docker_data/default_pipeline.yml /cpac_resources/default_pipeline.yml
 COPY dev/circleci_data/pipe-test_ci.yml /cpac_resources/pipe-test_ci.yml
 COPY . /code
 RUN pip install -e /code