fix merge

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,4 @@
+[core]
+    remote = scil-data
+['remote "scil-data"']
+    url = https://scil.usherbrooke.ca/scil_test_data/dvc-store

.dvc/test_descriptors.yml

@@ -0,0 +1,2 @@
+aodf:
+    revision: ee0596f9474d8a24f7620e4cb00f2e2874ca02d1

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

requirements.txt

@@ -10,6 +10,13 @@ deepdiff==6.3.0
 dmri-amico==2.0.*
 dmri-commit==2.1.*
 docopt==0.6.*
+dvc==3.39.0
+dvc-data==3.7.0
+dvc-http==2.32.0
+dvc-objects==3.0.6
+dvc-render==1.0.0
+dvc-studio-client==0.18.0
+dvc-task==0.3.0
 formulaic==0.3.*
 fury==0.9.*
 future==0.18.*

scilpy/__init__.py

@@ -0,0 +1,19 @@
+
+
+def get_home():
+    import os
+    """ Set a user-writeable file-system location to put files. """
+    if 'SCILPY_HOME' in os.environ:
+        scilpy_home = os.environ['SCILPY_HOME']
+    else:
+        scilpy_home = os.path.join(os.path.expanduser('~'), '.scilpy')
+    return scilpy_home
+
+
+def get_root():
+    import os
+    return os.path.realpath(f"{os.path.dirname(os.path.abspath(__file__))}/..")
+
+
+SCILPY_HOME = get_home()
+SCILPY_ROOT = get_root()

scilpy/image/tests/test_volume_operations.py

@@ -3,27 +3,28 @@
 import os
 import tempfile
 
-from dipy.io.gradients import read_bvals_bvecs
 import nibabel as nib
 import numpy as np
+from dipy.io.gradients import read_bvals_bvecs
 from numpy.testing import assert_equal
 
-from scilpy.image.volume_operations import (flip_volume,
-                                            crop_volume,
-                                            apply_transform,
+from scilpy import SCILPY_HOME
+from scilpy.image.volume_operations import (apply_transform,
                                             compute_snr,
+                                            crop_volume,
+                                            flip_volume,
                                             resample_volume)
-from scilpy.io.fetcher import fetch_data, get_testing_files_dict, get_home
+from scilpy.io.fetcher import fetch_data, get_testing_files_dict
 from scilpy.utils.util import compute_nifti_bounding_box
 
 # Fetching testing dwi data.
 fetch_data(get_testing_files_dict(), keys='processing.zip')
 tmp_dir = tempfile.TemporaryDirectory()
-in_dwi = os.path.join(get_home(), 'processing', 'dwi.nii.gz')
-in_bval = os.path.join(get_home(), 'processing', 'dwi.bval')
-in_bvec = os.path.join(get_home(), 'processing', 'dwi.bvec')
-in_mask = os.path.join(get_home(), 'processing', 'cc.nii.gz')
-in_noise_mask = os.path.join(get_home(), 'processing',
+in_dwi = os.path.join(SCILPY_HOME, 'processing', 'dwi.nii.gz')
+in_bval = os.path.join(SCILPY_HOME, 'processing', 'dwi.bval')
+in_bvec = os.path.join(SCILPY_HOME, 'processing', 'dwi.bvec')
+in_mask = os.path.join(SCILPY_HOME, 'processing', 'cc.nii.gz')
+in_noise_mask = os.path.join(SCILPY_HOME, 'processing',
                              'small_roi_gm_mask.nii.gz')
 
 

scilpy/io/dvc.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+
+
+import os
+
+import yaml
+from dvc import api, config
+
+from scilpy import SCILPY_HOME, SCILPY_ROOT
+
+DVC_REPOSITORY = "https://github.com/scilus/neurogister.git"
+DEFAULT_CACHE_CONFIG = {
+    "type": "symlink",
+    "shared": "group",
+    "dir": os.path.join(SCILPY_HOME, "dvc-cache")
+}
+
+
+def get_default_config():
+    return config.Config(config={
+        "cache": DEFAULT_CACHE_CONFIG
+    })
+
+
+def pull_test_case_package(package_name):
+    """
+    Pull a package for a test case located in the `scilpy_tests` vendor. Refer
+    to the `Scilpy Tests Vendor`_ for available packages.
+
+    Parameters
+    ----------
+    package_name : str
+        Name of the package to pull from the `scilpy_tests` vendor.
+
+    Returns
+    -------
+    str
+        The path to the pulled package.
+
+    .. _Scilpy Tests Vendor:
+        https://github.com/scilus/neurogister/tree/main/store/scilpy_tests/meta.yml # noqa
+    """
+    with open(f"{SCILPY_ROOT}/.dvc/test_descriptors.yml", 'r') as f:
+        test_descriptors = yaml.safe_load(f)
+
+        if package_name not in test_descriptors:
+            raise ValueError(f"Unknown test package: {package_name}")
+
+        pull_package_from_dvc_repository(
+            "scilpy_tests", f"{package_name}", f"{SCILPY_HOME}/test_data",
+            git_revision=test_descriptors[package_name]["revision"])
+
+        return f"{SCILPY_HOME}/test_data/{package_name}"
+
+
+def pull_package_from_dvc_repository(vendor, package_name, local_destination,
+                                     git_remote_url=DVC_REPOSITORY,
+                                     git_revision="main",
+                                     dvc_remote_name="scil-data",
+                                     dvc_config_root=f"{SCILPY_ROOT}/.dvc"):
+    """
+    Pull a package from a correctly configured DVC remote repository. Packages
+    are located in the store directory, organized in vendors with different
+    purposes. Refer to the `SCIL Data Store`_ for more information.
+
+    Parameters
+    ----------
+    vendor : str
+        Name of the vendor to pull the package from.
+    package_name : str
+        Name of the package to pull from the vendor.
+    local_destination : str
+        Destination where to pull the package.
+    git_remote_url : str
+        URL of the Git repository containing DVC artifacts.
+    git_revision : str
+        Git revision of the DVC artifacts repository.
+    dvc_remote_name : str
+        Name of the DVC remote to use.
+    dvc_config_root : str
+        Location of the DVC configuration files.
+
+    Returns
+    -------
+    str
+        The path to the pulled package.
+
+    .. _SCIL Data Store:
+        https://github.com/scilus/neurogister/tree/main/store
+    """
+    return pull_from_dvc_repository(f"store/{vendor}/{package_name}",
+                                    f"{local_destination}/",
+                                    git_remote_url, git_revision,
+                                    dvc_remote_name, dvc_config_root)
+
+
+def pull_from_dvc_repository(remote_location, local_destination,
+                             git_remote_url=DVC_REPOSITORY,
+                             git_revision="main",
+                             dvc_remote_name="scil-data",
+                             dvc_config_root=f"{SCILPY_ROOT}/.dvc"):
+    """
+    Pull data from a DVC remote repository to the specified location.
+
+    Parameters
+    ----------
+    remote_location : str
+        Location of the data to pull in the repository.
+    local_destination : str
+        Destination where to pull the data.
+    git_remote_url : str
+        URL of the Git repository containing DVC artifacts.
+    git_revision : str
+        Git revision of the DVC artifacts repository.
+    dvc_remote_name : str
+        Name of the DVC remote to use.
+    dvc_config_root : str
+        Location of the DVC configuration files.
+
+    Returns
+    -------
+    str
+        The path to the pulled package.
+    """
+    repository_config = get_default_config()
+    repository_config.merge(config.Config(dvc_config_root))
+    remote_config = repository_config["remote"][dvc_remote_name]
+
+    registry = api.DVCFileSystem(git_remote_url, git_revision,
+                                 config=repository_config,
+                                 remote_name=dvc_remote_name,
+                                 remote_config=remote_config)
+
+    registry.get(remote_location, local_destination, True, cache=True)

scilpy/io/fetcher.py

@@ -1,13 +1,15 @@
 # -*- coding: utf-8 -*-
 
+import hashlib
 import inspect
 import logging
-import hashlib
 import os
 import pathlib
-import requests
 import zipfile
 
+import requests
+
+from scilpy import SCILPY_HOME
 
 DVC_URL = "https://scil.usherbrooke.ca/scil_test_data/dvc-store/files/md5"
 
@@ -35,15 +37,6 @@ def save_response_content(response, destination):
     save_response_content(response, destination)
 
 
-def get_home():
-    """ Set a user-writeable file-system location to put files. """
-    if 'SCILPY_HOME' in os.environ:
-        scilpy_home = os.environ['SCILPY_HOME']
-    else:
-        scilpy_home = os.path.join(os.path.expanduser('~'), '.scilpy')
-    return scilpy_home
-
-
 def get_testing_files_dict():
     """ Get dictionary linking zip file to their GDrive ID & MD5SUM """
     return {
@@ -76,25 +69,24 @@ def fetch_data(files_dict, keys=None):
     But with too many data accesses, downloaded become denied.
     Using trick from https://github.com/wkentaro/gdown/issues/43.
     """
-    scilpy_home = get_home()
 
-    if not os.path.exists(scilpy_home):
-        os.makedirs(scilpy_home)
+    if not os.path.exists(SCILPY_HOME):
+        os.makedirs(SCILPY_HOME)
 
     if keys is None:
         keys = files_dict.keys()
     elif isinstance(keys, str):
         keys = [keys]
     for f in keys:
         url_md5 = files_dict[f]
-        full_path = os.path.join(scilpy_home, f)
+        full_path = os.path.join(SCILPY_HOME, f)
         full_path_no_ext, ext = os.path.splitext(full_path)
 
         CURR_URL = DVC_URL + "/" + url_md5[:2] + "/" + url_md5[2:]
         if not os.path.isdir(full_path_no_ext):
             if ext == '.zip' and not os.path.isdir(full_path_no_ext):
                 logging.warning('Downloading and extracting {} from url {} to '
-                                '{}'.format(f, CURR_URL, scilpy_home))
+                                '{}'.format(f, CURR_URL, SCILPY_HOME))
 
                 # Robust method to Virus/Size check from GDrive
                 download_file_from_google_drive(CURR_URL, full_path)