diff --git a/README.md b/README.md index 2a9b2898..bdc85850 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # mepo [![Actions Status](https://github.com/pchakraborty/mepo/workflows/Unit%20testing%20of%20mepo/badge.svg)](https://github.com/pchakraborty/mepo/actions) [![DOI](https://zenodo.org/badge/215067850.svg)](https://zenodo.org/badge/latestdoi/215067850) -`mepo` is a tool, written in Python3 (3.6.0+), to manage (m)ultiple git r(epo)sitories, by attempting to create an illusion of a 'single repository' for multi-repository projects. Please see the [Wiki](../../wiki) for examples of `mepo` workflows. +`mepo` is a tool, written in Python3 (3.9.0+), to manage (m)ultiple git r(epo)sitories, by attempting to create an illusion of a 'single repository' for multi-repository projects. Please see the [Wiki](../../wiki) for examples of `mepo` workflows. ## Commands diff --git a/bin/mepo b/bin/mepo3 similarity index 90% rename from bin/mepo rename to bin/mepo3 index fb10b0bc..719239bb 100755 --- a/bin/mepo +++ b/bin/mepo3 @@ -13,5 +13,5 @@ SRC_D = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'src') sys.path.insert(0, SRC_D) if __name__ == '__main__': - from mepo.main import main + from mepo3.__main__ import main main() diff --git a/docs/make_md_docs.py b/docs/make_md_docs.py index 07ba6d08..076785d4 100644 --- a/docs/make_md_docs.py +++ b/docs/make_md_docs.py @@ -2,6 +2,7 @@ import os import io +import glob from mdutils.mdutils import MdUtils import subprocess as sp @@ -13,52 +14,58 @@ doc_dir_path = os.path.dirname(os.path.realpath(__file__)) # Then we need to get to the mepo/mepo.d/command directory. First the "main" dir main_dir_path = os.path.dirname(doc_dir_path) -# Now add 'src/mepo' -mepod_dir_path = os.path.join(main_dir_path, 'src', 'mepo') -# And then 'command' -command_dir_path = os.path.join(mepod_dir_path, 'command') +# Now add "src/mepo" +mepod_dir_path = os.path.join(main_dir_path, "src", "mepo3") +# And then "command" +command_dir_path = os.path.join(mepod_dir_path, "command") -mepo_command_path = os.path.join(main_dir_path, 'bin', 'mepo') +mepo_command_path = os.path.join(main_dir_path, "bin", "mepo3") def get_command_list(directory): - # Walk the tree - roots = [x[0] for x in os.walk(directory)] + # # Walk the tree + # roots = [x[0] for x in os.walk(directory)] - # Now remove "." from the list - roots = roots[1:] + # # Now remove "." from the list + # roots = roots[1:] + # print(f"roots: {roots}") - # Just get the relative paths - rel_roots = [os.path.relpath(x,directory) for x in roots] + # # Just get the relative paths + # rel_roots = [os.path.relpath(x,directory) for x in roots] + # print(f"rel roots: {rel_roots}") - # Now exclude __pycache__ - command_dirs = [x for x in rel_roots if '__pycache__' not in x] + # # Now exclude __pycache__ + # command_dirs = [x for x in rel_roots if "__pycache__" not in x] + # print(f"command dirs: {command_dirs}") # Convert slashes to spaces - all_commands = [x.replace('/',' ') for x in command_dirs] - - # Now let's find the commands that have subcommands - ## First we get commands with spaces - commands_with_spaces = [x for x in all_commands if ' ' in x] - ## Now let's just get the first elements - temp = [x.split()[0] for x in commands_with_spaces] - ## Get the uniques - commands_with_subcommands = list(set(temp)) + all_commands_py = glob.glob(os.path.join(directory, "*.py")) + all_commands = [os.path.basename(x).replace(".py", "") for x in all_commands_py] + all_commands.remove("command") # manually remove + + # Now let"s find the commands that have subcommands + ## First we get commands with underscore + commands_with_underscore = [x for x in all_commands if "_" in x] + commands_with_subcommands = [x.replace("_", " ") for x in commands_with_underscore] # Now remove those from our list - all_useful_commands = [x for x in all_commands if x not in commands_with_subcommands] + all_useful_commands = [x for x in all_commands if x not in commands_with_underscore] + all_useful_commands += commands_with_subcommands return sorted(all_useful_commands) def create_markdown_from_usage(command, mdFile): - cmd = [mepo_command_path, command, '--help'] + cmd = [mepo_command_path, command, "--help"] # Some commands have spaces, so we need to break it up again - cmd = ' '.join(cmd).split() + cmd = " ".join(cmd).split() + print(f"command: {cmd}") - result = sp.run(cmd, capture_output=True, universal_newlines=True, env={'COLUMNS':'256'}) + result = sp.run(cmd, capture_output=True, universal_newlines=True, env={"COLUMNS":"256"}) output = result.stdout + print(f"output: {output}") output_list = output.split("\n") + print(f"output list: {output_list}") # Command summary summary = output_list[2] @@ -66,16 +73,16 @@ def create_markdown_from_usage(command, mdFile): # Usage usage = output_list[0] - usage = usage.replace('usage: ','') + usage = usage.replace("usage: ","") mdFile.new_header(level=3, title="Usage") mdFile.insert_code(usage) - positional_arguments = output.partition('positional arguments:\n')[2].partition('\n\n')[0] + positional_arguments = output.partition("positional arguments:\n")[2].partition("\n\n")[0] if positional_arguments: mdFile.new_header(level=3, title="Positional Arguments") mdFile.insert_code(positional_arguments) - optional_arguments = output.partition('optional arguments:\n')[2].partition('\n\n')[0] + optional_arguments = output.partition("optional arguments:\n")[2].partition("\n\n")[0] # Remove extra blank lines optional_arguments = os.linesep.join([s for s in optional_arguments.splitlines() if s]) if optional_arguments: @@ -84,7 +91,7 @@ def create_markdown_from_usage(command, mdFile): if __name__ == "__main__": - doc_file = 'Mepo-Commands.md' + doc_file = "Mepo-Commands.md" mdFile = MdUtils(file_name=doc_file) mdFile.new_header(level=1, title="Overview") @@ -92,10 +99,11 @@ def create_markdown_from_usage(command, mdFile): mdFile.new_header(level=1, title="Commands") command_list = get_command_list(command_dir_path) + print(f"command_list: {command_list}") for command in command_list: mdFile.new_header(level=2, title=command) create_markdown_from_usage(command,mdFile) - mdFile.new_table_of_contents(table_title='Table of Contents', depth=2) + mdFile.new_table_of_contents(table_title="Table of Contents", depth=2) mdFile.create_md_file() - print(f'Generated {doc_file}.') + print(f"Generated {doc_file}.") diff --git a/etc/mepo-cd.bash b/etc/mepo-cd.bash index 1db437c9..53fc33b8 100644 --- a/etc/mepo-cd.bash +++ b/etc/mepo-cd.bash @@ -16,9 +16,9 @@ function mepo-cd () { return 0 fi if [ "$#" -eq 0 ]; then - output=$(mepo whereis _root) + output=$(mepo3 whereis _root) else - output=$(mepo whereis -i $1) + output=$(mepo3 whereis -i $1) fi if [ $? -eq 0 ]; then cd $output diff --git a/etc/mepo-cd.csh b/etc/mepo-cd.csh index b558ce51..7695b4ad 100644 --- a/etc/mepo-cd.csh +++ b/etc/mepo-cd.csh @@ -1 +1 @@ -alias mepo-cd 'cd `mepo whereis -i \!:1`' +alias mepo-cd 'cd `mepo3 whereis -i \!:1`' diff --git a/etc/mepo-cd.zsh b/etc/mepo-cd.zsh index db6c1935..d0e75432 100644 --- a/etc/mepo-cd.zsh +++ b/etc/mepo-cd.zsh @@ -16,9 +16,9 @@ function mepo-cd () { return 0 fi if (( $# == 0 )); then - output=$(mepo whereis _root) + output=$(mepo3 whereis _root) else - output=$(mepo whereis -i $1) + output=$(mepo3 whereis -i $1) fi if [ $? -eq 0 ]; then cd $output diff --git a/pyproject.toml b/pyproject.toml index 0379e921..59da7717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,8 @@ requires = ["setuptools>=61.0"] build-backend = "setuptools.build_meta" [project] -name = "mepo" -version = "v2.0.0" +name = "mepo3" +version = "v1.68.0" authors = [ {name = "GMAO SI Team", email = "siteam@gmao.gsfc.nasa.gov"}, ] @@ -20,7 +20,7 @@ dependencies = [ Homepage = "https://github.com/GEOS-ESM/mepo" [project.scripts] -mepo = "mepo.main:main" +mepo3 = "mepo3.__main__:main" [tool.setuptools] include-package-data = true diff --git a/src/mepo/command/branch/branch.py b/src/mepo/command/branch/branch.py deleted file mode 100644 index 961465c5..00000000 --- a/src/mepo/command/branch/branch.py +++ /dev/null @@ -1,15 +0,0 @@ -import subprocess as sp - -from mepo.state.state import MepoState - -from mepo.command.branch.list import list -from mepo.command.branch.create import create -from mepo.command.branch.delete import delete - -def run(args): - d = { - 'list': list, - 'create': create, - 'delete': delete, - } - d[args.mepo_branch_cmd].run(args) diff --git a/src/mepo/command/config/config.py b/src/mepo/command/config/config.py deleted file mode 100644 index 211b7dad..00000000 --- a/src/mepo/command/config/config.py +++ /dev/null @@ -1,17 +0,0 @@ -import subprocess as sp - -from mepo.state.state import MepoState - -from mepo.command.config.get import get -from mepo.command.config.set import set -from mepo.command.config.delete import delete -from mepo.command.config.print import print - -def run(args): - d = { - 'get': get, - 'set': set, - 'delete': delete, - 'print': print - } - d[args.mepo_config_cmd].run(args) diff --git a/src/mepo/command/init/init.py b/src/mepo/command/init/init.py deleted file mode 100644 index b4ef2126..00000000 --- a/src/mepo/command/init/init.py +++ /dev/null @@ -1,22 +0,0 @@ -from mepo.state.state import MepoState -from mepo.utilities import mepoconfig - -def run(args): - if args.style: - style = args.style - elif mepoconfig.has_option('init','style'): - allowed_styles = ['naked','prefix','postfix'] - style = mepoconfig.get('init','style') - if style not in allowed_styles: - raise Exception(f'Detected style [{style}] from .mepoconfig is not an allowed style: {allowed_styles}') - else: - print(f'Found style [{style}] in .mepoconfig') - else: - style = None - - allcomps = MepoState.initialize(args.config,style) - - if not style: - print(f'Initializing mepo using {args.config}') - else: - print(f'Initializing mepo using {args.config} with {style} style') diff --git a/src/mepo/command/stash/stash.py b/src/mepo/command/stash/stash.py deleted file mode 100644 index fe0ef3eb..00000000 --- a/src/mepo/command/stash/stash.py +++ /dev/null @@ -1,19 +0,0 @@ -import subprocess as sp - -from mepo.state.state import MepoState - -from mepo.command.stash.list import list -from mepo.command.stash.pop import pop -from mepo.command.stash.apply import apply -from mepo.command.stash.push import push -from mepo.command.stash.show import show - -def run(args): - d = { - 'list': list, - 'pop': pop, - 'apply': apply, - 'push': push, - 'show': show, - } - d[args.mepo_stash_cmd].run(args) diff --git a/src/mepo/command/tag/tag.py b/src/mepo/command/tag/tag.py deleted file mode 100644 index 9b02fe89..00000000 --- a/src/mepo/command/tag/tag.py +++ /dev/null @@ -1,17 +0,0 @@ -import subprocess as sp - -from mepo.state.state import MepoState - -from mepo.command.tag.list import list -from mepo.command.tag.create import create -from mepo.command.tag.delete import delete -from mepo.command.tag.push import push - -def run(args): - d = { - 'list': list, - 'create': create, - 'delete': delete, - 'push': push - } - d[args.mepo_tag_cmd].run(args) diff --git a/src/mepo/main.py b/src/mepo/main.py deleted file mode 100644 index 30a31351..00000000 --- a/src/mepo/main.py +++ /dev/null @@ -1,6 +0,0 @@ -from mepo.cmdline.parser import MepoArgParser -from mepo.command import command - -def main(): - args = MepoArgParser().parse() - command.run(args) diff --git a/src/mepo3/__init__.py b/src/mepo3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/mepo3/__main__.py b/src/mepo3/__main__.py new file mode 100644 index 00000000..acad1815 --- /dev/null +++ b/src/mepo3/__main__.py @@ -0,0 +1,11 @@ +from .cmdline.parser import MepoArgParser +from .command import command + + +def main(): + args = MepoArgParser().parse() + command.run(args) + + +if __name__ == "__main__": + main() diff --git a/src/mepo/cmdline/branch_parser.py b/src/mepo3/cmdline/branch_parser.py similarity index 100% rename from src/mepo/cmdline/branch_parser.py rename to src/mepo3/cmdline/branch_parser.py diff --git a/src/mepo/cmdline/config_parser.py b/src/mepo3/cmdline/config_parser.py similarity index 100% rename from src/mepo/cmdline/config_parser.py rename to src/mepo3/cmdline/config_parser.py diff --git a/src/mepo/cmdline/parser.py b/src/mepo3/cmdline/parser.py similarity index 92% rename from src/mepo/cmdline/parser.py rename to src/mepo3/cmdline/parser.py index 237cc04b..3580850b 100644 --- a/src/mepo/cmdline/parser.py +++ b/src/mepo3/cmdline/parser.py @@ -1,10 +1,11 @@ import argparse -from mepo.cmdline.branch_parser import MepoBranchArgParser -from mepo.cmdline.stash_parser import MepoStashArgParser -from mepo.cmdline.tag_parser import MepoTagArgParser -from mepo.cmdline.config_parser import MepoConfigArgParser -from mepo.utilities import mepoconfig +from .branch_parser import MepoBranchArgParser +from .stash_parser import MepoStashArgParser +from .tag_parser import MepoTagArgParser +from .config_parser import MepoConfigArgParser + +from ..utilities import mepoconfig class MepoArgParser(object): @@ -19,7 +20,6 @@ def __init__(self): self.subparsers.dest = 'mepo_cmd' def parse(self): - self.__init() self.__clone() self.__list() self.__status() @@ -46,25 +46,6 @@ def parse(self): self.__config() return self.parser.parse_args() - def __init(self): - init = self.subparsers.add_parser( - 'init', - description = 'Initialize mepo based on `config-file`', - aliases=mepoconfig.get_command_alias('init')) - init.add_argument( - '--config', - metavar = 'config-file', - nargs = '?', - default = 'components.yaml', - help = 'default: %(default)s') - init.add_argument( - '--style', - metavar = 'style-type', - nargs = '?', - default = None, - choices = ['naked', 'prefix','postfix'], - help = 'Style of directory file, default: prefix, allowed options: %(choices)s') - def __clone(self): clone = self.subparsers.add_parser( 'clone', @@ -88,18 +69,18 @@ def __clone(self): default = None, help = 'Branch/tag of URL to initially clone (Only allowed with URL!)') clone.add_argument( - '--config', - metavar = 'config-file', + '--registry', + metavar = 'registry', nargs = '?', - default = None, - help = 'Configuration file (ignored if init already called)') + default = 'components.yaml', + help = 'Registry (default: components.yaml)') clone.add_argument( '--style', metavar = 'style-type', nargs = '?', default = None, choices = ['naked', 'prefix','postfix'], - help = 'Style of directory file, default: prefix, allowed options: %(choices)s (ignored if init already called)') + help = 'Style of directory file, default: prefix, allowed options: %(choices)s') clone.add_argument( '--allrepos', action = 'store_true', @@ -450,11 +431,11 @@ def __push(self): def __save(self): save = self.subparsers.add_parser( 'save', - description = 'Save current state in a yaml config file', + description = 'Save current state in a yaml registry', aliases=mepoconfig.get_command_alias('save')) save.add_argument( - 'config_file', - metavar = 'config-file', + 'registry', + metavar = 'registry', nargs = '?', default = 'components-new.yaml', help = 'default: %(default)s') diff --git a/src/mepo/cmdline/stash_parser.py b/src/mepo3/cmdline/stash_parser.py similarity index 100% rename from src/mepo/cmdline/stash_parser.py rename to src/mepo3/cmdline/stash_parser.py diff --git a/src/mepo/cmdline/tag_parser.py b/src/mepo3/cmdline/tag_parser.py similarity index 100% rename from src/mepo/cmdline/tag_parser.py rename to src/mepo3/cmdline/tag_parser.py diff --git a/src/mepo3/command/branch.py b/src/mepo3/command/branch.py new file mode 100644 index 00000000..4abecb1c --- /dev/null +++ b/src/mepo3/command/branch.py @@ -0,0 +1,11 @@ +from .branch_list import run as branch_list_run +from .branch_create import run as branch_create_run +from .branch_delete import run as branch_delete_run + +def run(args): + d = { + 'list': branch_list_run, + 'create': branch_create_run, + 'delete': branch_delete_run, + } + d[args.mepo_branch_cmd](args) diff --git a/src/mepo/command/branch/create/create.py b/src/mepo3/command/branch_create.py similarity index 62% rename from src/mepo/command/branch/create/create.py rename to src/mepo3/command/branch_create.py index ba17415b..d12afa61 100644 --- a/src/mepo/command/branch/create/create.py +++ b/src/mepo3/command/branch_create.py @@ -1,6 +1,6 @@ -from mepo.state.state import MepoState -from mepo.utilities import verify -from mepo.repository.git import GitRepository +from ..state import MepoState +from ..utilities import verify +from ..git import GitRepository def run(args): allcomps = MepoState.read_state() @@ -9,4 +9,4 @@ def run(args): for comp in comps2crtbr: git = GitRepository(comp.remote, comp.local) git.create_branch(args.branch_name) - print('+ {}: {}'.format(comp.name, args.branch_name)) + print(f"+ {comp.name}: {args.branch_name}") diff --git a/src/mepo/command/branch/delete/delete.py b/src/mepo3/command/branch_delete.py similarity index 75% rename from src/mepo/command/branch/delete/delete.py rename to src/mepo3/command/branch_delete.py index dc60bc92..1011e859 100644 --- a/src/mepo/command/branch/delete/delete.py +++ b/src/mepo3/command/branch_delete.py @@ -1,6 +1,6 @@ -from mepo.state.state import MepoState -from mepo.utilities import verify -from mepo.repository.git import GitRepository +from ..state import MepoState +from ..utilities import verify +from ..git import GitRepository def run(args): allcomps = MepoState.read_state() diff --git a/src/mepo/command/branch/list/list.py b/src/mepo3/command/branch_list.py similarity index 86% rename from src/mepo/command/branch/list/list.py rename to src/mepo3/command/branch_list.py index 16ce794d..b5740720 100644 --- a/src/mepo/command/branch/list/list.py +++ b/src/mepo3/command/branch_list.py @@ -1,6 +1,6 @@ -from mepo.state.state import MepoState -from mepo.utilities import verify -from mepo.repository.git import GitRepository +from ..state import MepoState +from ..utilities import verify +from ..git import GitRepository def run(args): allcomps = MepoState.read_state() diff --git a/src/mepo/command/changed-files/changed-files.py b/src/mepo3/command/changed-files.py similarity index 85% rename from src/mepo/command/changed-files/changed-files.py rename to src/mepo3/command/changed-files.py index 4ae24ae2..a1305c16 100644 --- a/src/mepo/command/changed-files/changed-files.py +++ b/src/mepo3/command/changed-files.py @@ -1,12 +1,14 @@ import os from shutil import get_terminal_size -from mepo.state.state import MepoState -from mepo.state.component import MepoVersion -from mepo.utilities import colors -from mepo.utilities import verify -from mepo.utilities.version import version_to_string, sanitize_version_string -from mepo.repository.git import GitRepository +from ..state import MepoState +from ..component import MepoVersion + +from ..utilities import colors +from ..utilities import verify +from ..utilities.version import version_to_string +from ..utilities.version import sanitize_version_string +from ..git import GitRepository VER_LEN = 30 diff --git a/src/mepo/command/checkout-if-exists/checkout-if-exists.py b/src/mepo3/command/checkout-if-exists.py similarity index 84% rename from src/mepo/command/checkout-if-exists/checkout-if-exists.py rename to src/mepo3/command/checkout-if-exists.py index dfea26c2..767fa879 100644 --- a/src/mepo/command/checkout-if-exists/checkout-if-exists.py +++ b/src/mepo3/command/checkout-if-exists.py @@ -1,7 +1,7 @@ -from mepo.state.state import MepoState -from mepo.utilities import verify -from mepo.repository.git import GitRepository -from mepo.utilities import colors +from ..state import MepoState +from ..utilities import verify +from ..git import GitRepository +from ..utilities import colors def run(args): allcomps = MepoState.read_state() diff --git a/src/mepo/command/checkout/checkout.py b/src/mepo3/command/checkout.py similarity index 88% rename from src/mepo/command/checkout/checkout.py rename to src/mepo3/command/checkout.py index 6e16ba1b..6647dbf7 100644 --- a/src/mepo/command/checkout/checkout.py +++ b/src/mepo3/command/checkout.py @@ -1,7 +1,7 @@ -from mepo.state.state import MepoState -from mepo.utilities import verify -from mepo.repository.git import GitRepository -from mepo.utilities import colors +from ..state import MepoState +from ..utilities import verify +from ..utilities import colors +from ..git import GitRepository def run(args): allcomps = MepoState.read_state() diff --git a/src/mepo/command/clone/clone.py b/src/mepo3/command/clone.py similarity index 58% rename from src/mepo/command/clone/clone.py rename to src/mepo3/command/clone.py index 5f0033a4..d69992da 100644 --- a/src/mepo/command/clone/clone.py +++ b/src/mepo3/command/clone.py @@ -4,10 +4,16 @@ import shlex from urllib.parse import urlparse -from mepo.state.state import MepoState, StateDoesNotExistError -from mepo.repository.git import GitRepository -from mepo.command.init import init as mepo_init -from mepo.utilities import shellcmd, colors, mepoconfig +from ..state import MepoState +from ..state import StateDoesNotExistError +from ..component import MepoComponent +from ..git import GitRepository +from ..utilities import shellcmd +from ..utilities import colors +from ..utilities import mepoconfig +from ..registry import Registry + +MAX_NAMELEN = 15 def run(args): @@ -34,18 +40,6 @@ def run(args): else: partial = None - - # If you pass in a config, with clone, it could be outside the repo. - # So use the full path - passed_in_config = False - if args.config: - passed_in_config = True - args.config = os.path.abspath(args.config) - else: - # If we don't pass in a config, we need to "reset" the arg to the - # default name because we pass args to mepo_init - args.config = 'components.yaml' - if args.repo_url: p = urlparse(args.repo_url) last_url_node = p.path.rsplit('/')[-1] @@ -62,43 +56,10 @@ def run(args): local_clone(args.repo_url,args.branch,git_url_directory,partial) os.chdir(git_url_directory) - # Copy the new file into the repo only if we pass it in - if passed_in_config: - try: - shutil.copy(args.config,os.getcwd()) - except shutil.SameFileError as e: - pass - - # This tries to read the state and if not, calls init, - # loops back, and reads the state - while True: - try: - allcomps = MepoState.read_state() - except StateDoesNotExistError: - mepo_init.run(args) - continue - break - - max_namelen = len(max([comp.name for comp in allcomps], key=len)) - for comp in allcomps: - if not comp.fixture: - git = GitRepository(comp.remote, comp.local) - version = comp.version.name - version = version.replace('origin/','') - recurse = comp.recurse_submodules - - # According to Git, treeless clones do not interact well with - # submodules. So we need to see if any comp has the recurse - # option set to True. If so, we need to clone that comp "normally" - - _partial = None if partial == 'treeless' and recurse else partial - - # We need the type to handle hashes in components.yaml - type = comp.version.type - git.clone(version,recurse,type,comp.name,_partial) - if comp.sparse: - git.sparsify(comp.sparse) - print_clone_info(comp, max_namelen) + root_component_dir = os.path.dirname(os.path.abspath(args.registry)) + all_components = list() + __recursive_clone(root_component_dir, all_components, partial) + MepoState().write_state(all_components) if args.allrepos: for comp in allcomps: @@ -109,9 +70,35 @@ def run(args): colors.RESET + comp.name + colors.RESET)) git.checkout(args.branch,detach=True) -def print_clone_info(comp, name_width): +def __recursive_clone(local_path, complist, partial): + registry = os.path.join(local_path, "components.yaml") + if os.path.isfile(registry): + complist_dict_from_file = Registry(registry).read_file() + for name, details in complist_dict_from_file.items(): + if "local" in details: # update local path of component + details["local"] = os.path.join(local_path, details["local"]) + comp = MepoComponent().to_component(name, details, None) + complist.append(comp) + if not comp.fixture: + git = GitRepository(comp.remote, os.path.join(local_path, comp.local)) + version = comp.version.name + recurse_submodules = comp.recurse_submodules + # According to Git, treeless clones do not interact well with + # submodules. So we need to see if any comp has the recurse + # option set to True. If so, we need to clone that comp "normally" + # TODO: Add 'partial' abilities + _partial = None if partial == 'treeless' and recurse else partial + # We need the type to handle hashes in components.yaml + _type = comp.version.type + git.clone(version, recurse_submodules, _type, comp.name, _partial) + if comp.sparse: + git.sparsify(comp.sparse) + __print_clone_info(comp) + __recursive_clone(os.path.join(local_path, comp.local), complist, partial) + +def __print_clone_info(comp): ver_name_type = '({}) {}'.format(comp.version.type, comp.version.name) - print('{:<{width}} | {: 1: - raise Exception("Only one fixture allowed") - - complist.append(MepoComponent().to_component(name, comp, directory_style)) - cls.write_state(complist) - @classmethod def read_state(cls): if not cls.exists(): - raise StateDoesNotExistError('Error! mepo state does not exist') + raise StateDoesNotExistError('Error! mepo3 state does not exist') with open(cls.get_file(), 'rb') as fin: allcomps = pickle.load(fin) return allcomps diff --git a/src/mepo/utilities/colors.py b/src/mepo3/utilities/colors.py similarity index 100% rename from src/mepo/utilities/colors.py rename to src/mepo3/utilities/colors.py diff --git a/src/mepo/state/exceptions.py b/src/mepo3/utilities/exceptions.py similarity index 76% rename from src/mepo/state/exceptions.py rename to src/mepo3/utilities/exceptions.py index a6e0aa04..888e01fe 100644 --- a/src/mepo/state/exceptions.py +++ b/src/mepo3/utilities/exceptions.py @@ -10,12 +10,12 @@ class RepoAlreadyClonedError(SystemExit): """Raised when the repository has already been cloned""" pass -class ConfigFileNotFoundError(FileNotFoundError): - """Raised when the config file is not found""" +class RegistryNotFoundError(FileNotFoundError): + """Raised when the registry is not found""" pass class SuffixNotRecognizedError(RuntimeError): - """Raised when the config suffix is not recognized""" + """Raised when the registry suffix is not recognized""" pass class NotInRootDirError(SystemExit): diff --git a/src/mepo/utilities/mepoconfig.py b/src/mepo3/utilities/mepoconfig.py similarity index 100% rename from src/mepo/utilities/mepoconfig.py rename to src/mepo3/utilities/mepoconfig.py diff --git a/src/mepo/utilities/shellcmd.py b/src/mepo3/utilities/shellcmd.py similarity index 100% rename from src/mepo/utilities/shellcmd.py rename to src/mepo3/utilities/shellcmd.py diff --git a/src/mepo/utilities/verify.py b/src/mepo3/utilities/verify.py similarity index 100% rename from src/mepo/utilities/verify.py rename to src/mepo3/utilities/verify.py diff --git a/src/mepo/utilities/version.py b/src/mepo3/utilities/version.py similarity index 100% rename from src/mepo/utilities/version.py rename to src/mepo3/utilities/version.py diff --git a/tests/output/compare_brief_output.txt b/tests/output/compare_brief_output.txt index f7f32d05..efd00132 100644 --- a/tests/output/compare_brief_output.txt +++ b/tests/output/compare_brief_output.txt @@ -1,5 +1,5 @@ -Repo | Original | Current --------- | -------------------- | ------- -env | (t) v4.8.0 (DH) | (b) main -cmake | (t) v3.21.0 (DH) | (b) develop -fvdycore | (t) geos/v1.5.0 (DH) | (b) geos/develop +Repo | Original | Current +-------- | -------------------------------- | ------- +env | (t) v4.25.1 (DH) | (b) main +FMS | (t) geos/2019.01.02+noaff.8 (DH) | (b) geos/release/2019.01 +fvdycore | (t) geos/v2.8.1 (DH) | (b) geos/develop diff --git a/tests/output/compare_full_output.txt b/tests/output/compare_full_output.txt index 02aaa0b2..9b70724c 100644 --- a/tests/output/compare_full_output.txt +++ b/tests/output/compare_full_output.txt @@ -1,11 +1,12 @@ -Repo | Original | Current ----------------------- | -------------------------------- | ------- -GEOSfvdycore | (t) v1.13.0 (DH) | (t) v1.13.0 (DH) -env | (t) v4.8.0 (DH) | (b) main -cmake | (t) v3.21.0 (DH) | (b) develop -ecbuild | (t) geos/v1.3.0 (DH) | (t) geos/v1.3.0 (DH) -GMAO_Shared | (t) v1.6.3 (DH) | (t) v1.6.3 (DH) -MAPL | (t) v2.33.0 (DH) | (t) v2.33.0 (DH) -FMS | (t) geos/2019.01.02+noaff.8 (DH) | (t) geos/2019.01.02+noaff.8 (DH) -FVdycoreCubed_GridComp | (t) v1.12.1 (DH) | (t) v1.12.1 (DH) -fvdycore | (t) geos/v1.5.0 (DH) | (b) geos/develop +Repo | Original | Current +---------------------- | ---------------------------------------------------- | ------- +GEOSfvdycore | (b) main | (b) main +env | (t) v4.25.1 (DH) | (b) main +cmake | (b) feature/pchakrab/hierarchical-mepo (DH, b01dd9a) | (b) feature/pchakrab/hierarchical-mepo (DH, b01dd9a) +ecbuild | (t) geos/v1.3.0 (DH) | (t) geos/v1.3.0 (DH) +GMAO_Shared | (b) feature/pchakrab/hierarchical-mepo (DH, 4faf224) | (b) feature/pchakrab/hierarchical-mepo (DH, 4faf224) +GEOS_Util | (t) v2.0.5 (DH) | (t) v2.0.5 (DH) +MAPL | (b) feature/pchakrab/hierarchical-mepo (DH, 7b57017) | (b) feature/pchakrab/hierarchical-mepo (DH, 7b57017) +FMS | (t) geos/2019.01.02+noaff.8 (DH) | (b) geos/release/2019.01 +FVdycoreCubed_GridComp | (b) feature/pchakrab/hierarchical-mepo (DH, 92dbd83) | (b) feature/pchakrab/hierarchical-mepo (DH, 92dbd83) +fvdycore | (t) geos/v2.8.1 (DH) | (b) geos/develop diff --git a/tests/output/list_output.txt b/tests/output/list_output.txt index 608fc124..7ce9c2bb 100644 --- a/tests/output/list_output.txt +++ b/tests/output/list_output.txt @@ -1 +1 @@ -GEOSfvdycore env cmake ecbuild GMAO_Shared MAPL FMS FVdycoreCubed_GridComp fvdycore +GEOSfvdycore env cmake ecbuild GMAO_Shared GEOS_Util MAPL FMS FVdycoreCubed_GridComp fvdycore diff --git a/tests/output/status_output.txt b/tests/output/status_output.txt index 1b4a1566..a46fdaa8 100644 --- a/tests/output/status_output.txt +++ b/tests/output/status_output.txt @@ -1,10 +1,11 @@ Checking status... -GEOSfvdycore | (t) v1.13.0 (DH) +GEOSfvdycore | (b) main env | (b) main -cmake | (b) develop +cmake | (b) feature/pchakrab/hierarchical-mepo (DH, b01dd9a) ecbuild | (t) geos/v1.3.0 (DH) -GMAO_Shared | (t) v1.6.3 (DH) -MAPL | (t) v2.33.0 (DH) -FMS | (t) geos/2019.01.02+noaff.8 (DH) -FVdycoreCubed_GridComp | (t) v1.12.1 (DH) +GMAO_Shared | (b) feature/pchakrab/hierarchical-mepo (DH, 4faf224) +GEOS_Util | (t) v2.0.5 (DH) +MAPL | (b) feature/pchakrab/hierarchical-mepo (DH, 7b57017) +FMS | (b) geos/release/2019.01 +FVdycoreCubed_GridComp | (b) feature/pchakrab/hierarchical-mepo (DH, 92dbd83) fvdycore | (b) geos/develop diff --git a/tests/test_mepo_commands.py b/tests/test_mepo_commands.py index dbdd88ef..4f9a4db2 100644 --- a/tests/test_mepo_commands.py +++ b/tests/test_mepo_commands.py @@ -1,7 +1,7 @@ import os import sys THIS_DIR = os.path.dirname(os.path.realpath(__file__)) -sys.path.insert(0, os.path.join(THIS_DIR, '..', 'src')) +sys.path.insert(0, os.path.join(THIS_DIR, "..", "src")) import shutil import shlex import unittest @@ -10,12 +10,11 @@ from input import args -from mepo.command.init import init as mepo_init -from mepo.command.clone import clone as mepo_clone -from mepo.command.list import list as mepo_list -from mepo.command.status import status as mepo_status -from mepo.command.compare import compare as mepo_compare -from mepo.command.develop import develop as mepo_develop +from mepo3.command.clone import run as mepo_clone +from mepo3.command.list import run as mepo_list +from mepo3.command.status import run as mepo_status +from mepo3.command.compare import run as mepo_compare +from mepo3.command.develop import run as mepo_develop class TestMepoCommands(unittest.TestCase): @@ -23,51 +22,53 @@ class TestMepoCommands(unittest.TestCase): @classmethod def __checkout_fixture(cls): - remote = 'https://github.com/GEOS-ESM/{}.git'.format(cls.fixture) - cmd = 'git clone -b {} {} {}'.format(cls.tag, remote, cls.fixture_dir) + remote = f"https://github.com/pchakraborty/{cls.fixture}.git" + git_clone = "git clone " + if cls.tag: + git_clone += f"-b {cls.tag}" + cmd = f"{git_clone} {remote} {cls.fixture_dir}" + print(f"clone command: {cmd}") sp.run(shlex.split(cmd)) @classmethod def __copy_config_file(cls): - src = os.path.join(cls.input_dir, 'components.yaml') + src = os.path.join(cls.input_dir, "components.yaml") dst = os.path.join(cls.fixture_dir) shutil.copy(src, dst) @classmethod def setUpClass(cls): - cls.input_dir = os.path.join(THIS_DIR, 'input') - cls.output_dir = os.path.join(THIS_DIR, 'output') - cls.fixture = 'GEOSfvdycore' - cls.tag = 'v1.13.0' - cls.tmpdir = os.path.join(THIS_DIR, 'tmp') + cls.input_dir = os.path.join(THIS_DIR, "input") + cls.output_dir = os.path.join(THIS_DIR, "output") + cls.fixture = "GEOSfvdycore" + cls.tag = None + cls.tmpdir = os.path.join(THIS_DIR, "tmp") cls.fixture_dir = os.path.join(cls.tmpdir, cls.fixture) if os.path.isdir(cls.fixture_dir): shutil.rmtree(cls.fixture_dir) cls.__checkout_fixture() #cls.__copy_config_file() - args.config = 'components.yaml' - args.style = 'prefix' + args.style = "prefix" os.chdir(cls.fixture_dir) - mepo_init.run(args) - args.config = None args.repo_url = None args.branch = None args.directory = None - args.partial = 'blobless' - mepo_clone.run(args) + args.partial = "blobless" + args.registry = "components.yaml" + mepo_clone(args) # In order to better test compare, we need to do *something* - args.comp_name = ['env','cmake','fvdycore'] + args.comp_name = ["env","FMS","fvdycore"] args.quiet = False - mepo_develop.run(args) + mepo_develop(args) def setUp(self): pass def test_list(self): sys.stdout = output = StringIO() - mepo_list.run(args) + mepo_list(args) sys.stdout = sys.__stdout__ - with open(os.path.join(self.__class__.output_dir, 'list_output.txt'), 'r') as fin: + with open(os.path.join(self.__class__.output_dir, "list_output.txt"), "r") as fin: saved_output = fin.read() self.assertEqual(output.getvalue(), saved_output) @@ -76,9 +77,9 @@ def test_status(self): args.ignore_permissions=False args.nocolor=True args.hashes=False - mepo_status.run(args) + mepo_status(args) sys.stdout = sys.__stdout__ - with open(os.path.join(self.__class__.output_dir, 'status_output.txt'), 'r') as fin: + with open(os.path.join(self.__class__.output_dir, "status_output.txt"), "r") as fin: saved_output = fin.read() self.assertEqual(output.getvalue(), saved_output) @@ -87,9 +88,9 @@ def test_compare_brief(self): args.all=False args.nocolor=True args.wrap=True - mepo_compare.run(args) + mepo_compare(args) sys.stdout = sys.__stdout__ - with open(os.path.join(self.__class__.output_dir, 'compare_brief_output.txt'), 'r') as fin: + with open(os.path.join(self.__class__.output_dir, "compare_brief_output.txt"), "r") as fin: saved_output = fin.read() self.assertEqual(output.getvalue(), saved_output) @@ -98,9 +99,9 @@ def test_compare_full(self): args.all=True args.nocolor=True args.wrap=True - mepo_compare.run(args) + mepo_compare(args) sys.stdout = sys.__stdout__ - with open(os.path.join(self.__class__.output_dir, 'compare_full_output.txt'), 'r') as fin: + with open(os.path.join(self.__class__.output_dir, "compare_full_output.txt"), "r") as fin: saved_output = fin.read() self.assertEqual(output.getvalue(), saved_output) @@ -112,5 +113,5 @@ def tearDownClass(cls): os.chdir(THIS_DIR) shutil.rmtree(cls.tmpdir) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main()