diff --git a/.coveragerc b/.coveragerc index 4c62180950..04ab2270b8 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,3 +3,4 @@ relative_files = True source = metplus omit = metplus/wrappers/cyclone_plotter_wrapper.py + metplus/produtil/** \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/documentation.md b/.github/ISSUE_TEMPLATE/documentation.md new file mode 100644 index 0000000000..b93c8438f4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.md @@ -0,0 +1,68 @@ +--- +name: Documentation +about: Update the documentation +title: 'Documentation: ' +labels: 'alert: NEED ACCOUNT KEY, alert: NEED MORE DEFINITION, alert: NEED CYCLE ASSIGNMENT, type: documentation' +assignees: '' + +--- + +*Replace italics below with details for this issue.* + +## Describe the Task ## +*Provide a description of the task here.* + +### Time Estimate ### +*Estimate the amount of work required here.* +*Issues should represent approximately 1 to 3 days of work.* + +### Sub-Issues ### +Consider breaking the task down into sub-issues. +- [ ] *Add a checkbox for each sub-issue here.* + +### Relevant Deadlines ### +*List relevant project deadlines here or state NONE.* + +### Funding Source ### +*Define the source of funding and account keys here or state NONE.* + +## Define the Metadata ## + +### Assignee ### +- [ ] Select appropriate **assignee** for this issue + +### Labels ### +- [ ] Review default **alert** labels +- [ ] Select **component(s)** +- [ ] Select **priority** +- [ ] Select **requestor(s)** + +### Milestone and Projects ### +- [ ] Select **Milestone** as a **METplus-Wrappers-X.Y.Z** version, **Consider for Next Release**, or **Backlog of Development Ideas** +- [ ] For a **METplus-Wrappers-X.Y.Z** version, select the **METplus-Wrappers-X.Y.Z Development** project + +## Define Related Issue(s) ## +Consider the impact to the other METplus components. +- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose) + +## Task Checklist ## +See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details. +- [ ] Complete the issue definition above, including the **Time Estimate** and **Funding Source**. +- [ ] Fork this repository or create a branch of **develop**. +Branch name: `feature__` +- [ ] Complete the development and test your changes. +- [ ] Add/update log messages for easier debugging. +- [ ] Add/update unit tests. +- [ ] Add/update documentation. +- [ ] Add any new Python packages to the [METplus Components Python Requirements](https://metplus.readthedocs.io/en/develop/Users_Guide/appendixA.html#metplus-components-python-packages) table. +- [ ] For any new datasets, an entry to the [METplus Verification Datasets Guide](https://metplus.readthedocs.io/en/latest/Verification_Datasets/index.html). +- [ ] Push local changes to GitHub. +- [ ] Submit a pull request to merge into **develop**. +Pull request: `feature ` +- [ ] Define the pull request metadata, as permissions allow. +Select: **Reviewer(s)** and **Development** issue +Select: **Milestone** as the next official version +Select: **METplus-Wrappers-X.Y.Z Development** project for development toward the next official release +- [ ] Iterate until the reviewer(s) accept and merge your changes. +- [ ] Delete your fork or branch. +- [ ] Close this issue. diff --git a/.github/ISSUE_TEMPLATE/enhancement_request.md b/.github/ISSUE_TEMPLATE/enhancement_request.md index 479825cabc..d90bbf44de 100644 --- a/.github/ISSUE_TEMPLATE/enhancement_request.md +++ b/.github/ISSUE_TEMPLATE/enhancement_request.md @@ -1,7 +1,7 @@ --- name: Enhancement request about: Improve something that it's currently doing -title: '' +title: 'Enhancement: ' labels: 'alert: NEED ACCOUNT KEY, alert: NEED MORE DEFINITION, alert: NEED CYCLE ASSIGNMENT, type: enhancement' assignees: '' diff --git a/.github/jobs/docker_build_metplus_images.sh b/.github/jobs/docker_build_metplus_images.sh new file mode 100755 index 0000000000..e0f6770017 --- /dev/null +++ b/.github/jobs/docker_build_metplus_images.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# assumes SOURCE_BRANCH is set before calling script + +source "${GITHUB_WORKSPACE}"/.github/jobs/bash_functions.sh + +dockerhub_repo=dtcenter/metplus +dockerhub_repo_analysis=dtcenter/metplus-analysis + +# check if tag is official or bugfix release -- no -betaN or -rcN suffix +is_official=1 +if [[ "${SOURCE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + is_official=0 +fi + +# remove v prefix +metplus_version=${SOURCE_BRANCH:1} + +# if rc is in version number, get main_vX.Y, otherwise get X.Y-latest or develop +if [[ "${metplus_version}" =~ rc ]]; then + tag_format="main_v{X}.{Y}" +else + tag_format="{X}.{Y}-latest" +fi + +# Get MET tag and adjust MET Docker repo if develop +met_tag=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f ${tag_format} --no-get_dev_version) +echo "$met_tag" + +MET_DOCKER_REPO=met +if [ "$met_tag" == "develop" ] || [[ "${met_tag}" =~ ^main_v[0-9]+\.[0-9]+ ]]; then + MET_DOCKER_REPO=met-dev +fi + +# get METplus Analysis tool versions +METDATAIO_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METdataio) +METCALCPY_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METcalcpy) +METPLOTPY_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METplotpy) + +# Build metplus image +METPLUS_IMAGE_NAME=${dockerhub_repo}:${metplus_version} +if ! time_command docker build -t "$METPLUS_IMAGE_NAME" \ + --build-arg SOURCE_VERSION="$SOURCE_BRANCH" \ + --build-arg MET_TAG="$met_tag" \ + --build-arg MET_DOCKER_REPO="$MET_DOCKER_REPO" \ + -f "${GITHUB_WORKSPACE}"/internal/scripts/docker/Dockerfile \ + "${GITHUB_WORKSPACE}"; then + exit 1 +fi + +# Build metplus-analysis image +METPLUS_A_IMAGE_NAME=${dockerhub_repo_analysis}:${metplus_version} +if ! time_command docker build -t "$METPLUS_A_IMAGE_NAME" \ + --build-arg METPLUS_BASE_TAG="${metplus_version}" \ + --build-arg METDATAIO_VERSION="${METDATAIO_VERSION}" \ + --build-arg METCALCPY_VERSION="${METCALCPY_VERSION}" \ + --build-arg METPLOTPY_VERSION="${METPLOTPY_VERSION}" \ + -f "${GITHUB_WORKSPACE}"/internal/scripts/docker/Dockerfile.metplus-analysis \ + "${GITHUB_WORKSPACE}"; then + exit 1 +fi + +# if official release, create X.Y-latest tag as well +if [ "${is_official}" == 0 ]; then + LATEST_TAG=$(echo "$metplus_version" | cut -f1,2 -d'.')-latest + docker tag "${METPLUS_IMAGE_NAME}" "${dockerhub_repo}:${LATEST_TAG}" + docker tag "${METPLUS_A_IMAGE_NAME}" "${dockerhub_repo_analysis}:${LATEST_TAG}" + echo LATEST_TAG="${LATEST_TAG}" >> "$GITHUB_OUTPUT" +fi diff --git a/.github/jobs/docker_push_metplus_images.sh b/.github/jobs/docker_push_metplus_images.sh new file mode 100755 index 0000000000..1ae5e87378 --- /dev/null +++ b/.github/jobs/docker_push_metplus_images.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# assumes SOURCE_BRANCH is set before calling script +# assumes latest_tag will be set if pushing an official or bugfix release + +source "${GITHUB_WORKSPACE}"/.github/jobs/bash_functions.sh + +# get names of images to push + +dockerhub_repo=dtcenter/metplus +dockerhub_repo_analysis=dtcenter/metplus-analysis + +# remove v prefix +metplus_version=${SOURCE_BRANCH:1} + +METPLUS_IMAGE_NAME=${dockerhub_repo}:${metplus_version} +METPLUS_A_IMAGE_NAME=${dockerhub_repo_analysis}:${metplus_version} + +# skip docker push if credentials are not set +if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then + echo "DockerHub credentials not set. Skipping docker push" + exit 0 +fi + +echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin + +# push images + +if ! time_command docker push "${METPLUS_IMAGE_NAME}"; then + exit 1 +fi + +if ! time_command docker push "${METPLUS_A_IMAGE_NAME}"; then + exit 1 +fi + +# only push X.Y-latest tag if official or bugfix release +# shellcheck disable=SC2154 +if [ "${LATEST_TAG}" != "" ]; then + if ! time_command docker push "${dockerhub_repo}:${LATEST_TAG}"; then + exit 1 + fi + if ! time_command docker push "${dockerhub_repo_analysis}:${LATEST_TAG}"; then + exit 1 + fi +fi diff --git a/.github/jobs/docker_setup.sh b/.github/jobs/docker_setup.sh index 8cab9ad44d..11568cd178 100755 --- a/.github/jobs/docker_setup.sh +++ b/.github/jobs/docker_setup.sh @@ -31,10 +31,19 @@ echo "TIMING: docker pull ${DOCKERHUB_TAG} took `printf '%02d' $(($duration / 60 # set DOCKERFILE_PATH that is used by docker hook script get_met_version export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile -MET_TAG=`${GITHUB_WORKSPACE}/internal/scripts/docker/hooks/get_met_version` +metplus_version=$(head -n 1 "${GITHUB_WORKSPACE}/metplus/VERSION") + +# if rc is in version number, get main_vX.Y, otherwise get X.Y-latest or develop +if [[ "${metplus_version}" =~ rc ]]; then + tag_format="main_v{X}.{Y}" +else + tag_format="{X}.{Y}-latest" +fi + +MET_TAG=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f ${tag_format} --no-get_dev_version) MET_DOCKER_REPO=met-dev -if [ "${MET_TAG}" != "develop" ]; then +if [ "${MET_TAG}" != "develop" ] && ! [[ "${MET_TAG}" =~ ^main_v[0-9]+\.[0-9]+ ]]; then MET_DOCKER_REPO=met elif [ "${EXTERNAL_TRIGGER}" == "true" ]; then # if MET tag is develop and external repo triggered workflow diff --git a/.github/jobs/docker_utils.py b/.github/jobs/docker_utils.py index edf0b00167..65c412456f 100644 --- a/.github/jobs/docker_utils.py +++ b/.github/jobs/docker_utils.py @@ -39,10 +39,13 @@ def get_dockerhub_url(branch_name): def docker_get_volumes_last_updated(current_branch): import requests dockerhub_url = get_dockerhub_url(current_branch) - dockerhub_request = requests.get(dockerhub_url, timeout=60) + dockerhub_request = requests.get(dockerhub_url, timeout=90) if dockerhub_request.status_code != 200: - print(f"Could not find DockerHub URL: {dockerhub_url}") - return None + print(f"Retrying DockerHub request: {dockerhub_url}") + dockerhub_request = requests.get(dockerhub_url, timeout=60) + if dockerhub_request.status_code != 200: + print(f"Could not query DockerHub URL: {dockerhub_url}") + return None # get version number to search for if main_vX.Y branch if current_branch.startswith('main_v'): diff --git a/.github/jobs/get_data_volumes.py b/.github/jobs/get_data_volumes.py index e492711710..cbf3a4e01d 100755 --- a/.github/jobs/get_data_volumes.py +++ b/.github/jobs/get_data_volumes.py @@ -7,8 +7,6 @@ import sys import os -import subprocess -import shlex from docker_utils import docker_get_volumes_last_updated, get_branch_name from docker_utils import get_data_repo, DOCKERHUB_METPLUS_DATA_DEV @@ -54,7 +52,12 @@ def main(args): branch_name = branch_name[5:] # get all docker data volumes associated with current branch - available_volumes = docker_get_volumes_last_updated(branch_name).keys() + try: + available_volumes = docker_get_volumes_last_updated(branch_name).keys() + except AttributeError: + print("ERROR: Could not get available Docker data volumes " + f"for {branch_name}. Try rerunning failed jobs in GitHub Actions") + return None # loop through arguments and get data volume for each category for model_app_name in args: @@ -78,7 +81,7 @@ def main(args): volume_name = f'{prefix}-{model_app_name}' # add output- to model app name - model_app_name=f'output-{model_app_name}' + model_app_name = f'output-{model_app_name}' # set DockerHub repo to dev version because all output data # should be in dev repository @@ -108,11 +111,11 @@ def main(args): if __name__ == "__main__": # split up command line args that have commas before passing into main - args = [] + arg_list = [] for arg in sys.argv[1:]: - args.extend(arg.split(',')) - out = main(args) + arg_list.extend(arg.split(',')) + out = main(arg_list) if out is None: print("ERROR: Something went wrong") sys.exit(1) diff --git a/.github/jobs/get_use_case_commands.py b/.github/jobs/get_use_case_commands.py index 7212019889..1c69828aa4 100755 --- a/.github/jobs/get_use_case_commands.py +++ b/.github/jobs/get_use_case_commands.py @@ -17,6 +17,7 @@ from metplus.util.string_manip import expand_int_string_to_list from docker_utils import VERSION_EXT from metplus import get_metplus_version +from metplus.component_versions import get_component_version # path to METplus install location in Docker METPLUS_DOCKER_LOC = '/metplus/METplus' @@ -39,7 +40,7 @@ ] -def handle_automation_env(host_name, reqs, work_dir): +def handle_automation_env(host_name, reqs): # if no env is specified, use metplus base environment conda_env = METPLUS_BASE_ENV @@ -60,8 +61,9 @@ def handle_automation_env(host_name, reqs, work_dir): conda_env_w_ext = f'{conda_env}{VERSION_EXT}' # start building commands to run before run_metplus.py in Docker - setup_env = [] - setup_env.append(_add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT')) + setup_env = [ + _add_to_bashrc('# BELOW WAS ADDED BY TEST SCRIPT') + ] # add conda bin to beginning of PATH python_dir = os.path.join('/usr', 'local', 'conda', 'envs', @@ -76,35 +78,34 @@ def handle_automation_env(host_name, reqs, work_dir): else: py_embed_arg = '' - # get METplus version to determine Externals file to use - # to get METplotpy/METcalcpy/METdataio + # get METplus version to determine which version of + # METplotpy/METcalcpy/METdataio to use # If stable release, get main branch, otherwise get develop - is_stable_release = len(get_metplus_version().split('-')) == 1 - externals_ext = '_stable.cfg' if is_stable_release else '.cfg' + metplus_version = get_metplus_version() # if any metplotpy/metcalcpy keywords are in requirements list, # add command to obtain and install METplotpy and METcalcpy + components = [] if any([item for item in PLOTCALC_KEYWORDS if item in str(reqs).lower()]): - ce_file = os.path.join(work_dir, '.github', 'parm', - f'Externals_metplotcalcpy{externals_ext}') - setup_env.extend(( - f'cd {METPLUS_DOCKER_LOC}', - f'{work_dir}/manage_externals/checkout_externals -e {ce_file}', - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METplotpy', - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METcalcpy', - 'cd -', - )) + components.extend(('METplotpy', 'METcalcpy')) # if metdataio is in requirements list, add command to obtain METdataio if 'metdataio' in str(reqs).lower(): - ce_file = os.path.join(work_dir, '.github', 'parm', - f'Externals_metdataio{externals_ext}') + components.append('METdataio') + + setup_env.append(f'cd {METPLUS_DOCKER_LOC}/..') + for component in components: + version = get_component_version(input_component='METplus', + input_version=metplus_version, + output_component=component, + output_format='main_v{X}.{Y}', + get_dev=False) setup_env.extend(( - f'cd {METPLUS_DOCKER_LOC}', - f'{work_dir}/manage_externals/checkout_externals -e {ce_file}', - f'{python_path} -m pip install {METPLUS_DOCKER_LOC}/../METdataio', - 'cd -', + 'git --version', + f'git clone --single-branch --branch {version} https://github.com/dtcenter/{component}', + f'{python_path} -m pip install --no-deps {METPLUS_DOCKER_LOC}/../{component}', )) + setup_env.append('cd -') # if metplus is in requirements list, # add top of METplus repo to PYTHONPATH so metplus can be imported @@ -118,7 +119,7 @@ def handle_automation_env(host_name, reqs, work_dir): setup_env.extend(( f'echo Using environment: dtcenter/metplus-envs:{conda_env_w_ext}', f'echo cat /usr/local/conda/envs/{conda_env_w_ext}/environments.yml', - f'echo ----------------------------------------', + 'echo ----------------------------------------', f'cat /usr/local/conda/envs/{conda_env_w_ext}/environments.yml', 'echo ----------------------------------------', )) @@ -140,6 +141,24 @@ def main(categories, subset_list, work_dir=None, test_suite = METplusUseCaseSuite() test_suite.add_use_case_groups(categories, subset_list) + for group_name, use_cases_by_req in test_suite.category_groups.items(): + for use_case_by_requirement in use_cases_by_req: + reqs = use_case_by_requirement.requirements + + setup_env, py_embed_arg = handle_automation_env(host_name, reqs) + + use_case_cmds = _get_use_case_cmds(host_name, use_case_by_requirement, work_dir, group_name, py_embed_arg) + + # add commands to set up environment before use case commands + all_commands.append((setup_env, use_case_cmds, reqs)) + + return all_commands + + +def _get_use_case_cmds(host_name, use_case_by_requirement, work_dir, group_name, py_embed_arg): + # use status variable to track if any use cases failed + use_case_cmds = [] + output_top_dir = os.environ.get('METPLUS_TEST_OUTPUT_BASE', '/data/output') # use METPLUS_TEST_SETTINGS_CONF if set @@ -150,58 +169,53 @@ def main(categories, subset_list, work_dir=None, 'parm', 'test_settings.conf') - for group_name, use_cases_by_req in test_suite.category_groups.items(): - for use_case_by_requirement in use_cases_by_req: - reqs = use_case_by_requirement.requirements + if host_name != 'docker': + use_case_cmds.append('status=0') + for use_case in use_case_by_requirement.use_cases: + # add parm/use_cases path to config args if they are conf files + config_args = _get_config_args(use_case.config_args, work_dir, host_name) + + output_base = os.path.join(output_top_dir, + group_name.split('-')[0], + use_case.name) + use_case_cmd = (f"run_metplus.py" + f" {' '.join(config_args)}" + f" {py_embed_arg}{test_settings_conf}" + f" config.OUTPUT_BASE={output_base}") + use_case_cmds.append(use_case_cmd) + # check exit code from use case command and + # set status to non-zero value on error + if host_name != 'docker': + use_case_cmds.append("if [ $? != 0 ]; then status=1; fi") + + # if any use cases failed, force non-zero exit code with false + if host_name != 'docker': + use_case_cmds.append("if [ $status != 0 ]; then false; fi") - setup_env, py_embed_arg = handle_automation_env(host_name, reqs, - work_dir) - - # use status variable to track if any use cases failed - use_case_cmds = [] - if host_name != 'docker': - use_case_cmds.append('status=0') - for use_case in use_case_by_requirement.use_cases: - # add parm/use_cases path to config args if they are conf files - config_args = [] - ci_overrides = None - for config_arg in use_case.config_args: - if config_arg.endswith('.conf'): - config_arg = os.path.join(work_dir, 'parm', - 'use_cases', config_arg) - - # look for CI overrides conf file - override_path = os.path.join(config_arg[0:-5], - 'ci_overrides.conf') - if os.path.exists(override_path): - ci_overrides = override_path - - config_args.append(config_arg) - - # add CI overrides config file if running in docker - if ci_overrides and host_name == 'docker': - config_args.append(ci_overrides) - - output_base = os.path.join(output_top_dir, - group_name.split('-')[0], - use_case.name) - use_case_cmd = (f"run_metplus.py" - f" {' '.join(config_args)}" - f" {py_embed_arg}{test_settings_conf}" - f" config.OUTPUT_BASE={output_base}") - use_case_cmds.append(use_case_cmd) - # check exit code from use case command and - # set status to non-zero value on error - if host_name != 'docker': - use_case_cmds.append("if [ $? != 0 ]; then status=1; fi") - - # if any use cases failed, force non-zero exit code with false - if host_name != 'docker': - use_case_cmds.append("if [ $status != 0 ]; then false; fi") - # add commands to set up environment before use case commands - all_commands.append((setup_env, use_case_cmds, reqs)) + return use_case_cmds - return all_commands + +def _get_config_args(input_config_args, work_dir, host_name): + config_args = [] + ci_overrides = None + for config_arg in input_config_args: + if config_arg.endswith('.conf'): + config_arg = os.path.join(work_dir, 'parm', + 'use_cases', config_arg) + + # look for CI overrides conf file + override_path = os.path.join(config_arg[0:-5], + 'ci_overrides.conf') + if os.path.exists(override_path): + ci_overrides = override_path + + config_args.append(config_arg) + + # add CI overrides config file if running in docker + if ci_overrides and host_name == 'docker': + config_args.append(ci_overrides) + + return config_args def handle_command_line_args(): @@ -232,9 +246,9 @@ def handle_command_line_args(): if __name__ == '__main__': - categories, subset_list, _ = handle_command_line_args() - all_commands = main(categories, subset_list) - for setup_commands, use_case_commands, requirements in all_commands: + input_categories, input_subset_list, _ = handle_command_line_args() + commands = main(input_categories, input_subset_list) + for setup_commands, use_case_commands, requirements in commands: print(f"REQUIREMENTS: {','.join(requirements)}") if setup_commands: command_format = ';\\\n'.join(setup_commands.split(';')) diff --git a/.github/jobs/setup_and_run_use_cases.py b/.github/jobs/setup_and_run_use_cases.py index a2e55f8fa0..37b47537f6 100755 --- a/.github/jobs/setup_and_run_use_cases.py +++ b/.github/jobs/setup_and_run_use_cases.py @@ -63,7 +63,7 @@ def main(): # use BuildKit to build image os.environ['DOCKER_BUILDKIT'] = '1' - isOK = True + is_ok = True failed_use_cases = [] for setup_commands, use_case_commands, requirements in all_commands: # get environment image tag @@ -79,45 +79,38 @@ def main(): f"-f {DOCKERFILE_DIR}/{dockerfile_name} ." ) - print(f'Building Docker environment/branch image...') + print('Building Docker environment/branch image...') if not run_commands(docker_build_cmd): - isOK = False + is_ok = False continue - commands = [] - - # print list of existing docker images - commands.append('docker images') - + # print list of existing docker images, # remove docker image after creating run env or prune untagged images - commands.append(f'docker image rm dtcenter/metplus-dev:{branch_name} -f') - commands.append('docker image prune -f') - run_commands(commands) - - commands = [] - - # list docker images again after removal - commands.append('docker images') - - # start interactive container in the background - commands.append( - f"docker run -d --rm -it -e GITHUB_WORKSPACE " - f"--name {RUN_TAG} " - f"{os.environ.get('NETWORK_ARG', '')} " - f"{' '.join(VOLUME_MOUNTS)} " - f"{volumes_from} --workdir {GITHUB_WORKSPACE} " - f'{RUN_TAG} bash' - ) - - # list running containers - commands.append('docker ps -a') - - # execute setup commands in running docker container - commands.append(_format_docker_exec_command(setup_commands)) + run_commands([ + 'docker images', + f'docker image rm dtcenter/metplus-dev:{branch_name} -f', + 'docker image prune -f', + ]) + + # list docker images again after removal, + # start interactive container in the background, + # list running containers, + # then execute setup commands in running docker container + commands = [ + 'docker images', + (f"docker run -d --rm -it -e GITHUB_WORKSPACE " + f"--name {RUN_TAG} " + f"{os.environ.get('NETWORK_ARG', '')} " + f"{' '.join(VOLUME_MOUNTS)} " + f"{volumes_from} --workdir {GITHUB_WORKSPACE} " + f'{RUN_TAG} bash'), + 'docker ps -a', + _format_docker_exec_command(setup_commands), + ] # run docker commands and skip running cases if something went wrong if not run_commands(commands): - isOK = False + is_ok = False # force remove container if setup step fails run_commands(f'docker rm -f {RUN_TAG}') @@ -132,7 +125,7 @@ def main(): for use_case_command in use_case_commands: if not run_commands(_format_docker_exec_command(use_case_command)): failed_use_cases.append(use_case_command) - isOK = False + is_ok = False # print bashrc file to see what was added by setup commands # then force remove container to stop and remove it @@ -140,15 +133,20 @@ def main(): _format_docker_exec_command('cat /root/.bashrc'), f'docker rm -f {RUN_TAG}', ]): - isOK = False + is_ok = False # print summary of use cases that failed - for failed_use_case in failed_use_cases: - print(f'ERROR: Use case failed: {failed_use_case}') + _print_failed_use_cases(failed_use_cases) - if not isOK: + if not is_ok: print("ERROR: Some commands failed.") - sys.exit(1) + + return is_ok + + +def _print_failed_use_cases(failed_use_cases): + for failed_use_case in failed_use_cases: + print(f'ERROR: Use case failed: {failed_use_case}') def _format_docker_exec_command(command): @@ -201,4 +199,7 @@ def _get_dockerfile_name(requirements): if __name__ == '__main__': - main() + ret = main() + # exit non-zero if anything went wrong + if not ret: + sys.exit(1) diff --git a/.github/labels/README.md b/.github/labels/README.md index 927671a2bd..634256fca1 100644 --- a/.github/labels/README.md +++ b/.github/labels/README.md @@ -1,54 +1 @@ -METplus Repository Labels -========================= - -The contents of this directory define GitHub issue labels that are common -to all of the METplus-related repositories in the DTCenter organization. - -These labels should be defined consistently across all of the repositories. -However, individual repositories are encouraged to add labels specific to -their codebase. - -Labels are defined for the following categories: -- "alert" flags issues for discussion -- "component" classifies the type of work (customization encouraged) -- "priority" defines the priority level -- "reporting" defines project management reporting requirements -- "requestor" identifies the relevant organization(s) -- "type" corresponds to GitHub issue templates (customization allowed) - -In general, new "alert", "priority", "reporting", and "requestor" labels -should be added to this common location and defined consistently across -all the METplus repositories. - -Each repository should define a relevant set of "component" labels since -they will vary widely depending on the code base. - -New "type" labels should only be added to a repository when a corresponding -GitHub issue template is also added. - -To add a new common label, manually update the common_labels.txt file. - -Sample commands for processing all METplus repos: - -``` -# List of METplus repositories -REPO_LIST="metplus met metplotpy metcalcpy metdataio metviewer \ - metexpress metplus-training metplus-internal \ - metbaseimage"; - -# Build commands to add/update common labels -for repo in ${REPO_LIST}; do - echo $repo; - ./post_patch_labels.sh [user] [auth] $repo common_labels.txt; -done - -# Build commands to delete extra labels -for repo in ${REPO_LIST}; do - echo $repo; - ./delete_labels.sh [user] [auth] $repo; -done -``` - -The resulting commands are written to bash shell scripts in the commands -directory. Those commands should be carefully reviewed before executing them. - +Please see the [GitHub Labels to Organize Issues](https://metplus.readthedocs.io/en/develop/Contributors_Guide/github_workflow.html#github-labels-to-organize-issues) section of the METplus Contributor's Guide. diff --git a/.github/labels/common_labels.txt b/.github/labels/common_labels.txt index 4ea4bd760b..4ed7ffa623 100644 --- a/.github/labels/common_labels.txt +++ b/.github/labels/common_labels.txt @@ -24,6 +24,7 @@ {"name": "reporting: DTC NOAA R2O","color": "fbca04","description": "NOAA Research to Operations DTC Project"} {"name": "reporting: HSUP 1-3a","color": "fbca04","description": "Hurricane Supplemental Project 1-3a"} {"name": "reporting: HSUP 4-1b","color": "fbca04","description": "Hurricane Supplemental Project 4-1b"} +{"name": "reporting: NRL METplus","color": "fbca04","description": "Naval Research Laboratory METplus Project"} {"name": "requestor: Army/ARL","color": "3101c1","description": "United States Army Research Laboratory"} {"name": "requestor: Australian BOM","color": "3101c1","description": "Australian Bureau of Meteorology"} {"name": "requestor: Community","color": "3101c1","description": "General Community"} diff --git a/.github/labels/delete_labels.sh b/.github/labels/delete_labels.sh index 4ff004ebe8..479f28d3b6 100755 --- a/.github/labels/delete_labels.sh +++ b/.github/labels/delete_labels.sh @@ -6,8 +6,7 @@ # - repository name if [[ $# -ne 3 ]]; then - echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, and repository name." - echo "ERROR: repo list: metplus, met, metdataio, metcalcpy, metplotpy, metviewer, metexpress, metplus-training" + echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, and METplus repository name." exit 1 else user=$1 diff --git a/.github/labels/get_labels.sh b/.github/labels/get_labels.sh index b6bd6b0c5d..a4fb0844d3 100755 --- a/.github/labels/get_labels.sh +++ b/.github/labels/get_labels.sh @@ -6,8 +6,7 @@ # - repository name if [[ $# -ne 3 ]]; then - echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, and repository name." - echo "ERROR: repo names: metplus, met, metdataio, metcalcpy, metplotpy, metviewer, metexpress, metplus-training" + echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, and METplus repository name." exit 1 else user=$1 diff --git a/.github/labels/post_patch_labels.sh b/.github/labels/post_patch_labels.sh index 5c3cc53441..5d5ee3a36d 100755 --- a/.github/labels/post_patch_labels.sh +++ b/.github/labels/post_patch_labels.sh @@ -7,8 +7,7 @@ # - label file if [[ $# -ne 4 ]]; then - echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, repository name, and label file." - echo "ERROR: repo names: metplus, met, metdataio, metcalcpy, metplotpy, metviewer, metexpress, metplus-training, metplus-internal, metbaseimage" + echo "ERROR: `basename $0` ... must specify the GitHub username, authorization key, METplus repository name, and label file." exit 1 else user=$1 diff --git a/.github/labels/process_labels.sh b/.github/labels/process_labels.sh index a23bf7c90c..a75b8675a4 100755 --- a/.github/labels/process_labels.sh +++ b/.github/labels/process_labels.sh @@ -17,7 +17,7 @@ SCRIPT_DIR=`dirname $0` # List of METplus repositories REPO_LIST="metplus met metplotpy metcalcpy metdataio metviewer \ - metexpress metplus-training"; + metexpress metbaseimage metplus-internal metplus-training"; # Process each repository for REPO in ${REPO_LIST}; do diff --git a/.github/parm/Externals_metdataio.cfg b/.github/parm/Externals_metdataio.cfg deleted file mode 100644 index b7f7544f81..0000000000 --- a/.github/parm/Externals_metdataio.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[METdataio] -local_path = ../METdataio -protocol = git -required = True -repo_url = https://github.com/dtcenter/METdataio -branch = develop - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/Externals_metdataio_stable.cfg b/.github/parm/Externals_metdataio_stable.cfg deleted file mode 100644 index af45402334..0000000000 --- a/.github/parm/Externals_metdataio_stable.cfg +++ /dev/null @@ -1,9 +0,0 @@ -[METdataio] -local_path = ../METdataio -protocol = git -required = True -repo_url = https://github.com/dtcenter/METdataio -branch = main_v2.1 - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/Externals_metplotcalcpy.cfg b/.github/parm/Externals_metplotcalcpy.cfg deleted file mode 100644 index fdf1383555..0000000000 --- a/.github/parm/Externals_metplotcalcpy.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[METcalcpy] -local_path = ../METcalcpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METcalcpy -branch = develop - -[METplotpy] -local_path = ../METplotpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METplotpy -branch = develop - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/Externals_metplotcalcpy_stable.cfg b/.github/parm/Externals_metplotcalcpy_stable.cfg deleted file mode 100644 index 5aed1cb69b..0000000000 --- a/.github/parm/Externals_metplotcalcpy_stable.cfg +++ /dev/null @@ -1,16 +0,0 @@ -[METcalcpy] -local_path = ../METcalcpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METcalcpy -branch = main_v2.1 - -[METplotpy] -local_path = ../METplotpy -protocol = git -required = True -repo_url = https://github.com/dtcenter/METplotpy -branch = main_v2.1 - -[externals_description] -schema_version = 1.0.0 diff --git a/.github/parm/use_case_groups.json b/.github/parm/use_case_groups.json index 549f8cbc85..c4a3b5b7c8 100644 --- a/.github/parm/use_case_groups.json +++ b/.github/parm/use_case_groups.json @@ -9,6 +9,11 @@ "index_list": "30-58", "run": false }, + { + "category": "met_tool_wrapper", + "index_list": "65", + "run": false + }, { "category": "air_quality_and_comp", "index_list": "0", @@ -54,10 +59,14 @@ "index_list": "0-1", "run": false }, + { + "category": "fire", + "index_list": "0", + "run": false + }, { "category": "land_surface", "index_list": "0", - "disabled": true, "run": false }, { @@ -95,6 +104,13 @@ "index_list": "10", "run": false }, + { + "category": "marine_and_cryosphere", + "index_list": "11", + "disabled": true, + "disabled_reason": "exceeds GitHub Actions environment limits", + "run": false + }, { "category": "medium_range", "index_list": "0", @@ -107,7 +123,7 @@ }, { "category": "medium_range", - "index_list": "3-5,10", + "index_list": "3-5,10", "run": false }, { @@ -198,12 +214,12 @@ { "category": "s2s_mid_lat", "index_list": "0-2", - "run": true + "run": false }, { "category": "s2s_mid_lat", "index_list": "3", - "run": true + "run": false }, { "category": "s2s_mjo", @@ -225,6 +241,22 @@ "index_list": "0", "run": false }, + { + "category": "s2s_stratosphere", + "index_list": "1", + "disabled": true, + "disabled_reason": "exceeds GitHub Actions environment limits", + "data_location": "v6.0/additional_data_UserScript_fcstGFS_obsERA_StratospherePolar.tgz", + "run": false + }, + { + "category": "s2s_stratosphere", + "index_list": "2", + "disabled": true, + "disabled_reason": "exceeds GitHub Actions environment limits", + "data_location": "v6.0/additional_data_UserScript_fcstGFS_obsERA_StratosphereQBO.tgz", + "run": false + }, { "category": "short_range", "index_list": "0", @@ -269,8 +301,16 @@ "category": "short_range", "index_list": "14", "disabled": true, + "disabled_reason": "need new RRFS data with new line types", "run": false }, + { + "category": "short_range", + "index_list": "15", + "disabled": true, + "disabled_reason": "exceeds GitHub Actions environment limits", + "run":false + }, { "category": "space_weather", "index_list": "0-1", diff --git a/.github/workflows/create_conda_envs.yml b/.github/workflows/create_conda_envs.yml index c51f18d06f..950c47b2a3 100644 --- a/.github/workflows/create_conda_envs.yml +++ b/.github/workflows/create_conda_envs.yml @@ -283,6 +283,20 @@ jobs: - run: .github/jobs/build_conda_image.sh - run: .github/jobs/push_conda_image.sh + mp_analysis: + if: | + always() && (needs.check.outputs.no_skip == 'true' || + contains(fromJSON(needs.check.outputs.run_list), 'mp_analysis')) + runs-on: ubuntu-latest + needs: [check,metplotpy] + env: + ENV_NAME: ${{ github.job }} + BASE_ENV: metplotpy + steps: + - uses: actions/checkout@v4 + - run: .github/jobs/build_conda_image.sh + - run: .github/jobs/push_conda_image.sh + diff: if: | always() && (needs.check.outputs.no_skip == 'true' || diff --git a/.github/workflows/release-docker-images.yml b/.github/workflows/release-docker-images.yml new file mode 100644 index 0000000000..1ebe6ff2c3 --- /dev/null +++ b/.github/workflows/release-docker-images.yml @@ -0,0 +1,34 @@ +name: Create Docker images for release + +on: + release: + types: + - published + +jobs: + build_and_push: + name: Build and Push Images + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Get and check tag name + id: get_tag_name + run: | + SOURCE_BRANCH=${GITHUB_REF#refs/tags/} + if [[ ! "${SOURCE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + echo "ERROR: Tag name (${SOURCE_BRANCH}) does not start with vX.Y.Z format" + exit 1 + fi + echo SOURCE_BRANCH=${SOURCE_BRANCH} >> $GITHUB_OUTPUT + - name: Build metplus and metplus-analysis images + id: build_images + run: .github/jobs/docker_build_metplus_images.sh + env: + SOURCE_BRANCH: ${{ steps.get_tag_name.outputs.SOURCE_BRANCH }} + - name: Push metplus and metplus-analysis images + run: .github/jobs/docker_push_metplus_images.sh + env: + SOURCE_BRANCH: ${{ steps.get_tag_name.outputs.SOURCE_BRANCH }} + LATEST_TAG: ${{ steps.build_images.outputs.LATEST_TAG }} + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.github/workflows/sonarqube.yml b/.github/workflows/sonarqube.yml index 694697b94c..1c9fcd226d 100644 --- a/.github/workflows/sonarqube.yml +++ b/.github/workflows/sonarqube.yml @@ -14,8 +14,6 @@ on: - '.github/pull_request_template.md' - '.github/ISSUE_TEMPLATE/**' - '.github/labels/**' - - 'build_components/**' - - 'manage_externals/**' - '**/README.md' - '**/LICENSE.md' @@ -30,8 +28,6 @@ on: - '.github/pull_request_template.md' - '.github/ISSUE_TEMPLATE/**' - '.github/labels/**' - - 'build_components/**' - - 'manage_externals/**' - '**/README.md' - '**/LICENSE.md' @@ -64,6 +60,9 @@ jobs: python3 -m pip install --upgrade pip python3 -m pip install -r internal/tests/pytests/requirements.txt + - name: Install ImageMagick convert + run: sudo apt install imagemagick + - name: Run Pytests run: coverage run -m pytest internal/tests/pytests env: diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index bd1feac438..d111f593a5 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -1,5 +1,5 @@ name: Testing - +run-name: ${{ (github.event_name == 'workflow_dispatch' && (github.event.inputs.title || github.event.inputs.repository || 'Manual Trigger Testing')) || (github.event_name == 'pull_request' && github.event.pull_request.title) || (github.event_name == 'push' && github.event.head_commit.message) || github.workflow }} on: push: @@ -14,8 +14,6 @@ on: - '.github/pull_request_template.md' - '.github/ISSUE_TEMPLATE/**' - '.github/labels/**' - - 'build_components/**' - - 'manage_externals/**' - '**/README.md' - '**/LICENSE.md' @@ -29,13 +27,13 @@ on: - '.github/pull_request_template.md' - '.github/ISSUE_TEMPLATE/**' - '.github/labels/**' - - 'build_components/**' - - 'manage_externals/**' - '**/README.md' - '**/LICENSE.md' workflow_dispatch: inputs: + title: + description: 'Optional title of workflow run' force_met_image: description: 'MET DockerHub repo to force run to use, e.g. met:11.1.0 or met-dev:feature_XYZ_name-PR. Leave this blank to determine repo automatically.' repository: @@ -52,15 +50,18 @@ on: jobs: event_info: - name: "Trigger: ${{ github.event_name != 'workflow_dispatch' && github.event_name || github.event.inputs.repository }} ${{ github.event_name != 'workflow_dispatch' && 'event' || github.event.inputs.sha }} ${{ github.event_name != 'workflow_dispatch' && 'local' || github.event.inputs.actor }} " + name: "Trigger: ${{ github.event_name != 'workflow_dispatch' && github.event_name || (github.event.inputs.repository || 'Manual Trigger Testing') }} ${{ github.event_name != 'workflow_dispatch' && 'event' || github.event.inputs.sha }} ${{ github.event_name != 'workflow_dispatch' && 'local' || github.event.inputs.actor }} " runs-on: ubuntu-latest steps: - name: Print GitHub values for reference env: GITHUB_CONTEXT: ${{ toJson(github) }} run: echo "$GITHUB_CONTEXT" + # continue if this step fails, which can happen when the merge commit + # message is too long + continue-on-error: true - name: Build URL for commit that triggered workflow - if: github.event_name == 'workflow_dispatch' + if: github.event_name == 'workflow_dispatch' && startsWith(github.event.inputs.repository, 'dtcenter/') && github.event.inputs.sha run: echo https://github.com/${{ github.event.inputs.repository }}/commit/${{ github.event.inputs.sha }} job_control: @@ -141,6 +142,8 @@ jobs: run: | python3 -m pip install --upgrade pip python3 -m pip install -r internal/tests/pytests/requirements.txt + - name: Install ImageMagick convert + run: sudo apt install imagemagick - name: Run Pytests run: coverage run -m pytest internal/tests/pytests env: diff --git a/.github/workflows/update_input_data.yml b/.github/workflows/update_input_data.yml index 2f774c68f8..5235a03511 100644 --- a/.github/workflows/update_input_data.yml +++ b/.github/workflows/update_input_data.yml @@ -13,7 +13,7 @@ jobs: - run: | branch_name=$(echo ${{ github.event.inputs.branch || github.ref }} | cut -d/ -f3) echo "branch_name=$branch_name" >> "$GITHUB_ENV" - - uses: dtcenter/metplus-action-data-update@v2 + - uses: dtcenter/metplus-action-data-update@v3 with: docker_name: ${{ secrets.DOCKER_USERNAME }} docker_pass: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.github/workflows/update_truth.yml b/.github/workflows/update_truth.yml index 6b190e42fa..344183c30c 100644 --- a/.github/workflows/update_truth.yml +++ b/.github/workflows/update_truth.yml @@ -65,11 +65,11 @@ jobs: # get truth change log from *-ref branch cmd="git checkout origin/${branch_name}-ref -- ${change_log_path}" echo $cmd - $cmd + $cmd || true # create or append to file to track truth data changes # and ensure that PR merge into *-ref branch triggered testing workflow - change_entry="[$(date +%Y%m%d_%H:%M:%S) ${branch_name}] ${{ github.event.inputs.pull_requests }} - ${{ github.event.inputs.change_summary }}" + change_entry='[$(date +%Y%m%d_%H:%M:%S) ${branch_name}] ${{ github.event.inputs.pull_requests }} - ${{ github.event.inputs.change_summary }}' echo "${change_entry}" >> ${change_log_path} # add file if it does not already exist diff --git a/.idea/METplus.iml b/.idea/METplus.iml index aad402c4e5..1644746a0c 100644 --- a/.idea/METplus.iml +++ b/.idea/METplus.iml @@ -1,5 +1,13 @@ - + + + + + + + + +