Merge branch 'develop' into feature_2390_CTP_HumIndex

.coveragerc

@@ -3,3 +3,4 @@ relative_files = True
 source = metplus
 omit =
   metplus/wrappers/cyclone_plotter_wrapper.py
+  metplus/produtil/**

.github/ISSUE_TEMPLATE/documentation.md

@@ -0,0 +1,68 @@
+---
+name: Documentation
+about: Update the documentation
+title: 'Documentation: '
+labels: 'alert: NEED ACCOUNT KEY, alert: NEED MORE DEFINITION, alert: NEED CYCLE ASSIGNMENT, type: documentation'
+assignees: ''
+
+---
+
+*Replace italics below with details for this issue.*
+
+## Describe the Task ##
+*Provide a description of the task here.*
+
+### Time Estimate ###
+*Estimate the amount of work required here.*
+*Issues should represent approximately 1 to 3 days of work.*
+
+### Sub-Issues ###
+Consider breaking the task down into sub-issues.
+- [ ] *Add a checkbox for each sub-issue here.*
+
+### Relevant Deadlines ###
+*List relevant project deadlines here or state NONE.*
+
+### Funding Source ###
+*Define the source of funding and account keys here or state NONE.*
+
+## Define the Metadata ##
+
+### Assignee ###
+- [ ] Select appropriate **assignee** for this issue
+
+### Labels ###
+- [ ] Review default **alert** labels
+- [ ] Select **component(s)**
+- [ ] Select **priority**
+- [ ] Select **requestor(s)**
+
+### Milestone and Projects ###
+- [ ] Select **Milestone** as a **METplus-Wrappers-X.Y.Z** version, **Consider for Next Release**, or **Backlog of Development Ideas**
+- [ ] For a **METplus-Wrappers-X.Y.Z** version, select the **METplus-Wrappers-X.Y.Z Development** project
+
+## Define Related Issue(s) ##
+Consider the impact to the other METplus components.
+- [ ] [METplus](https://github.com/dtcenter/METplus/issues/new/choose), [MET](https://github.com/dtcenter/MET/issues/new/choose), [METdataio](https://github.com/dtcenter/METdataio/issues/new/choose), [METviewer](https://github.com/dtcenter/METviewer/issues/new/choose), [METexpress](https://github.com/dtcenter/METexpress/issues/new/choose), [METcalcpy](https://github.com/dtcenter/METcalcpy/issues/new/choose), [METplotpy](https://github.com/dtcenter/METplotpy/issues/new/choose)
+
+## Task Checklist ##
+See the [METplus Workflow](https://metplus.readthedocs.io/en/latest/Contributors_Guide/github_workflow.html) for details.
+- [ ] Complete the issue definition above, including the **Time Estimate** and **Funding Source**.
+- [ ] Fork this repository or create a branch of **develop**.
+Branch name: `feature_<Issue Number>_<Description>`
+- [ ] Complete the development and test your changes.
+- [ ] Add/update log messages for easier debugging.
+- [ ] Add/update unit tests.
+- [ ] Add/update documentation.
+- [ ] Add any new Python packages to the [METplus Components Python Requirements](https://metplus.readthedocs.io/en/develop/Users_Guide/appendixA.html#metplus-components-python-packages) table.
+- [ ] For any new datasets, an entry to the [METplus Verification Datasets Guide](https://metplus.readthedocs.io/en/latest/Verification_Datasets/index.html).
+- [ ] Push local changes to GitHub.
+- [ ] Submit a pull request to merge into **develop**.
+Pull request: `feature <Issue Number> <Description>`
+- [ ] Define the pull request metadata, as permissions allow.
+Select: **Reviewer(s)** and **Development** issue
+Select: **Milestone** as the next official version
+Select: **METplus-Wrappers-X.Y.Z Development** project for development toward the next official release
+- [ ] Iterate until the reviewer(s) accept and merge your changes.
+- [ ] Delete your fork or branch.
+- [ ] Close this issue.

.github/ISSUE_TEMPLATE/enhancement_request.md

@@ -1,7 +1,7 @@
 ---
 name: Enhancement request
 about: Improve something that it's currently doing
-title: ''
+title: 'Enhancement: '
 labels: 'alert: NEED ACCOUNT KEY, alert: NEED MORE DEFINITION, alert: NEED CYCLE ASSIGNMENT, type: enhancement'
 assignees: ''
 

.github/jobs/docker_build_metplus_images.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+# assumes SOURCE_BRANCH is set before calling script
+
+source "${GITHUB_WORKSPACE}"/.github/jobs/bash_functions.sh
+
+dockerhub_repo=dtcenter/metplus
+dockerhub_repo_analysis=dtcenter/metplus-analysis
+
+# check if tag is official or bugfix release -- no -betaN or -rcN suffix
+is_official=1
+if [[ "${SOURCE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+  is_official=0
+fi
+
+# remove v prefix
+metplus_version=${SOURCE_BRANCH:1}
+
+# if rc is in version number, get main_vX.Y, otherwise get X.Y-latest or develop
+if [[ "${metplus_version}" =~ rc ]]; then
+  tag_format="main_v{X}.{Y}"
+else
+  tag_format="{X}.{Y}-latest"
+fi
+
+# Get MET tag and adjust MET Docker repo if develop
+met_tag=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f ${tag_format} --no-get_dev_version)
+echo "$met_tag"
+
+MET_DOCKER_REPO=met
+if [ "$met_tag" == "develop" ] || [[ "${met_tag}" =~ ^main_v[0-9]+\.[0-9]+ ]]; then
+    MET_DOCKER_REPO=met-dev
+fi
+
+# get METplus Analysis tool versions
+METDATAIO_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METdataio)
+METCALCPY_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METcalcpy)
+METPLOTPY_VERSION=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o METplotpy)
+
+# Build metplus image
+METPLUS_IMAGE_NAME=${dockerhub_repo}:${metplus_version}
+if ! time_command docker build -t "$METPLUS_IMAGE_NAME" \
+       --build-arg SOURCE_VERSION="$SOURCE_BRANCH" \
+       --build-arg MET_TAG="$met_tag" \
+       --build-arg MET_DOCKER_REPO="$MET_DOCKER_REPO" \
+       -f "${GITHUB_WORKSPACE}"/internal/scripts/docker/Dockerfile \
+       "${GITHUB_WORKSPACE}"; then
+    exit 1
+fi
+
+# Build metplus-analysis image
+METPLUS_A_IMAGE_NAME=${dockerhub_repo_analysis}:${metplus_version}
+if ! time_command docker build -t "$METPLUS_A_IMAGE_NAME" \
+       --build-arg METPLUS_BASE_TAG="${metplus_version}" \
+       --build-arg METDATAIO_VERSION="${METDATAIO_VERSION}" \
+       --build-arg METCALCPY_VERSION="${METCALCPY_VERSION}" \
+       --build-arg METPLOTPY_VERSION="${METPLOTPY_VERSION}" \
+       -f "${GITHUB_WORKSPACE}"/internal/scripts/docker/Dockerfile.metplus-analysis \
+       "${GITHUB_WORKSPACE}"; then
+    exit 1
+fi
+
+# if official release, create X.Y-latest tag as well
+if [ "${is_official}" == 0 ]; then
+    LATEST_TAG=$(echo "$metplus_version" | cut -f1,2 -d'.')-latest
+    docker tag "${METPLUS_IMAGE_NAME}" "${dockerhub_repo}:${LATEST_TAG}"
+    docker tag "${METPLUS_A_IMAGE_NAME}" "${dockerhub_repo_analysis}:${LATEST_TAG}"
+    echo LATEST_TAG="${LATEST_TAG}" >> "$GITHUB_OUTPUT"
+fi

.github/jobs/docker_push_metplus_images.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# assumes SOURCE_BRANCH is set before calling script
+# assumes latest_tag will be set if pushing an official or bugfix release
+
+source "${GITHUB_WORKSPACE}"/.github/jobs/bash_functions.sh
+
+# get names of images to push
+
+dockerhub_repo=dtcenter/metplus
+dockerhub_repo_analysis=dtcenter/metplus-analysis
+
+# remove v prefix
+metplus_version=${SOURCE_BRANCH:1}
+
+METPLUS_IMAGE_NAME=${dockerhub_repo}:${metplus_version}
+METPLUS_A_IMAGE_NAME=${dockerhub_repo_analysis}:${metplus_version}
+
+# skip docker push if credentials are not set
+if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then
+    echo "DockerHub credentials not set. Skipping docker push"
+    exit 0
+fi
+
+echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
+
+# push images
+
+if ! time_command docker push "${METPLUS_IMAGE_NAME}"; then
+  exit 1
+fi
+
+if ! time_command docker push "${METPLUS_A_IMAGE_NAME}"; then
+  exit 1
+fi
+
+# only push X.Y-latest tag if official or bugfix release
+# shellcheck disable=SC2154
+if [ "${LATEST_TAG}" != "" ]; then
+    if ! time_command docker push "${dockerhub_repo}:${LATEST_TAG}"; then
+      exit 1
+    fi
+    if ! time_command docker push "${dockerhub_repo_analysis}:${LATEST_TAG}"; then
+      exit 1
+    fi
+fi

.github/jobs/docker_setup.sh

@@ -31,10 +31,19 @@ echo "TIMING: docker pull ${DOCKERHUB_TAG} took `printf '%02d' $(($duration / 60
 # set DOCKERFILE_PATH that is used by docker hook script get_met_version
 export DOCKERFILE_PATH=${GITHUB_WORKSPACE}/internal/scripts/docker/Dockerfile
 
-MET_TAG=`${GITHUB_WORKSPACE}/internal/scripts/docker/hooks/get_met_version`
+metplus_version=$(head -n 1 "${GITHUB_WORKSPACE}/metplus/VERSION")
+
+# if rc is in version number, get main_vX.Y, otherwise get X.Y-latest or develop
+if [[ "${metplus_version}" =~ rc ]]; then
+  tag_format="main_v{X}.{Y}"
+else
+  tag_format="{X}.{Y}-latest"
+fi
+
+MET_TAG=$("${GITHUB_WORKSPACE}"/metplus/component_versions.py -v "${metplus_version}" -o MET -f ${tag_format} --no-get_dev_version)
 
 MET_DOCKER_REPO=met-dev
-if [ "${MET_TAG}" != "develop" ]; then
+if [ "${MET_TAG}" != "develop" ] && ! [[ "${MET_TAG}" =~ ^main_v[0-9]+\.[0-9]+ ]]; then
     MET_DOCKER_REPO=met
 elif [ "${EXTERNAL_TRIGGER}" == "true" ]; then
     # if MET tag is develop and external repo triggered workflow

.github/jobs/docker_utils.py

@@ -39,10 +39,13 @@ def get_dockerhub_url(branch_name):
 def docker_get_volumes_last_updated(current_branch):
     import requests
     dockerhub_url = get_dockerhub_url(current_branch)
-    dockerhub_request = requests.get(dockerhub_url, timeout=60)
+    dockerhub_request = requests.get(dockerhub_url, timeout=90)
     if dockerhub_request.status_code != 200:
-        print(f"Could not find DockerHub URL: {dockerhub_url}")
-        return None
+        print(f"Retrying DockerHub request: {dockerhub_url}")
+        dockerhub_request = requests.get(dockerhub_url, timeout=60)
+        if dockerhub_request.status_code != 200:
+            print(f"Could not query DockerHub URL: {dockerhub_url}")
+            return None
 
     # get version number to search for if main_vX.Y branch
     if current_branch.startswith('main_v'):

.github/jobs/get_data_volumes.py

@@ -7,8 +7,6 @@
 
 import sys
 import os
-import subprocess
-import shlex
 
 from docker_utils import docker_get_volumes_last_updated, get_branch_name
 from docker_utils import get_data_repo, DOCKERHUB_METPLUS_DATA_DEV
@@ -54,7 +52,12 @@ def main(args):
         branch_name = branch_name[5:]
 
     # get all docker data volumes associated with current branch
-    available_volumes = docker_get_volumes_last_updated(branch_name).keys()
+    try:
+        available_volumes = docker_get_volumes_last_updated(branch_name).keys()
+    except AttributeError:
+        print("ERROR: Could not get available Docker data volumes "
+              f"for {branch_name}. Try rerunning failed jobs in GitHub Actions")
+        return None
 
     # loop through arguments and get data volume for each category
     for model_app_name in args:
@@ -78,7 +81,7 @@ def main(args):
             volume_name = f'{prefix}-{model_app_name}'
 
             # add output- to model app name
-            model_app_name=f'output-{model_app_name}'
+            model_app_name = f'output-{model_app_name}'
 
             # set DockerHub repo to dev version because all output data
             # should be in dev repository
@@ -108,11 +111,11 @@ def main(args):
 
 if __name__ == "__main__":
     # split up command line args that have commas before passing into main
-    args = []
+    arg_list = []
 
     for arg in sys.argv[1:]:
-        args.extend(arg.split(','))
-    out = main(args)
+        arg_list.extend(arg.split(','))
+    out = main(arg_list)
     if out is None:
         print("ERROR: Something went wrong")
         sys.exit(1)