diff --git a/.github/workflows/scale-web-template.yml b/.github/workflows/scale-web-template.yml new file mode 100644 index 000000000..08151b752 --- /dev/null +++ b/.github/workflows/scale-web-template.yml @@ -0,0 +1,62 @@ +--- +name: Scale Web Template + +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + environ: + required: true + type: string + app_names: + required: true + type: string + + secrets: + CF_SERVICE_USER: + required: true + CF_SERVICE_AUTH: + required: true + +jobs: + scale: + name: scale (${{ inputs.environ }}) + environment: ${{ inputs.environ }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.app_names) }} + steps: + - name: checkout datagov + uses: actions/checkout@v3 + with: + repository: gsa/data.gov + path: './datagov' + - name: scale ${{ matrix.app }} + uses: cloud-gov/cg-cli-tools@main + with: + command: datagov/bin/check-and-renew ${{ matrix.app }} scale + cf_org: gsa-datagov + cf_space: ${{ inputs.environ }} + cf_username: ${{secrets.CF_SERVICE_USER}} + cf_password: ${{secrets.CF_SERVICE_AUTH}} + - name: smoke test + if: ${{ matrix.smoketest }} + run: | + sleep 10 + curl --fail --silent ${{ inputs.app_url }}\ + /api/action/status_show?$(date +%s) + - name: Create Issue if it fails 😢 + if: ${{ failure() && github.ref == 'refs/heads/main' }} + uses: JasonEtco/create-an-issue@v2 + env: + GITHUB_TOKEN: ${{ secrets.ADD_TO_PROJECT_PAT }} + GITHUB_JOB: ${{ toJson(github)['job'] }} + GITHUB_ATTEMPTS: ${{ github.run_attempt }} + LAST_COMMIT: ${{ github.sha }} + LAST_RUN_BY: ${{ github.actor }} + RUN_ID: ${{ github.run_id }} + REPO: ${{ github.repository }} + with: + filename: datagov/.github/restart_failure.md + assignees: ${{ github.actor }} + update_existing: true diff --git a/bin/check-and-renew b/bin/check-and-renew index 1b789c6c2..e6b9721a7 100755 --- a/bin/check-and-renew +++ b/bin/check-and-renew @@ -21,8 +21,8 @@ function check_running_job { # Input whether it's deploy or restart if [[ -n "$2" ]]; then - if [[ ! $2 =~ ^(deploy|restart)$ ]]; then - echo "Need to specify 'deploy' or 'restart'" + if [[ ! $2 =~ ^(deploy|restart|scale)$ ]]; then + echo "Need to specify 'deploy', 'restart', or 'scale'." exit 1 fi action=$2 @@ -119,6 +119,23 @@ if [[ "$action" == "restart" ]]; then fi fi +if [[ "$action" == "scale" ]]; then + if [[ $ok_to_proceed == "YES" ]]; then + # call scale functions to get the scale_to value + source "$(dirname "$0")/scale_calculate.sh" + set_scale_number + if [[ -z $scale_to ]]; then + echo "Scale to value not set. Cancelling scale." + exit 1 + fi + # get $scale_to from the environment vars + echo "Scaling $app_to_check to $scale_to" + cf scale "$app_to_check" -i "$scale_to" + else + echo "scaling of $app_to_check cancelled." + fi +fi + if [[ "$action" == "deploy" ]]; then if [[ $ok_to_proceed == "YES" ]]; then cf push "$app_to_check" --vars-file vars.$space.yml --strategy rolling diff --git a/bin/scale_calculate.sh b/bin/scale_calculate.sh new file mode 100644 index 000000000..37a49f5d9 --- /dev/null +++ b/bin/scale_calculate.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +MAX_INSTANCES=9 +MIN_INSTANCES=5 +SCALE_STEP=2 + +CPU_BUSY_THRESHOLD=320 +CPU_IDLE_THRESHOLD=250 + +app_status=$(cf app catalog-web) +scale_direction="" + +# This function will check the average CPU usage of all running instances of an app +function set_scale_direction { + # Extract lines containing 'running' + running_lines=$(echo "$app_status" | grep "#[0-9]*[[:space:]]*running") + + cpu_sum=0 + count=0 + + if [ -n "$running_lines" ]; then + while IFS= read -r line + do + # Extract CPU usage and remove '%' character + cpu_usage=$(echo $line | cut -d' ' -f4 | tr -d '%') + cpu_sum=$(echo "$cpu_sum + $cpu_usage" | bc) + ((count++)) + done <<< "$running_lines" + fi + + # Calculate average CPU usage + average_cpu=$(echo "scale=2; $cpu_sum / $count" | bc) + + if [ "$(echo "$average_cpu > $CPU_BUSY_THRESHOLD" | bc)" -eq 1 ]; then + echo "Average CPU is $average_cpu. Too High." + scale_direction="up" + elif [ "$(echo "$average_cpu < $CPU_IDLE_THRESHOLD" | bc)" -eq 1 ]; then + echo "Average CPU is $average_cpu. Too Low." + scale_direction="down" + else + echo "Average CPU is $average_cpu. Just Right." + scale_direction="same" + fi +} + +function set_scale_number { + # get the total number of instances + total_instances=$(echo "$app_status" | grep '^instances:' | awk '{print $2}' | cut -d '/' -f 2) + # exit if the total instances is not a number + if ! [[ "$total_instances" =~ ^[0-9]+$ ]]; then + echo "Total instances is not a number. Exiting. Here is the output of the app status:" + echo "$app_status" + exit 1 + fi + echo "Current total instances: $total_instances" + # call the scale direction + set_scale_direction + + # if the direction is up and the total instances is less than the max instances + if [ "$scale_direction" == "up" ] && [ "$total_instances" -lt "$MAX_INSTANCES" ]; then + export scale_to=$((total_instances + SCALE_STEP)) + echo "Scaling up to $scale_to" + elif [ "$scale_direction" == "down" ] && [ "$total_instances" -gt "$MIN_INSTANCES" ]; then + # export the scale_to value but make sure scale_to is large than 1 + export scale_to=$((total_instances - SCALE_STEP < 1 ? 1 : total_instances - SCALE_STEP)) + echo "Scaling down to $scale_to" + else + export scale_to="$total_instances" + echo "Remain at the $scale_direction scale level $scale_to" + fi +}