Merge pull request #4609 from GSA/scale

add scale-web-template

.github/workflows/scale-web-template.yml

@@ -0,0 +1,62 @@
+---
+name: Scale Web Template
+
+on:   # yamllint disable-line rule:truthy
+  workflow_call:
+    inputs:
+      environ:
+        required: true
+        type: string
+      app_names:
+        required: true
+        type: string
+
+    secrets:
+      CF_SERVICE_USER:
+        required: true
+      CF_SERVICE_AUTH:
+        required: true
+
+jobs:
+  scale:
+    name: scale (${{ inputs.environ }})
+    environment: ${{ inputs.environ }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.app_names) }}
+    steps:
+      - name: checkout datagov
+        uses: actions/checkout@v3
+        with:
+          repository: gsa/data.gov
+          path: './datagov'
+      - name: scale ${{ matrix.app }}
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          command: datagov/bin/check-and-renew ${{ matrix.app }} scale
+          cf_org: gsa-datagov
+          cf_space: ${{ inputs.environ }}
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}
+      - name: smoke test
+        if: ${{ matrix.smoketest }}
+        run: |
+          sleep 10
+          curl --fail --silent ${{ inputs.app_url }}\
+          /api/action/status_show?$(date +%s)
+      - name: Create Issue if it fails 😢
+        if: ${{ failure() && github.ref == 'refs/heads/main' }}
+        uses: JasonEtco/create-an-issue@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.ADD_TO_PROJECT_PAT }}
+          GITHUB_JOB: ${{ toJson(github)['job'] }}
+          GITHUB_ATTEMPTS: ${{ github.run_attempt }}
+          LAST_COMMIT: ${{ github.sha }}
+          LAST_RUN_BY: ${{ github.actor }}
+          RUN_ID: ${{ github.run_id }}
+          REPO: ${{ github.repository }}
+        with:
+          filename: datagov/.github/restart_failure.md
+          assignees: ${{ github.actor }}
+          update_existing: true

bin/check-and-renew

@@ -21,8 +21,8 @@ function check_running_job {
 
   # Input whether it's deploy or restart
   if [[ -n "$2" ]]; then
-    if [[ ! $2 =~ ^(deploy|restart)$ ]]; then
-      echo "Need to specify 'deploy' or 'restart'"
+    if [[ ! $2 =~ ^(deploy|restart|scale)$ ]]; then
+      echo "Need to specify 'deploy', 'restart', or 'scale'."
       exit 1
     fi
     action=$2
@@ -119,6 +119,23 @@ if [[ "$action" == "restart" ]]; then
   fi
 fi
 
+if [[ "$action" == "scale" ]]; then
+  if [[ $ok_to_proceed == "YES" ]]; then
+    # call scale functions to get the scale_to value
+    source "$(dirname "$0")/scale_calculate.sh"
+    set_scale_number
+    if [[ -z $scale_to ]]; then
+      echo "Scale to value not set. Cancelling scale."
+      exit 1
+    fi
+    # get $scale_to from the environment vars
+    echo "Scaling $app_to_check to $scale_to"
+    cf scale "$app_to_check" -i "$scale_to"
+  else
+    echo "scaling of $app_to_check cancelled."
+  fi
+fi
+
 if [[ "$action" == "deploy" ]]; then
   if [[ $ok_to_proceed == "YES" ]]; then
     cf push "$app_to_check" --vars-file vars.$space.yml --strategy rolling

bin/scale_calculate.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+
+MAX_INSTANCES=9
+MIN_INSTANCES=5
+SCALE_STEP=2
+
+CPU_BUSY_THRESHOLD=320
+CPU_IDLE_THRESHOLD=250
+
+app_status=$(cf app catalog-web)
+scale_direction=""
+
+# This function will check the average CPU usage of all running instances of an app
+function set_scale_direction {
+  # Extract lines containing 'running'
+  running_lines=$(echo "$app_status" | grep "#[0-9]*[[:space:]]*running")
+
+  cpu_sum=0
+  count=0
+
+  if [ -n "$running_lines" ]; then
+    while IFS= read -r line
+    do
+      # Extract CPU usage and remove '%' character
+      cpu_usage=$(echo $line | cut -d' ' -f4 | tr -d '%')
+      cpu_sum=$(echo "$cpu_sum + $cpu_usage" | bc)
+      ((count++))
+    done <<< "$running_lines"
+  fi
+
+  # Calculate average CPU usage
+  average_cpu=$(echo "scale=2; $cpu_sum / $count" | bc)
+
+  if [ "$(echo "$average_cpu > $CPU_BUSY_THRESHOLD" | bc)" -eq 1 ]; then
+    echo "Average CPU is $average_cpu. Too High."
+    scale_direction="up"
+  elif [ "$(echo "$average_cpu < $CPU_IDLE_THRESHOLD" | bc)" -eq 1 ]; then
+    echo "Average CPU is $average_cpu. Too Low."
+    scale_direction="down"
+  else
+    echo "Average CPU is $average_cpu. Just Right."
+    scale_direction="same"
+  fi
+}
+
+function set_scale_number {
+  # get the total number of instances
+  total_instances=$(echo "$app_status" | grep '^instances:' | awk '{print $2}' | cut -d '/' -f 2)
+  # exit if the total instances is not a number
+  if ! [[ "$total_instances" =~ ^[0-9]+$ ]]; then
+    echo "Total instances is not a number. Exiting. Here is the output of the app status:"
+    echo "$app_status"
+    exit 1
+  fi
+  echo "Current total instances: $total_instances"
+  # call the scale direction
+  set_scale_direction
+
+  # if the direction is up and the total instances is less than the max instances
+  if [ "$scale_direction" == "up" ] && [ "$total_instances" -lt "$MAX_INSTANCES" ]; then
+    export scale_to=$((total_instances + SCALE_STEP))
+    echo "Scaling up to $scale_to"
+  elif [ "$scale_direction" == "down" ] && [ "$total_instances" -gt "$MIN_INSTANCES" ]; then
+    # export the scale_to value but make sure scale_to is large than 1
+    export scale_to=$((total_instances - SCALE_STEP < 1 ? 1 : total_instances - SCALE_STEP))
+    echo "Scaling down to $scale_to"
+  else
+    export scale_to="$total_instances"
+    echo "Remain at the $scale_direction scale level $scale_to"
+  fi
+}