From 51104afef66999c18b2fa60e16638dfce712ce04 Mon Sep 17 00:00:00 2001
From: Fuhu Xia <fxia@reisystems.com>
Date: Tue, 6 Feb 2024 17:09:00 -0500
Subject: [PATCH 1/3] add scale-web-template

---
 .github/workflows/scale-web-template.yml | 62 +++++++++++++++++++++++
 bin/check-and-renew                      | 20 +++++++-
 bin/scale_calculate.sh                   | 64 ++++++++++++++++++++++++
 3 files changed, 144 insertions(+), 2 deletions(-)
 create mode 100644 .github/workflows/scale-web-template.yml
 create mode 100644 bin/scale_calculate.sh

diff --git a/.github/workflows/scale-web-template.yml b/.github/workflows/scale-web-template.yml
new file mode 100644
index 000000000..08151b752
--- /dev/null
+++ b/.github/workflows/scale-web-template.yml
@@ -0,0 +1,62 @@
+---
+name: Scale Web Template
+
+on:   # yamllint disable-line rule:truthy
+  workflow_call:
+    inputs:
+      environ:
+        required: true
+        type: string
+      app_names:
+        required: true
+        type: string
+
+    secrets:
+      CF_SERVICE_USER:
+        required: true
+      CF_SERVICE_AUTH:
+        required: true
+
+jobs:
+  scale:
+    name: scale (${{ inputs.environ }})
+    environment: ${{ inputs.environ }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(inputs.app_names) }}
+    steps:
+      - name: checkout datagov
+        uses: actions/checkout@v3
+        with:
+          repository: gsa/data.gov
+          path: './datagov'
+      - name: scale ${{ matrix.app }}
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          command: datagov/bin/check-and-renew ${{ matrix.app }} scale
+          cf_org: gsa-datagov
+          cf_space: ${{ inputs.environ }}
+          cf_username: ${{secrets.CF_SERVICE_USER}}
+          cf_password: ${{secrets.CF_SERVICE_AUTH}}
+      - name: smoke test
+        if: ${{ matrix.smoketest }}
+        run: |
+          sleep 10
+          curl --fail --silent ${{ inputs.app_url }}\
+          /api/action/status_show?$(date +%s)
+      - name: Create Issue if it fails 😢
+        if: ${{ failure() && github.ref == 'refs/heads/main' }}
+        uses: JasonEtco/create-an-issue@v2
+        env:
+          GITHUB_TOKEN: ${{ secrets.ADD_TO_PROJECT_PAT }}
+          GITHUB_JOB: ${{ toJson(github)['job'] }}
+          GITHUB_ATTEMPTS: ${{ github.run_attempt }}
+          LAST_COMMIT: ${{ github.sha }}
+          LAST_RUN_BY: ${{ github.actor }}
+          RUN_ID: ${{ github.run_id }}
+          REPO: ${{ github.repository }}
+        with:
+          filename: datagov/.github/restart_failure.md
+          assignees: ${{ github.actor }}
+          update_existing: true
diff --git a/bin/check-and-renew b/bin/check-and-renew
index 1b789c6c2..fd0eb772e 100755
--- a/bin/check-and-renew
+++ b/bin/check-and-renew
@@ -21,8 +21,8 @@ function check_running_job {
 
   # Input whether it's deploy or restart
   if [[ -n "$2" ]]; then
-    if [[ ! $2 =~ ^(deploy|restart)$ ]]; then
-      echo "Need to specify 'deploy' or 'restart'"
+    if [[ ! $2 =~ ^(deploy|restart|scale)$ ]]; then
+      echo "Need to specify 'deploy', 'restart', or 'scale'."
       exit 1
     fi
     action=$2
@@ -119,6 +119,22 @@ if [[ "$action" == "restart" ]]; then
   fi
 fi
 
+if [[ "$action" == "scale" ]]; then
+  if [[ $ok_to_proceed == "YES" ]]; then
+    # set scale_to to a safe number but it should not be used
+    # since the scale_calculate.sh always export a new number
+    scale_to=7
+    # call scale functions to get the scale_to value
+    source "$(dirname "$0")/scale_calculate.sh"
+    set_scale_number
+    # get $scale_to from the environment vars
+    echo "Scaling $app_to_check to $scale_to"
+    cf scale "$app_to_check" -i "$scale_to"
+  else
+    echo "scaling of $app_to_check cancelled."
+  fi
+fi
+
 if [[ "$action" == "deploy" ]]; then
   if [[ $ok_to_proceed == "YES" ]]; then
     cf push "$app_to_check" --vars-file vars.$space.yml --strategy rolling
diff --git a/bin/scale_calculate.sh b/bin/scale_calculate.sh
new file mode 100644
index 000000000..dd184f93a
--- /dev/null
+++ b/bin/scale_calculate.sh
@@ -0,0 +1,64 @@
+#!/bin/bash
+
+MAX_INSTANCES=9
+MIN_INSTANCES=5
+SCALE_STEP=2
+
+CPU_BUSY_THRESHOLD=320
+CPU_IDLE_THRESHOLD=250
+
+app_status=$(cf app catalog-web)
+scale_direction=""
+
+# This function will check the average CPU usage of all running instances of an app
+function set_scale_direction {
+  # Extract lines containing 'running'
+  running_lines=$(echo "$app_status" | grep "#[0-9]*[[:space:]]*running")
+
+  cpu_sum=0
+  count=0
+
+  if [ -n "$running_lines" ]; then
+    while IFS= read -r line
+    do
+      # Extract CPU usage and remove '%' character
+      cpu_usage=$(echo $line | cut -d' ' -f4 | tr -d '%')
+      cpu_sum=$(echo "$cpu_sum + $cpu_usage" | bc)
+      ((count++))
+    done <<< "$running_lines"
+  fi
+
+  # Calculate average CPU usage
+  average_cpu=$(echo "scale=2; $cpu_sum / $count" | bc)
+
+  if [ "$(echo "$average_cpu > $CPU_BUSY_THRESHOLD" | bc)" -eq 1 ]; then
+    echo "Average CPU is $average_cpu. Too High."
+    scale_direction="up"
+  elif [ "$(echo "$average_cpu < $CPU_IDLE_THRESHOLD" | bc)" -eq 1 ]; then
+    echo "Average CPU is $average_cpu. Too Low."
+    scale_direction="down"
+  else
+    echo "Average CPU is $average_cpu. Just Right."
+    scale_direction="same"
+  fi
+}
+
+function set_scale_number {
+  # get the total number of instances
+  total_instances=$(echo "$app_status" | grep -oP 'instances:\s+\K\d+')
+  echo "Current total instances: $total_instances"
+  # call the scale direction
+  set_scale_direction
+
+  # if the direction is up and the total instances is less than the max instances
+  if [ "$scale_direction" == "up" ] && [ "$total_instances" -lt "$MAX_INSTANCES" ]; then
+    export scale_to=$((total_instances + SCALE_STEP))
+    echo "Scaling up to $scale_to"
+  elif [ "$scale_direction" == "down" ] && [ "$total_instances" -gt "$MIN_INSTANCES" ]; then
+    export scale_to=$((total_instances - SCALE_STEP))
+    echo "Scaling down to $scale_to"
+  else
+    export scale_to="$total_instances"
+    echo "Remain at the $scale_direction scale level $scale_to"
+  fi
+}

From 9484740680519e4f31cf3c7948d1ab1ea447ccbb Mon Sep 17 00:00:00 2001
From: Fuhu Xia <fxia@reisystems.com>
Date: Wed, 7 Feb 2024 09:11:36 -0500
Subject: [PATCH 2/3] handle edge case

---
 bin/check-and-renew    | 7 ++++---
 bin/scale_calculate.sh | 8 +++++++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/bin/check-and-renew b/bin/check-and-renew
index fd0eb772e..e6b9721a7 100755
--- a/bin/check-and-renew
+++ b/bin/check-and-renew
@@ -121,12 +121,13 @@ fi
 
 if [[ "$action" == "scale" ]]; then
   if [[ $ok_to_proceed == "YES" ]]; then
-    # set scale_to to a safe number but it should not be used
-    # since the scale_calculate.sh always export a new number
-    scale_to=7
     # call scale functions to get the scale_to value
     source "$(dirname "$0")/scale_calculate.sh"
     set_scale_number
+    if [[ -z $scale_to ]]; then
+      echo "Scale to value not set. Cancelling scale."
+      exit 1
+    fi
     # get $scale_to from the environment vars
     echo "Scaling $app_to_check to $scale_to"
     cf scale "$app_to_check" -i "$scale_to"
diff --git a/bin/scale_calculate.sh b/bin/scale_calculate.sh
index dd184f93a..2a90ef62d 100644
--- a/bin/scale_calculate.sh
+++ b/bin/scale_calculate.sh
@@ -45,7 +45,13 @@ function set_scale_direction {
 
 function set_scale_number {
   # get the total number of instances
-  total_instances=$(echo "$app_status" | grep -oP 'instances:\s+\K\d+')
+  total_instances=$(echo "$app_status" | grep '^instances:' | awk '{print $2}' | cut -d '/' -f 2)
+  # exit if the total instances is not a number
+  if ! [[ "$total_instances" =~ ^[0-9]+$ ]]; then
+    echo "Total instances is not a number. Exiting. Here is the output of the app status:"
+    echo "$app_status"
+    exit 1
+  fi
   echo "Current total instances: $total_instances"
   # call the scale direction
   set_scale_direction

From 1248441f18a97d09940c0a84a561e484d73a0670 Mon Sep 17 00:00:00 2001
From: Fuhu Xia <fxia@reisystems.com>
Date: Wed, 7 Feb 2024 09:21:44 -0500
Subject: [PATCH 3/3] handle edge case

---
 bin/scale_calculate.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/scale_calculate.sh b/bin/scale_calculate.sh
index 2a90ef62d..37a49f5d9 100644
--- a/bin/scale_calculate.sh
+++ b/bin/scale_calculate.sh
@@ -61,7 +61,8 @@ function set_scale_number {
     export scale_to=$((total_instances + SCALE_STEP))
     echo "Scaling up to $scale_to"
   elif [ "$scale_direction" == "down" ] && [ "$total_instances" -gt "$MIN_INSTANCES" ]; then
-    export scale_to=$((total_instances - SCALE_STEP))
+    # export the scale_to value but make sure scale_to is large than 1
+    export scale_to=$((total_instances - SCALE_STEP < 1 ? 1 : total_instances - SCALE_STEP))
     echo "Scaling down to $scale_to"
   else
     export scale_to="$total_instances"