Fail script on background process failure (#10062)

add logic to exit if background process fails Signed-off-by: Jesse Nelson <[email protected]>
hiero-ledger · Jan 8, 2025 · 38d240b · 38d240b
1 parent 45e4e78
commit 38d240b
Show file tree

Hide file tree

Showing 3 changed files with 17 additions and 2 deletions.
diff --git a/docs/runbook/scripts/restore-volume-snapshot.sh b/docs/runbook/scripts/restore-volume-snapshot.sh
@@ -188,7 +188,7 @@ function replaceDisks() {
     local snapshotFullName="projects/${GCP_SNAPSHOT_PROJECT}/global/snapshots/${snapshotName}"
     log "Recreating disk ${diskName} in ${GCP_PROJECT} with snapshot ${snapshotName}"
     gcloud compute disks delete "${diskName}" --project "${GCP_PROJECT}" --zone "${diskZone}" --quiet
-    gcloud compute disks create "${diskName}" --project "${GCP_PROJECT}" --zone "${diskZone}" --source-snapshot "${snapshotFullName}" --type=pd-balanced --quiet &
+    watchInBackground "$$" gcloud compute disks create "${diskName}" --project "${GCP_PROJECT}" --zone "${diskZone}" --source-snapshot "${snapshotFullName}" --type=pd-balanced --quiet &
   done
 
   log "Waiting for disks to be created"

diff --git a/docs/runbook/scripts/utils.sh b/docs/runbook/scripts/utils.sh
@@ -2,6 +2,21 @@
 set -euo pipefail
 shopt -s expand_aliases
 
+function backgroundErrorHandler() {
+  log "Background command failure signalled. Exiting..."
+  trap - INT
+  wait
+  exit 1
+}
+
+trap backgroundErrorHandler INT
+
+function watchInBackground() {
+  local -ir pid="$1"
+  shift
+  "$@" || kill -INT -- -"$pid"
+}
+
 function checkCitusMetadataSyncStatus() {
   TIMEOUT_SECONDS=600
   local namespace="${1}"

diff --git a/docs/runbook/scripts/volume-snapshot.sh b/docs/runbook/scripts/volume-snapshot.sh
@@ -61,7 +61,7 @@ do
   SNAPSHOT_REGION=$(echo "${DISK_NODE_ID}" | cut -d '-' -f 2-3)
   DISK_ZONE=$(echo "${DISK_NODE_ID}" | cut -d '-' -f 2-4)
   log "Creating snapshot ${SNAPSHOT_NAME} for ${diskName} with ${SNAPSHOT_DESCRIPTION} in ${SNAPSHOT_REGION}"
-  gcloud compute snapshots create "${SNAPSHOT_NAME}" \
+  watchInBackground "$$" gcloud compute snapshots create "${SNAPSHOT_NAME}" \
   --project="${GCP_PROJECT}"  \
   --source-disk="${diskName}" \
   --source-disk-zone="${DISK_ZONE}" \