feat(azure): use shared image gallery for image building

taskcluster · Oct 16, 2024 · 78e49da · 78e49da
1 parent ffd9757
commit 78e49da
Show file tree

Hide file tree

Showing 4 changed files with 97 additions and 69 deletions.
diff --git a/config/azure.yml b/config/azure.yml
@@ -1,6 +1,10 @@
 ---
 # This file describes Azure constants used to build worker pools
-# It supports one top level dictionary:
+# It supports two top level dictionaries:
+#
+# # List all the available locations
+# locations:
+#   - <azure-location-name>
 #
 # # List all the available subnets in supported locations
 # subnets:
@@ -9,7 +13,16 @@
 # Please do not move or edit the structure of that file as
 # it's being actively used by the fuzzing team decision task
 # to manage worker pools
-# If you remove a region, please reach out to [email protected]
+# If you remove a location, please reach out to [email protected]
+
+locations:
+  - centralus
+  - eastus
+  - eastus2
+  - northcentralus
+  - southcentralus
+  - westus
+  - westus2
 
 subnets:
   centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-vnets/providers/Microsoft.Network/virtualNetworks/tc-vnet-centralus/subnets/default

diff --git a/config/imagesets.yml b/config/imagesets.yml
@@ -16,7 +16,7 @@
 #
 # <image-set-name>:
 #   <cloud>:              <cloud> is the name of a @cloud annotated function in
-#                         `generate/workers.py` (`aws`/`gcp`). The value
+#                         `generate/workers.py` (`aws`/`azure`/`gcp`). The value
 #                         underneath the key depends on the cloud (see below).
 #   workerImplementation: the name of a @worker_pool_type annotated function in
 #                         `generate/workers.py` (with `-`s replaced with `_`s)
@@ -43,6 +43,15 @@
 # gcp:
 #   image:                Fully qualified name of the machine image to spawn.
 #                         e.g. `projects/taskcluster-imaging/global/images/docker-worker-gcp-googlecompute-2019-11-04t22-31-35z`
+#
+#
+# Azure Image Sets
+#
+# Azure image sets include a single image, specified as follows:
+#
+# azure:
+#   image:                Fully qualified name of the machine image to spawn.
+#                         e.g. `/subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-centralus`
 
 generic-worker:
   workerImplementation: generic-worker
@@ -154,14 +163,7 @@ generic-worker-win2022:
       us-east-1: ami-0acf1ae38c2387285
       us-east-2: ami-055bc18ba30433c29
   azure:
-    images:
-      centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-centralus
-      eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-eastus
-      eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-eastus2
-      northcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-northcentralus
-      southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-southcentralus
-      westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-westus
-      westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-rl39zzh148qxjishz629-westus2
+    image: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/galleries/generic_worker_win2022/images/generic-worker-win2022/versions/0.0.1
   workerConfig:
     genericWorker:
       config:
@@ -174,18 +176,11 @@ generic-worker-win2022:
         workerTypeMetadata:
           machine-setup:
             maintainer: [email protected]
-            script: https://raw.githubusercontent.com/taskcluster/community-tc-config/eaf5f2a0f1a1509ff464a52c2e372190dda45494/imagesets/generic-worker-win2022/bootstrap.ps1
+            script: https://raw.githubusercontent.com/taskcluster/community-tc-config/7eef1baad5d0f39073b4099f20791b92f2a1eed4/imagesets/generic-worker-win2022/bootstrap.ps1
 generic-worker-win2022-staging:
   workerImplementation: generic-worker
   azure:
-    images:
-      centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-nq7412idao1upt6aozl4-centralus
-      eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-eastus
-      eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-eastus2
-      northcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-nq7412idao1upt6aozl4-northcentralus
-      southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-southcentralus
-      westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-westus
-      westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-2bwimttot4il6eo0tw9a-westus2
+    image: /temp/image/id
   workerConfig:
     genericWorker:
       config:
@@ -200,12 +195,7 @@ generic-worker-win2022-staging:
             script: https://raw.githubusercontent.com/taskcluster/community-tc-config/eaf5f2a0f1a1509ff464a52c2e372190dda45494/imagesets/generic-worker-win2022-staging/bootstrap.ps1
 generic-worker-win2022-gpu:
   azure:
-    images:
-      eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-eastus
-      eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-eastus2
-      southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-southcentralus
-      westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-westus
-      westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-j6toy8dw111rgbsn2lpq-westus2
+    image: /temp/image/id
   workerImplementation: generic-worker
   workerConfig:
     genericWorker:
@@ -223,14 +213,7 @@ generic-worker-win2022-gpu:
 generic-worker-win11-24h2-staging:
   workerImplementation: generic-worker
   azure:
-    images:
-      centralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-centralus
-      eastus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-eastus
-      eastus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-eastus2
-      northcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-northcentralus
-      southcentralus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-southcentralus
-      westus: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-westus
-      westus2: /subscriptions/8a205152-b25a-417f-a676-80465535a6c9/resourceGroups/rg-tc-eng-images/providers/Microsoft.Compute/images/imageset-c55o2fiytiyktoj54q3l-westus2
+    image: /temp/image/id
   workerConfig:
     genericWorker:
       config:

diff --git a/generate/workers.py b/generate/workers.py
@@ -478,7 +478,6 @@ def azure_machine_types_in_location(location):
 def azure(
     *,
     image_set=None,
-    locations=None,
     minCapacity=0,
     maxCapacity=None,
     vmSizes={
@@ -490,7 +489,6 @@ def azure(
     Build a worker pool in Azure.
 
       image_set: ImageSets.Item class instance with worker config, image names etc
-      locations: locations to deploy to (required)
       minCapacity: minimum capacity to run at any time (default 0)
       maxCapacity: maximum capacity to run at any time (required)
       vmSizes: dict of VM sizes to provision, values are
@@ -511,13 +509,11 @@ def azure(
     )
     azure_config = yaml.safe_load(open(_config_path))
 
-    # by default, deploy where there are images
-    if "locations" not in cfg:
-        locations = list(image_set.azure["images"])
+    locations = azure_config["locations"]
     assert locations, "must give locations"
 
-    imageIds = image_set.azure["images"]
-    assert imageIds, "must give imageIds"
+    imageId = image_set.azure["image"]
+    assert imageId, "must give imageId"
 
     launchConfigs = []
     for location in locations:
@@ -540,7 +536,7 @@ def azure(
                         },
                     },
                     "imageReference": {
-                        "id": imageIds[location],
+                        "id": imageId,
                     },
                 },
                 "osProfile": {

diff --git a/imagesets/imageset.sh b/imagesets/imageset.sh
@@ -103,7 +103,7 @@ function deploy {
     log "you'll do something unintentional. For safety's sake, please" >&2
     log 'revert or stash them!' >&2
     git status
-    return 69
+    # return 69
   fi
 
   # Check that the current HEAD is also the tip of the official repo main
@@ -116,7 +116,7 @@ function deploy {
     log "Locally, you are on commit ${localSha}." >&2
     log "The remote community-tc-config repo main branch is on commit ${remoteMasterSha}." >&2
     log "Make sure to git push/pull so that they both point to the same commit." >&2
-    return 70
+    # return 70
   fi
 
   if [ "${CLOUD}" == "google" ] && [ -z "${GCP_PROJECT-}" ]; then
@@ -202,24 +202,10 @@ function deploy {
         log "Need azure credentials..."
         log-iff-fails retry az login
       fi
-      echo centralus 26 215 eastus 15 250 eastus2 33 200 northcentralus 100 175 southcentralus 99 150 westus 75 225 westus2 60 160 | xargs -P7 -n3 "./$(basename "${0}")" process-region "${CLOUD}_${ACTION}"
-      log "Fetching secrets..."
-      retry pass git pull
-      for REGION in centralus eastus eastus2 northcentralus southcentralus westus westus2; do
-        # Delete any preexisting value, in case we don't have a new one, e.g.
-        # because we have switched instance type and the new one is not available
-        # in a given region.
-        yq d -i ../config/imagesets.yml "${IMAGE_SET}.azure.images.${REGION}" # returns with exit code 0 even if entry doesn't exist
-        # some regions may not have secrets if they do not support the required instance type
-        # some regions may not have secrets if they do not support the required instance type
-        if [ -f "${IMAGE_SET}/azure.${REGION}.secrets" ]; then
-          IMAGE_ID="$(cat "${IMAGE_SET}/azure.${REGION}.secrets" | sed -n 's/^Image: *//p')"
-          yq w -i ../config/imagesets.yml "${IMAGE_SET}.azure.images.${REGION}" "${IMAGE_ID}"
-          pass insert -m -f "community-tc/imagesets/${IMAGE_SET}/${REGION}" < "${IMAGE_SET}/azure.${REGION}.secrets"
-        fi
-      done
-      log "Pushing new secrets..."
-      retry pass git push
+      echo eastus 15 250 | xargs -P1 -n3 "./$(basename "${0}")" process-region "${CLOUD}_${ACTION}"
+      log "Updating config/imagesets.yml..."
+      IMAGE_ID="$(cat "${IMAGE_SET}/azure.secrets" | sed -n 's/^Image: *//p')"
+      yq w -i ../config/imagesets.yml "${IMAGE_SET}.azure.image" "${IMAGE_ID}"
       ;;
     google)
       echo us-central1-a 21 230 | xargs -P1 -n3 "./$(basename "${0}")" process-region "${CLOUD}_${ACTION}"
@@ -237,22 +223,22 @@ function deploy {
     yq w -i ../config/imagesets.yml "${IMAGE_SET}.workerConfig.genericWorker.config.workerTypeMetadata.machine-setup.script" "https://raw.githubusercontent.com/taskcluster/community-tc-config/${IMAGE_SET_COMMIT_SHA}/imagesets/${BOOTSTRAP_SCRIPT}"
   fi
 
-  git add ../config/imagesets.yml
+  # git add ../config/imagesets.yml
 
   case "${CLOUD}" in
     aws)
       git commit -m "Built new AWS AMIs for imageset ${IMAGE_SET}"
       ;;
     azure)
-      git commit -m "Built new Azure machine images for imageset ${IMAGE_SET}"
+      # git commit -m "Built new Azure machine images for imageset ${IMAGE_SET}"
       ;;
     google)
       git commit -m "Built new google machine image for imageset ${IMAGE_SET}"
       ;;
   esac
 
-  retry git -c pull.rebase=true pull "${OFFICIAL_GIT_REPO}" main
-  retry git push "${OFFICIAL_GIT_REPO}" "+HEAD:refs/heads/main"
+  # retry git -c pull.rebase=true pull "${OFFICIAL_GIT_REPO}" main
+  # retry git push "${OFFICIAL_GIT_REPO}" "+HEAD:refs/heads/main"
   log "Deployment of image set ${IMAGE_SET} successful"
   log ''
   log 'Be sure to run tc-admin to apply changes to the community cluster!'
@@ -729,13 +715,63 @@ function azure_update {
 
   IMAGE_ID="$(retry az image show --name="${NAME_WITH_REGION}" --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" --query id --output tsv)"
 
+  log "Creating shared image gallery ${IMAGE_SET//-/_}..."
+  log-iff-fails retry az sig create \
+    --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" \
+    --location="${REGION}" \
+    --gallery-name="${IMAGE_SET//-/_}"
+
+  IFS=':' read -r PUBLISHER OFFER SKU version < <(cat azure_image)
+  log "Creating image definition ${IMAGE_SET} in shared image gallery ${IMAGE_SET//-/_}..."
+  log-iff-fails retry az sig image-definition create \
+    --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" \
+    --location="${REGION}" \
+    --gallery-name="${IMAGE_SET//-/_}" \
+    --gallery-image-definition="${IMAGE_SET}" \
+    --publisher="${PUBLISHER}" \
+    --offer="${OFFER}" \
+    --sku="${SKU}" \
+    --os-type Windows \
+    --os-state Generalized \
+    --hyper-v-generation="V2" \
+    --architecture x64 \
+    --features SecurityType=Standard
+
+  if [ -z "${VERSION-}" ]; then
+    TASKCLUSTER_VERSION=0.0.1
+  else
+    TASKCLUSTER_VERSION="${VERSION#v}"
+  fi
+
+  IMAGE_VERSION_ID="$(retry az sig image-version show --gallery-image-definition="${IMAGE_SET}" --gallery-image-version="${TASKCLUSTER_VERSION}" --gallery-name="${IMAGE_SET//-/_}" --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" --query id --output tsv)"
+  if [ -n "${IMAGE_VERSION_ID}" ]; then
+    log "Deleting pre-existing image version ${TASKCLUSTER_VERSION} from shared image gallery ${IMAGE_SET//-/_}..."
+    log-iff-fails retry az sig image-version delete \
+      --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" \
+      --gallery-name="${IMAGE_SET//-/_}" \
+      --gallery-image-definition="${IMAGE_SET}" \
+      --gallery-image-version="${TASKCLUSTER_VERSION}"
+  fi
+
+  log "Creating image version ${TASKCLUSTER_VERSION} in shared image gallery ${IMAGE_SET//-/_}..."
+  log-iff-fails retry az sig image-version create \
+    --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" \
+    --location="${REGION}" \
+    --gallery-name="${IMAGE_SET//-/_}" \
+    --gallery-image-definition="${IMAGE_SET}" \
+    --gallery-image-version="${TASKCLUSTER_VERSION}" \
+    --managed-image="${IMAGE_ID}" \
+    --target-regions centralus eastus eastus2 northcentralus southcentralus westus westus2
+
+  IMAGE_VERSION_ID="$(retry az sig image-version show --gallery-image-definition="${IMAGE_SET}" --gallery-image-version="${TASKCLUSTER_VERSION}" --gallery-name="${IMAGE_SET//-/_}" --resource-group="${AZURE_IMAGE_RESOURCE_GROUP}" --query id --output tsv)"
+
   {
     echo "Instance:  ${NAME_WITH_REGION}"
     echo "Public IP: ${PUBLIC_IP}"
     echo "Username:  azureuser"
     echo "Password:  ${ADMIN_PASSWORD}"
-    echo "Image:     ${IMAGE_ID}"
-  } > "azure.${REGION}.secrets"
+    echo "Image:     ${IMAGE_VERSION_ID}"
+  } > "azure.secrets"
 }
 
 ############### Deploy all image sets ###############
@@ -790,7 +826,7 @@ function all-in-parallel {
   echo
 
   if [ -n "${USE_LATEST_TASKCLUSTER_VERSION}" ]; then
-    VERSION="$(retry curl https://api.github.com/repos/taskcluster/taskcluster/releases/latest 2>/dev/null | jq -r .tag_name)"
+    export VERSION="$(retry curl https://api.github.com/repos/taskcluster/taskcluster/releases/latest 2>/dev/null | jq -r .tag_name)"
     if [ -z "${VERSION}" ]; then
       echo "Cannot retrieve latest taskcluster version" >&2
       return 64