mspnp · v-fearam · Dec 16, 2022 · Dec 30, 2022
diff --git a/cluster-stamp.bicep b/cluster-stamp.bicep
@@ -52,6 +52,10 @@ param gitOpsBootstrappingRepoHttpsUrl string
 @minLength(1)
 param gitOpsBootstrappingRepoBranch string = 'main'
 
+@description('Subnet resource Id for the AKS jumpbox subnet')
+@minLength(79)
+param aksJumpboxSubnetResourceId string
+
 /*** VARIABLES ***/
 
 var kubernetesVersion = '1.23.12'
@@ -102,6 +106,24 @@ var pdEnforceImageSourceId = tenantResourceId('Microsoft.Authorization/policyDef
 
 /*** EXISTING RESOURCE GROUP RESOURCES ***/
 
+@description('The resource group name containing virtual network in which Jumpbox will be dropped.')
+resource rgJumpBoxVirutalNetwork 'Microsoft.Resources/resourceGroups@2021-04-01' existing = {
+  scope: subscription()
+  name: split(aksJumpboxSubnetResourceId, '/')[4]
+}
+
+@description('Jumpbox Spoke Virtual Network')
+resource aksJumpBoxSpokeVnet 'Microsoft.Network/virtualNetworks@2022-01-01' existing = {
+  scope: rgJumpBoxVirutalNetwork
+  name: split(aksJumpboxSubnetResourceId, '/')[8]
+}
+
+@description('Jumpbox subnet')
+resource aksJumpboxSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-01-01' existing = {
+  parent: aksJumpBoxSpokeVnet
+  name: last(split(aksJumpboxSubnetResourceId, '/'))
+}
+
 @description('Spoke resource group')
 resource spokeResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = {
   scope: subscription()
@@ -123,11 +145,6 @@ resource vnetSpoke 'Microsoft.Network/virtualNetworks@2022-01-01' existing = {
     name: 'snet-privatelinkendpoints'
   }
 
-  // spoke virtual network's subnet for managment ops
-  resource snetManagmentOps 'subnets' existing = {
-    name: 'snet-management-ops'
-  }
-
   // spoke virtual network's subnet for managment acr agent pools
   resource snetManagmentCrAgents 'subnets' existing = {
     name: 'snet-management-acragents'
@@ -677,7 +694,7 @@ resource vmssJumpboxes 'Microsoft.Compute/virtualMachineScaleSets@2020-12-01' =
                     privateIPAddressVersion: 'IPv4'
                     publicIPAddressConfiguration: null
                     subnet: {
-                      id: vnetSpoke::snetManagmentOps.id
+                      id: aksJumpboxSubnet.id
                     }
                   }
                 }

diff --git a/docs/deploy/06-aks-jumpboximage.md b/docs/deploy/06-aks-jumpboximage.md
@@ -34,7 +34,7 @@ You are going to be using Azure Image Builder to generate a Kubernetes-specific
 
 ### Deploy the spoke
 
-1. Create the AKS jump box image builder network spoke.
+1. Create the AKS jump box image builder and JumpBox network spoke.
 
    ```bash
    RESOURCEID_VNET_HUB=$(az deployment group show -g rg-enterprise-networking-hubs -n hub-region.v0 --query properties.outputs.hubVnetId.value -o tsv)
@@ -52,8 +52,10 @@ You are going to be using Azure Image Builder to generate a Kubernetes-specific
    ```bash
    RESOURCEID_SUBNET_AIB=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-00 --query properties.outputs.imageBuilderSubnetResourceId.value -o tsv)
 
+   RESOURCEID_SUBNET_JUMPBOX=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-00 --query properties.outputs.jumpboxSubnetResourceId.value -o tsv)
+
    # [This takes about five minutes to run.]
-   az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v1.bicep -p location=eastus2 aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}"
+   az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v1.bicep -p location=eastus2 aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}" aksJumpboxSubnetResourceId="${RESOURCEID_SUBNET_JUMPBOX}"
    ```
 
 ### Build and deploy the jump box image

diff --git a/docs/deploy/08-cluster-networking.md b/docs/deploy/08-cluster-networking.md
@@ -33,10 +33,8 @@ Your `rg-enterprise-networking-spokes` will be populated with the dedicated regi
    > :eyes: If you're curious to see what changed in the regional hub, [view the diff](https://diffviewer.azureedge.net/?l=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v1.bicep&r=https://raw.githubusercontent.com/mspnp/aks-baseline-regulated/main/networking/hub-region.v2.bicep).
 
    ```bash
-   RESOURCEID_SUBNET_AIB=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-00 --query properties.outputs.imageBuilderSubnetResourceId.value -o tsv)
    RESOURCEID_SUBNET_NODEPOOLS="['$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-01 --query "properties.outputs.nodepoolSubnetResourceIds.value | join ('\',\'',@)" -o tsv)']"
-   RESOURCEID_SUBNET_JUMPBOX=$(az deployment group show -g rg-enterprise-networking-spokes -n spoke-BU0001A0005-01 --query properties.outputs.jumpboxSubnetResourceId.value -o tsv)
-
+
    # [This takes about seven minutes to run.]
    az deployment group create -g rg-enterprise-networking-hubs -f networking/hub-region.v2.bicep -p location=eastus2 aksImageBuilderSubnetResourceId="${RESOURCEID_SUBNET_AIB}" nodepoolSubnetResourceIds="${RESOURCEID_SUBNET_NODEPOOLS}" aksJumpboxSubnetResourceId="${RESOURCEID_SUBNET_JUMPBOX}"
    ```

diff --git a/docs/deploy/09-pre-cluster-stamp.md b/docs/deploy/09-pre-cluster-stamp.md
@@ -39,7 +39,7 @@ An Azure user managed identity is going to be deployed. This identity is the ing
 
    ```bash
    # [This takes about eight minutes.]
-   az deployment group create -g rg-bu0001a0005 -f pre-cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} aksIngressControllerCertificate=${INGRESS_CONTROLLER_CERTIFICATE_BASE64} location=eastus2
+   az deployment group create -g rg-bu0001a0005 -f pre-cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} aksIngressControllerCertificate=${AKS_INGRESS_CONTROLLER_CERTIFICATE_BASE64} location=eastus2
    ```
 
 ## Quarantine pattern

diff --git a/docs/deploy/10-aks-cluster.md b/docs/deploy/10-aks-cluster.md
@@ -50,7 +50,7 @@ Now that the all the [necessary bootstrapping requirements are deployed](./09-pr
    echo GITOPS_CURRENT_BRANCH_NAME: $GITOPS_CURRENT_BRANCH_NAME
 
    # [This takes about 20 minutes to run.]
-   az deployment group create -g rg-bu0001a0005 -f cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_BASE64} jumpBoxImageResourceId=${RESOURCEID_IMAGE_JUMPBOX} jumpBoxCloudInitAsBase64=${CLOUDINIT_BASE64} gitOpsBootstrappingRepoHttpsUrl=${GITOPS_REPOURL} gitOpsBootstrappingRepoBranch=${GITOPS_CURRENT_BRANCH_NAME}
+   az deployment group create -g rg-bu0001a0005 -f cluster-stamp.bicep -p targetVnetResourceId=${RESOURCEID_VNET_CLUSTERSPOKE} clusterAdminAadGroupObjectId=${AADOBJECTID_GROUP_CLUSTERADMIN} k8sControlPlaneAuthorizationTenantId=${TENANTID_K8SRBAC} appGatewayListenerCertificate=${APP_GATEWAY_LISTENER_CERTIFICATE_BASE64} jumpBoxImageResourceId=${RESOURCEID_IMAGE_JUMPBOX} jumpBoxCloudInitAsBase64=${CLOUDINIT_BASE64} gitOpsBootstrappingRepoHttpsUrl=${GITOPS_REPOURL} gitOpsBootstrappingRepoBranch=${GITOPS_CURRENT_BRANCH_NAME}  aksJumpboxSubnetResourceId="${RESOURCEID_SUBNET_JUMPBOX}"
 
    # Or if you updated and wish to use the parameters file …
    #az deployment group create -g rg-bu0001a0005 -f cluster-stamp.bicep -p "@azuredeploy.parameters.prod.json"

diff --git a/networking/hub-region.v1.bicep b/networking/hub-region.v1.bicep
@@ -6,6 +6,10 @@ targetScope = 'resourceGroup'
 @minLength(79)
 param aksImageBuilderSubnetResourceId string
 
+@description('Subnet resource Id for the AKS jumpbox subnet')
+@minLength(79)
+param aksJumpboxSubnetResourceId string
+
 @allowed([
   'australiaeast'
   'canadacentral'
@@ -70,6 +74,24 @@ resource aksImageBuilderSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-0
   name: last(split(aksImageBuilderSubnetResourceId, '/'))
 }
 
+@description('The resource group name containing virtual network in which Jumpbox will be dropped.')
+resource rgJumpBoxVirutalNetwork 'Microsoft.Resources/resourceGroups@2021-04-01' existing = {
+  scope: subscription()
+  name: split(aksJumpboxSubnetResourceId, '/')[4]
+}
+
+@description('Jumpbox Spoke Virtual Network')
+resource aksJumpBoxSpokeVnet 'Microsoft.Network/virtualNetworks@2022-01-01' existing = {
+  scope: rgJumpBoxVirutalNetwork
+  name: split(aksJumpboxSubnetResourceId, '/')[8]
+}
+
+@description('Jumpbox subnet')
+resource aksJumpboxSubnet 'Microsoft.Network/virtualNetworks/subnets@2022-01-01' existing = {
+  parent: aksJumpBoxSpokeVnet
+  name: last(split(aksJumpboxSubnetResourceId, '/'))
+}
+
 resource networkWatcherResourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' existing = if (deployFlowLogResources) {
   scope: subscription()
   name: 'networkWatcherRG'
@@ -480,6 +502,17 @@ resource imageBuilder_ipgroups 'Microsoft.Network/ipGroups@2021-05-01' = {
   }
 }
 
+@description('This holds IP addresses of known AKS Jumpbox image building subnets in attached spokes.')
+resource jumpbox_ipgroups 'Microsoft.Network/ipGroups@2021-05-01' = {
+  name: 'ipg-${location}-AksJumpboxes'
+  location: location
+  properties: {
+    ipAddresses: [
+      aksJumpboxSubnet.properties.addressPrefix
+    ]
+  }
+}
+
 resource region_flowlog_storageAccount_diagnosticSettings 'Microsoft.Insights/diagnosticSettings@2017-05-01-preview' = if (deployFlowLogResources) {
   name: 'default'
   scope: flowlogs_storageAccount::blobStorage

diff --git a/networking/hub-region.v2.bicep b/networking/hub-region.v2.bicep
@@ -907,6 +907,22 @@ resource hubFirewall 'Microsoft.Network/azureFirewalls@2021-05-01' = {
                 'login.microsoftonline.com'
               ]
             }
+            {
+              name: 'api-server-address'
+              description: 'Allow jumpboxes to perform kubectl.'
+              sourceIpGroups: [
+                aksJumpbox_ipgroup.id
+              ]
+              protocols: [
+                {
+                  protocolType: 'Https'
+                  port: 443
+                }
+              ]
+              targetFqdns: [
+                '*.privatelink.${location}.azmk8s.io'
+              ]
+            }
             {
               name: 'az-management-api'
               description: 'Allow jumpboxes to communicate with Azure management APIs.'