Create one ASG per AZ

This is required for the cluster-autoscaler to properly work. It assumes that all nodes in a ASG are equal. This includes their AZ. Without this, the autoscaler won't be able to add nodes if a pod can't be scheduled due to AZ related constraints.
itskoko · Apr 16, 2018 · 2990156 · 2990156
1 parent 8e7e688
commit 2990156
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 17 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,6 +1,6 @@
 FROM debian:sid
 
-ENV KUBE_VERSION v1.8.4
+ENV KUBE_VERSION v1.10.1
 ENV KUBEADM_URL https://storage.googleapis.com/kubernetes-release/release/$KUBE_VERSION/bin/linux/amd64/kubeadm
 
 RUN apt-get -qy update && apt-get -qy install curl make awscli golang-cfssl jq \

diff --git a/kubernetes.yaml b/kubernetes.yaml
@@ -767,7 +767,27 @@ Resources:
               - "*"
             Condition:
               StringEquals:
-                "ec2:ResourceTag/aws:cloudformation:stack-id": !Ref WorkerPoolDefault
+                "ec2:ResourceTag/aws:cloudformation:stack-id": !Ref WorkerPoolDefaultA
+          - Effect: Allow
+            Action:
+              - "ec2:RevokeSecurityGroup*"
+              - "ec2:ModifyInstanceAttribute"
+              - "ec2:AuthorizeSecurityGroup*" # ...for the workers too
+            Resource:
+              - "*"
+            Condition:
+              StringEquals:
+                "ec2:ResourceTag/aws:cloudformation:stack-id": !Ref WorkerPoolDefaultB
+          - Effect: Allow
+            Action:
+              - "ec2:RevokeSecurityGroup*"
+              - "ec2:ModifyInstanceAttribute"
+              - "ec2:AuthorizeSecurityGroup*" # ...for the workers too
+            Resource:
+              - "*"
+            Condition:
+              StringEquals:
+                "ec2:ResourceTag/aws:cloudformation:stack-id": !Ref WorkerPoolDefaultC
           - Effect: Allow
             Action:
               - "ec2:AuthorizeSecurityGroup*"
@@ -1179,7 +1199,23 @@ Resources:
       ResourceRecords:
         - !GetAtt [ "ControllerELB", "DNSName" ]
 
-  WorkerPoolDefault:
+  WorkerPoolDefaultA:
+    Type: "AWS::CloudFormation::Stack"
+    Properties:
+      TemplateURL: !Sub
+        - "https://s3.amazonaws.com/${assetBucket}/${DomainName}/templates/worker.yaml"
+        - assetBucket: !Ref assetBucket
+          DomainName:  !Ref DomainName
+      Parameters:
+        DomainName:       !Ref DomainName
+        FeatureGates:     !Ref WorkerFeatureGates
+        CPUManagerPolicy: !Ref WorkerCPUManagerPolicy
+        assetBucket:      !Ref assetBucket
+        VPCID:            !Ref VPCID
+        PrivateSubnet:    !Ref PrivateSubnetA
+        KubeletImageTag:  !Ref KubeletImageTag
+
+  WorkerPoolDefaultB:
     Type: "AWS::CloudFormation::Stack"
     Properties:
       TemplateURL: !Sub
@@ -1192,9 +1228,23 @@ Resources:
         CPUManagerPolicy: !Ref WorkerCPUManagerPolicy
         assetBucket:      !Ref assetBucket
         VPCID:            !Ref VPCID
-        PrivateSubnetA:   !Ref PrivateSubnetA
-        PrivateSubnetB:   !Ref PrivateSubnetB
-        PrivateSubnetC:   !Ref PrivateSubnetC
+        PrivateSubnet:    !Ref PrivateSubnetB
+        KubeletImageTag:  !Ref KubeletImageTag
+
+  WorkerPoolDefaultC:
+    Type: "AWS::CloudFormation::Stack"
+    Properties:
+      TemplateURL: !Sub
+        - "https://s3.amazonaws.com/${assetBucket}/${DomainName}/templates/worker.yaml"
+        - assetBucket: !Ref assetBucket
+          DomainName:  !Ref DomainName
+      Parameters:
+        DomainName:       !Ref DomainName
+        FeatureGates:     !Ref WorkerFeatureGates
+        CPUManagerPolicy: !Ref WorkerCPUManagerPolicy
+        assetBucket:      !Ref assetBucket
+        VPCID:            !Ref VPCID
+        PrivateSubnet:    !Ref PrivateSubnetC
         KubeletImageTag:  !Ref KubeletImageTag
 
 Outputs:
@@ -1203,9 +1253,22 @@ Outputs:
     Export:
       Name: !Sub "${AWS::StackName}-ControllerRole"
   WorkerRole:
-    Value: !GetAtt WorkerPoolDefault.Outputs.WorkerRole
+    Value: !GetAtt WorkerPoolDefaultA.Outputs.WorkerRole
     Export:
       Name: !Sub "${AWS::StackName}-WorkerRole"
+  WorkerRoleA:
+    Value: !GetAtt WorkerPoolDefaultA.Outputs.WorkerRole
+    Export:
+      Name: !Sub "${AWS::StackName}-WorkerRoleA"
+  WorkerRoleB:
+    Value: !GetAtt WorkerPoolDefaultB.Outputs.WorkerRole
+    Export:
+      Name: !Sub "${AWS::StackName}-WorkerRoleB"
+  WorkerRoleC:
+    Value: !GetAtt WorkerPoolDefaultC.Outputs.WorkerRole
+    Export:
+      Name: !Sub "${AWS::StackName}-WorkerRoleC"
+
   WorkerTemplateURL:
     Value: !Sub
       - "https://s3.amazonaws.com/${assetBucket}/${DomainName}/templates/worker.yaml"

diff --git a/templates/worker.yaml b/templates/worker.yaml
@@ -52,13 +52,7 @@ Parameters:
     Description: Existing VPC with attached internet gateway to use for this cluster.
     Type: AWS::EC2::VPC::Id
 
-  PrivateSubnetA:
-    Type: String
-
-  PrivateSubnetB:
-    Type: String
-
-  PrivateSubnetC:
+  PrivateSubnet:
     Type: String
 
   assetBucket:
@@ -337,9 +331,7 @@ Resources:
     Type: AWS::AutoScaling::AutoScalingGroup
     Properties:
       VPCZoneIdentifier:
-        - !Ref PrivateSubnetA
-        - !Ref PrivateSubnetB
-        - !Ref PrivateSubnetC
+        - !Ref PrivateSubnet
       LaunchConfigurationName:
         Ref: WorkerLaunchConfiguration
       MaxSize: !Ref WorkerPoolSizeMax