Merge pull request GetSimpl#17 from Rippling/add-ability-to-disable-alb

Add ability to disable ALB and apply large cloud formation templates

cloudlift/config/region.py

@@ -2,15 +2,35 @@
 from cloudlift.exceptions import UnrecoverableException
 
 from cloudlift.config import EnvironmentConfiguration
-from cloudlift.config.logging import log_err
+
+local_cache = {}
+
 
 def get_region_for_environment(environment):
-    if environment:
-        return EnvironmentConfiguration(environment).get_config()[environment]['region']
-    else:
-        # Get the region from the AWS credentials used to execute cloudlift
-        aws_session = boto3.session.Session()
-        return aws_session.region_name
+    global local_cache
+    if 'region' not in local_cache:
+        if environment:
+            local_cache['region'] = EnvironmentConfiguration(environment).get_config()[environment]['region']
+        else:
+            # Get the region from the AWS credentials used to execute cloudlift
+            aws_session = boto3.session.Session()
+            local_cache['region'] = aws_session.region_name
+
+    return local_cache['region']
+
+
+def get_service_templates_bucket_for_environment(environment):
+    global local_cache
+
+    if 'bucket' not in local_cache:
+        config = EnvironmentConfiguration(environment).get_config()
+
+        if 'service_templates_bucket' not in config[environment]:
+            return None
+
+        local_cache['bucket'] = config[environment]['service_templates_bucket']
+
+    return local_cache['bucket']
 
 
 def get_client_for(resource, environment):

cloudlift/config/service_configuration.py

@@ -151,6 +151,9 @@ def _validate_changes(self, configuration):
                         "internal": {
                             "type": "boolean"
                         },
+                        "alb_enabled": {
+                            "type": "boolean",
+                        },
                         "restrict_access_to": {
                             "type": "array",
                             "items": {
@@ -168,7 +171,8 @@ def _validate_changes(self, configuration):
                     "required": [
                         "internal",
                         "restrict_access_to",
-                        "container_port"
+                        "container_port",
+                        "alb_enabled"
                     ]
                 },
                 "memory_reservation": {
@@ -267,6 +271,7 @@ def _default_service_configuration(self):
                 pascalcase(self.service_name): {
                     u'http_interface': {
                         u'internal': False,
+                        u'alb_enabled': True,
                         u'restrict_access_to': [u'0.0.0.0/0'],
                         u'container_port': 80,
                         u'health_check_path': u'/elb-check'

cloudlift/deployment/changesets.py

@@ -5,6 +5,7 @@
 import click
 
 from cloudlift.config.logging import log, log_bold, log_err
+from cloudlift.deployment.cloud_formation_stack import prepare_stack_options_for_template
 
 
 def create_change_set(client, service_template_body, stack_name,
@@ -17,13 +18,14 @@ def create_change_set(client, service_template_body, stack_name,
             'ParameterKey': 'KeyPair',
             'ParameterValue': key_name
         })
+    options = prepare_stack_options_for_template(service_template_body, environment, stack_name)
     create_change_set_res = client.create_change_set(
         StackName=stack_name,
         ChangeSetName="cg"+uuid.uuid4().hex,
-        TemplateBody=service_template_body,
         Parameters=change_set_parameters,
         Capabilities=['CAPABILITY_NAMED_IAM'],
-        ChangeSetType='UPDATE'
+        ChangeSetType='UPDATE',
+        **options,
     )
     log("Changeset creation initiated. Checking the progress...")
     change_set = client.describe_change_set(

cloudlift/deployment/cloud_formation_stack.py

@@ -0,0 +1,48 @@
+from cloudlift.config.region import (get_resource_for,
+                                     get_service_templates_bucket_for_environment,
+                                     get_region_for_environment)
+
+from tempfile import NamedTemporaryFile
+from cloudlift.config.logging import log_intent
+
+TEMPLATE_BODY_LIMIT = 51200
+
+
+def prepare_stack_options_for_template(template_body, environment, stack_name):
+    options = {}
+    if len(template_body) <= TEMPLATE_BODY_LIMIT:
+        options['TemplateBody'] = template_body
+    else:
+        s3 = get_resource_for('s3', environment)
+        bucket_name = get_service_templates_bucket_for_environment(environment)
+
+        if not bucket_name:
+            from cloudlift.exceptions import UnrecoverableException
+            raise UnrecoverableException(
+                'Configure "service_templates_bucket" in environment configuration to apply changes')
+
+        bucket = s3.Bucket(bucket_name)
+        region = get_region_for_environment(environment)
+
+        if bucket not in s3.buckets.all():
+            bucket.create(
+                ACL='private',
+                Bucket=bucket_name,
+                CreateBucketConfiguration={
+                    'LocationConstraint': region,
+                }
+            )
+            s3.BucketVersioning(bucket_name).enable()
+
+        with NamedTemporaryFile() as f:
+            f.write(template_body.encode())
+            path = '{}/{}.template'.format(environment, stack_name)
+            bucket.upload_file(f.name, path)
+
+            template_url = 'https://s3-{}.amazonaws.com/{}/{}'.format(region, bucket_name, path)
+
+            log_intent('Using S3 URL from deploying stack: {}'.format(template_url))
+
+            options['TemplateURL'] = template_url
+
+    return options

cloudlift/deployment/environment_creator.py

@@ -11,6 +11,7 @@
 from cloudlift.deployment.cluster_template_generator import ClusterTemplateGenerator
 from cloudlift.config.logging import log, log_bold, log_err
 from cloudlift.deployment.progress import get_stack_events, print_new_events
+from cloudlift.deployment.cloud_formation_stack import prepare_stack_options_for_template
 
 
 class EnvironmentCreator(object):
@@ -49,9 +50,10 @@ def run(self):
                 self.client,
                 self.cluster_name
             )
+            options = prepare_stack_options_for_template(
+                environment_stack_template_body, self.environment, self.cluster_name)
             environment_stack = self.client.create_stack(
                 StackName=self.cluster_name,
-                TemplateBody=environment_stack_template_body,
                 Parameters=[
                     {
                         'ParameterKey': 'KeyPair',
@@ -64,10 +66,11 @@ def run(self):
                 ],
                 OnFailure='DO_NOTHING',
                 Capabilities=['CAPABILITY_NAMED_IAM'],
+                **options,
             )
             log_bold("Submitted to cloudformation. Checking progress...")
             self.__print_progress()
-            log_bold(self.cluster_name+" stack created. ID: " +
+            log_bold(self.cluster_name + " stack created. ID: " +
                      environment_stack['StackId'])
 
     def run_update(self, update_ecs_agents):

cloudlift/deployment/service_creator.py

@@ -15,6 +15,7 @@
 from cloudlift.config.logging import log, log_bold, log_err
 from cloudlift.deployment.progress import get_stack_events, print_new_events
 from cloudlift.deployment.service_template_generator import ServiceTemplateGenerator
+from cloudlift.deployment.cloud_formation_stack import prepare_stack_options_for_template
 
 
 class ServiceCreator(object):
@@ -50,15 +51,17 @@ def create(self):
         service_template_body = template_generator.generate_service()
 
         try:
+            options = prepare_stack_options_for_template(
+                service_template_body, self.environment, self.stack_name)
             self.client.create_stack(
                 StackName=self.stack_name,
-                TemplateBody=service_template_body,
                 Parameters=[{
                     'ParameterKey': 'Environment',
                     'ParameterValue': self.environment,
                 }],
                 OnFailure='DO_NOTHING',
                 Capabilities=['CAPABILITY_NAMED_IAM'],
+                **options,
             )
             log_bold("Submitted to cloudformation. Checking progress...")
             self._print_progress()

cloudlift/deployment/service_template_generator.py

@@ -237,20 +237,26 @@ def _add_service(self, service_name, config):
         )
 
         if 'http_interface' in config:
-            alb, lb, service_listener, alb_sg = self._add_alb(cd, service_name, config, launch_type)
+            lb, target_group_name = self._add_ecs_lb(cd, service_name, config, launch_type)
+            alb_enabled = 'alb_enabled' in config['http_interface'] and config['http_interface']['alb_enabled']
+            if alb_enabled:
+                alb, service_listener, alb_sg = self._add_alb(service_name, config, target_group_name)
 
             if launch_type == self.LAUNCH_TYPE_FARGATE:
                 # if launch type is ec2, then services inherit the ec2 instance security group
                 # otherwise, we need to specify a security group for the service
+                security_group_ingress = {
+                    'IpProtocol': 'TCP',
+                    'ToPort': int(config['http_interface']['container_port']),
+                    'FromPort': int(config['http_interface']['container_port']),
+                }
+                if alb_enabled:
+                    security_group_ingress['SourceSecurityGroupId'] = Ref(alb_sg)
+
                 service_security_group = SecurityGroup(
                     pascalcase("FargateService" + self.env + service_name),
                     GroupName=pascalcase("FargateService" + self.env + service_name),
-                    SecurityGroupIngress=[{
-                        'IpProtocol': 'TCP',
-                        'SourceSecurityGroupId': Ref(alb_sg),
-                        'ToPort': int(config['http_interface']['container_port']),
-                        'FromPort': int(config['http_interface']['container_port']),
-                    }],
+                    SecurityGroupIngress=[security_group_ingress],
                     VpcId=Ref(self.vpc),
                     GroupDescription=pascalcase("FargateService" + self.env + service_name)
                 )
@@ -274,31 +280,39 @@ def _add_service(self, service_name, config):
                     'Role': Ref(self.ecs_service_role),
                     'PlacementStrategies': self.PLACEMENT_STRATEGIES
                 }
+
+            if alb_enabled:
+                print(service_listener.title)
+                launch_type_svc['DependsOn'] = service_listener.title
+
             svc = Service(
                 service_name,
                 LoadBalancers=[lb],
                 Cluster=self.cluster_name,
                 TaskDefinition=Ref(td),
                 DesiredCount=desired_count,
-                DependsOn=service_listener.title,
                 LaunchType=launch_type,
                 **launch_type_svc,
             )
+
             self.template.add_output(
                 Output(
                     service_name + 'EcsServiceName',
                     Description='The ECS name which needs to be entered',
                     Value=GetAtt(svc, 'Name')
                 )
             )
-            self.template.add_output(
-                Output(
-                    service_name + "URL",
-                    Description="The URL at which the service is accessible",
-                    Value=Sub("https://${" + alb.name + ".DNSName}")
-                )
-            )
+
             self.template.add_resource(svc)
+
+            if alb_enabled:
+                self.template.add_output(
+                    Output(
+                        service_name + "URL",
+                        Description="The URL at which the service is accessible",
+                        Value=Sub("https://${" + alb.name + ".DNSName}")
+                    )
+                )
         else:
             launch_type_svc = {}
             if launch_type == self.LAUNCH_TYPE_FARGATE:
@@ -358,7 +372,48 @@ def _gen_log_config(self, service_name):
             }
         )
 
-    def _add_alb(self, cd, service_name, config, launch_type):
+    def _add_ecs_lb(self, cd, service_name, config, launch_type):
+        target_group_name = "TargetGroup" + service_name
+        health_check_path = config['http_interface']['health_check_path'] if 'health_check_path' in config[
+            'http_interface'] else "/elb-check"
+        if config['http_interface']['internal']:
+            target_group_name = target_group_name + 'Internal'
+
+        target_group_config = {}
+        if launch_type == self.LAUNCH_TYPE_FARGATE:
+            target_group_config['TargetType'] = 'ip'
+
+        service_target_group = TargetGroup(
+            target_group_name,
+            HealthCheckPath=health_check_path,
+            HealthyThresholdCount=2,
+            HealthCheckIntervalSeconds=30,
+            TargetGroupAttributes=[
+                TargetGroupAttribute(
+                    Key='deregistration_delay.timeout_seconds',
+                    Value='30'
+                )
+            ],
+            VpcId=Ref(self.vpc),
+            Protocol="HTTP",
+            Matcher=Matcher(HttpCode="200-399"),
+            Port=int(config['http_interface']['container_port']),
+            HealthCheckTimeoutSeconds=10,
+            UnhealthyThresholdCount=3,
+            **target_group_config
+        )
+
+        self.template.add_resource(service_target_group)
+
+        lb = LoadBalancer(
+            ContainerName=cd.Name,
+            TargetGroupArn=Ref(service_target_group),
+            ContainerPort=int(config['http_interface']['container_port'])
+        )
+
+        return lb, target_group_name
+
+    def _add_alb(self, service_name, config, target_group_name):
         sg_name = 'SG' + self.env + service_name
         svc_alb_sg = SecurityGroup(
             re.sub(r'\W+', '', sg_name),
@@ -370,6 +425,7 @@ def _add_alb(self, cd, service_name, config, launch_type):
             GroupDescription=Sub(service_name + "-alb-sg")
         )
         self.template.add_resource(svc_alb_sg)
+
         alb_name = service_name + pascalcase(self.env)
         if config['http_interface']['internal']:
             alb_subnets = [
@@ -413,44 +469,6 @@ def _add_alb(self, cd, service_name, config, launch_type):
 
         self.template.add_resource(alb)
 
-        target_group_name = "TargetGroup" + service_name
-        health_check_path = config['http_interface']['health_check_path'] if 'health_check_path' in config['http_interface'] else "/elb-check"
-        if config['http_interface']['internal']:
-            target_group_name = target_group_name + 'Internal'
-
-        target_group_config = {}
-        if launch_type == self.LAUNCH_TYPE_FARGATE:
-            target_group_config['TargetType'] = 'ip'
-
-        service_target_group = TargetGroup(
-            target_group_name,
-            HealthCheckPath=health_check_path,
-            HealthyThresholdCount=2,
-            HealthCheckIntervalSeconds=30,
-            TargetGroupAttributes=[
-                TargetGroupAttribute(
-                    Key='deregistration_delay.timeout_seconds',
-                    Value='30'
-                )
-            ],
-            VpcId=Ref(self.vpc),
-            Protocol="HTTP",
-            Matcher=Matcher(HttpCode="200-399"),
-            Port=int(config['http_interface']['container_port']),
-            HealthCheckTimeoutSeconds=10,
-            UnhealthyThresholdCount=3,
-            **target_group_config
-        )
-
-        self.template.add_resource(service_target_group)
-        # Note: This is a ECS Loadbalancer definition. Not an ALB.
-        # Defining this causes the target group to add a target to the correct
-        # port in correct ECS cluster instance for the service container.
-        lb = LoadBalancer(
-            ContainerName=cd.Name,
-            TargetGroupArn=Ref(service_target_group),
-            ContainerPort=int(config['http_interface']['container_port'])
-        )
         target_group_action = Action(
             TargetGroupArn=Ref(target_group_name),
             Type="forward"
@@ -462,7 +480,7 @@ def _add_alb(self, cd, service_name, config, launch_type):
             config['http_interface']['internal']
         )
         self._add_alb_alarms(service_name, alb)
-        return alb, lb, service_listener, svc_alb_sg
+        return alb, service_listener, svc_alb_sg
 
     def _add_service_listener(self, service_name, target_group_action,
                               alb, internal):