Add support for sidecars to render Cloudformation templates (GetSimpl#35

)

* Add support for sidecars to render Cloudformation templates

* Accept sidecar name as an argument for getting and setting configs in parameter store

* Wire up container configurations in editing, deploying and cloud formation operations

* Fix sidecar config injection failures

* Add test for build_config

* Fix tests on recursive fetching from parameter store

* Retain old container name convention for backwards compatibility

* Add sample env for redis sidecar

* Ensure the container to deploy is present in containerDefinitions

* Mark sidecars as non essential containers

* Check if there are sample env files based on glob patterns

* Remove sidecar env sample file

* Allow service with empty configuration during deployment

* Add integration test to access sidecar

* Bump minor version

* Skip tests that are failing in master

cloudlift/__init__.py

@@ -97,8 +97,10 @@ def update_environment(environment, update_ecs_agents):
 in parameter store")
 @_require_name
 @_require_environment
-def edit_config(name, environment):
-    editor.edit_config(name, environment)
+@click.option('--sidecar', help='Choose which sidecar to edit the configuration. Defaults to the main container ' +
+                                'if not provided')
+def edit_config(name, environment, sidecar):
+    editor.edit_config(name, environment, sidecar)
 
 
 @cli.command()

cloudlift/config/parameter_store.py

@@ -19,64 +19,79 @@ def __init__(self, service_name, environment):
         # ssm_client = mfa_session.client('ssm')
         self.client = get_client_for('ssm', environment)
 
-    def get_existing_config_as_string(self):
-        environment_configs = self.get_existing_config()
+    def get_existing_config_as_string(self, sidecar_name=None):
+        environment_configs, sidecar_configs = self.get_existing_config()
+
+        result_configs = sidecar_configs.get(sidecar_name, {}) \
+            if sidecar_name is not None and sidecar_name != "" \
+            else environment_configs
+
         return '\n'.join('{}={}'.format(key, val) for key, val in sorted(
-            environment_configs.items()
+            result_configs.items()
         ))
 
     def get_existing_config(self):
         environment_configs = {}
+        sidecars_configs = {}
         next_token = None
         while True:
             if next_token:
                 response = self.client.get_parameters_by_path(
                     Path=self.path_prefix,
-                    Recursive=False,
+                    Recursive=True,
                     WithDecryption=True,
                     MaxResults=10,
                     NextToken=next_token
                 )
             else:
                 response = self.client.get_parameters_by_path(
                     Path=self.path_prefix,
-                    Recursive=False,
+                    Recursive=True,
                     WithDecryption=True,
                     MaxResults=10
                 )
             for parameter in response['Parameters']:
                 parameter_name = parameter['Name'].split(self.path_prefix)[1]
-                environment_configs[parameter_name] = parameter['Value']
+                if parameter_name.startswith('sidecars/'):
+                    sidecar_name, sidecar_parameter = parameter_name.replace('sidecars/', '', 1).split('/')
+                    if sidecar_name not in sidecars_configs:
+                        sidecars_configs[sidecar_name] = {}
+
+                    sidecars_configs[sidecar_name].update({sidecar_parameter: parameter['Value']})
+                else:
+                    environment_configs[parameter_name] = parameter['Value']
 
             try:
                 next_token = response['NextToken']
             except KeyError:
                 break
-        return environment_configs
+        return environment_configs, sidecars_configs
 
-    def set_config(self, differences):
+    def set_config(self, differences, sidecar_name=None):
         self._validate_changes(differences)
+        path_prefix = self.path_prefix if sidecar_name is None else '{}sidecars/{}/'.format(self.path_prefix,
+                                                                                            sidecar_name)
         for parameter_change in differences:
             if parameter_change[0] == 'change':
-                response = self.client.put_parameter(
-                    Name='%s%s' % (self.path_prefix, parameter_change[1]),
+                self.client.put_parameter(
+                    Name='%s%s' % (path_prefix, parameter_change[1]),
                     Value=parameter_change[2][1],
                     Type='SecureString',
                     KeyId='alias/aws/ssm',
                     Overwrite=True
                 )
             elif parameter_change[0] == 'add':
                 for added_parameter in parameter_change[2]:
-                    response = self.client.put_parameter(
-                        Name='%s%s' % (self.path_prefix, added_parameter[0]),
+                    self.client.put_parameter(
+                        Name='%s%s' % (path_prefix, added_parameter[0]),
                         Value=added_parameter[1],
                         Type='SecureString',
                         KeyId='alias/aws/ssm',
                         Overwrite=False
                     )
             elif parameter_change[0] == 'remove':
-                deleted_parameters = ["%s%s" % (self.path_prefix, item[0]) for item in parameter_change[2]]
-                response = self.client.delete_parameters(
+                deleted_parameters = ["%s%s" % (path_prefix, item[0]) for item in parameter_change[2]]
+                self.client.delete_parameters(
                     Names=deleted_parameters
                 )
 

cloudlift/config/service_configuration.py

@@ -258,6 +258,30 @@ def _validate_changes(self, configuration):
                         }
                     },
                 },
+                "sidecars": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "name": {
+                                "type": "string"
+                            },
+                            "image": {
+                                "type": "string"
+                            },
+                            "command": {
+                                "type": "array",
+                                "items": {
+                                    "type": "string"
+                                }
+                            },
+                            "memory_reservation": {
+                                "type": "number"
+                            }
+                        },
+                        "required": ["name", "image", "memory_reservation"]
+                    }
+                },
                 "system_controls": {
                     "type": "array",
                     "items": {

cloudlift/deployment/deployer.py

@@ -1,4 +1,4 @@
-import sys
+from os.path import basename
 from time import sleep, time
 from cloudlift.exceptions import UnrecoverableException
 from colorclass import Color
@@ -9,13 +9,21 @@
 from cloudlift.config.logging import log_bold, log_err, log_intent, log_with_color
 from datetime import datetime
 import boto3
+from glob import glob
+
+
+def find_essential_container(container_definitions):
+    for defn in container_definitions:
+        if defn[u'essential']:
+            return defn[u'name']
+
+    raise UnrecoverableException('no essential containers found')
 
 
 def deploy_new_version(client, cluster_name, ecs_service_name,
                        deploy_version_tag, service_name, sample_env_file_path,
                        timeout_seconds, env_name, color='white', complete_image_uri=None):
     log_bold("Starting to deploy " + ecs_service_name)
-    env_config = build_config(env_name, service_name, sample_env_file_path)
     deployment = DeployAction(client, cluster_name, ecs_service_name)
     if deployment.service.desired_count == 0:
         desired_count = 1
@@ -25,15 +33,26 @@ def deploy_new_version(client, cluster_name, ecs_service_name,
     task_definition = deployment.get_current_task_definition(
         deployment.service
     )
+
+    essential_container = find_essential_container(task_definition[u'containerDefinitions'])
+
+    container_configurations = build_config(
+        env_name,
+        service_name,
+        sample_env_file_path,
+        essential_container,
+    )
+
     if complete_image_uri is not None:
-        container_name = task_definition['containerDefinitions'][0]['name']
         task_definition.set_images(
+            essential_container,
             deploy_version_tag,
-            **{container_name: complete_image_uri}
+            **{essential_container: complete_image_uri}
         )
     else:
-        task_definition.set_images(deploy_version_tag)
+        task_definition.set_images(essential_container, deploy_version_tag)
     for container in task_definition.containers:
+        env_config = container_configurations.get(container[u'name'], [])
         task_definition.apply_container_environment(container, env_config)
     print_task_diff(ecs_service_name, task_definition.diff, color)
     new_task_definition = deployment.update_task_definition(task_definition)
@@ -54,26 +73,54 @@ def deploy_and_wait(deployment, new_task_definition, color, timeout_seconds):
     return wait_for_finish(deployment, existing_events, color, deploy_end_time)
 
 
-def build_config(env_name, service_name, sample_env_file_path):
-    service_config = read_config(open(sample_env_file_path).read())
+def build_config(env_name, cloudlift_service_name, sample_env_file_path, essential_container_name):
     try:
-        environment_config = ParameterStore(
-            service_name,
+        environment_config, sidecars_configs = ParameterStore(
+            cloudlift_service_name,
             env_name).get_existing_config()
     except Exception as err:
         log_intent(str(err))
         raise UnrecoverableException("Cannot find the configuration in parameter store \
-[env: %s | service: %s]." % (env_name, service_name))
-    missing_env_config = set(service_config) - set(environment_config)
-    if missing_env_config:
-        raise UnrecoverableException('There is no config value for the keys ' +
-                str(missing_env_config))
-    missing_env_sample_config = set(environment_config) - set(service_config)
-    if missing_env_sample_config:
-        raise UnrecoverableException('There is no config value for the keys in env.sample file ' +
-                str(missing_env_sample_config))
+[env: %s | service: %s]." % (env_name, cloudlift_service_name))
+
+    sidecar_filename_format = 'sidecar_{}_{}'
+    configs_to_check = [(essential_container_name, sample_env_file_path, environment_config)]
+    sample_filename = basename(sample_env_file_path)
+    for sidecar_name, sidecar_config in sidecars_configs.items():
+        configs_to_check.append(
+            (container_name(sidecar_name), sidecar_filename_format.format(sidecar_name, sample_filename),
+             sidecar_config)
+        )
+
+    all_sidecar_env_samples = sorted(glob(sidecar_filename_format.format('*', sample_filename)))
+    all_sidecars_in_parameter_store = [sidecar_filename_format.format(name, sample_filename) for name in
+                                              sidecars_configs.keys()]
 
-    return make_container_defn_env_conf(service_config, environment_config)
+    if all_sidecar_env_samples != all_sidecars_in_parameter_store:
+        raise UnrecoverableException('There is a mismatch in sidecar configuratons. '
+                                     'Env Samples found: {}, Configurations present for: {}'.format(
+            all_sidecar_env_samples,
+            all_sidecars_in_parameter_store,
+        ))
+
+    container_configurations = {}
+    for name, filepath, env_config in configs_to_check:
+        sample_config = read_config(open(filepath).read())
+        missing_actual_config = set(sample_config) - set(env_config)
+        if missing_actual_config:
+            raise UnrecoverableException(
+                'There is no config value for the keys of container {} '.format(name) +
+                str(missing_actual_config))
+
+        missing_sample_config = set(env_config) - set(sample_config)
+        if missing_sample_config:
+            raise UnrecoverableException('There is no config value for the keys in {} file '.format(filepath) +
+                                         str(missing_sample_config))
+
+        container_configurations[name] = make_container_defn_env_conf(sample_config,
+                                                                      env_config)
+
+    return container_configurations
 
 
 def read_config(file_content):
@@ -105,7 +152,6 @@ def wait_for_finish(action, existing_events, color, deploy_end_time):
             color
         )
 
-
         if is_deployed(service['deployments']):
             return True
 
@@ -192,7 +238,7 @@ def print_task_diff(ecs_service_name, diffs, color):
                     Color(
                         '{' + env_var_diff_color + '}' +
                         env_var +
-                        '{/'+env_var_diff_color+'}'
+                        '{/' + env_var_diff_color + '}'
                     ),
                     old_val,
                     current_val
@@ -206,3 +252,11 @@ def print_task_diff(ecs_service_name, diffs, color):
             ecs_service_name + " No change in environment variables",
             color
         )
+
+
+def container_name(service_name):
+    return service_name + "Container"
+
+
+def strip_container_name(name):
+    return name.replace("Container", "")

cloudlift/deployment/ecs.py

@@ -249,9 +249,12 @@ def get_overrides_command(command):
     def get_overrides_env(env):
         return [{"name": e, "value": env[e]} for e in env]
 
-    def set_images(self, tag=None, **images):
+    def set_images(self, container_to_deploy, tag=None, **images):
         self.validate_container_options(**images)
         for container in self.containers:
+            if container[u'name'] != container_to_deploy:
+                continue
+
             if container[u'name'] in images:
                 new_image = images[container[u'name']]
                 diff = EcsTaskDefinitionDiff(

cloudlift/deployment/editor.py

@@ -7,10 +7,10 @@
 from cloudlift.config.logging import log_warning
 
 
-def edit_config(name, environment):
+def edit_config(name, environment, sidecar_name):
     parameter_store = ParameterStore(name, environment)
-    env_config_strings = parameter_store.get_existing_config_as_string()
-    edited_config_content = click.edit(str(env_config_strings))
+    env_config_strings = parameter_store.get_existing_config_as_string(sidecar_name)
+    edited_config_content = click.edit(str(env_config_strings), extension=".properties")
 
     if edited_config_content is None:
         log_warning("No changes made, exiting.")
@@ -25,6 +25,6 @@ def edit_config(name, environment):
     else:
         print_parameter_changes(differences)
         if click.confirm('Do you want update the config?'):
-            parameter_store.set_config(differences)
+            parameter_store.set_config(differences, sidecar_name)
         else:
             log_warning("Changes aborted.")