diff --git a/examples/hello-lb-v2s.jsonnet b/examples/hello-lb-v2s.jsonnet new file mode 100644 index 0000000..b3bb802 --- /dev/null +++ b/examples/hello-lb-v2s.jsonnet @@ -0,0 +1,97 @@ +{ + scheduler: { + type: 'ecs', + region: 'ap-northeast-1', + cluster: 'eagletmt', + desired_count: 2, + role: 'ecsServiceRole', + // dynamic_port_mapping is enabled by default with elb_v2 + // dynamic_port_mapping: false, + // health_check_grace_period_seconds: 0, + elb_v2s: [{ + // VPC id where the target group is located + vpc_id: 'vpc-WWWWWWWW', + // If you want internal ELB, then use 'scheme'. (ex. internal service that like microservice inside VPC) + // scheme: internal + // Health check path of the target group + health_check_path: '/site/sha', + listeners: [ + { + port: 80, + protocol: 'HTTP', + }, + { + port: 443, + protocol: 'HTTPS', + certificate_arn: 'arn:aws:iam::012345678901:server-certificate/hello-lb-v2.example.com', + }, + ], + subnets: ['subnet-XXXXXXXX', 'subnet-YYYYYYYY'], + security_groups: ['sg-ZZZZZZZZ'], + load_balancer_attributes: { + 'access_logs.s3.enabled': 'true', + 'access_logs.s3.bucket': 'hako-access-logs', + 'access_logs.s3.prefix': 'hako-hello-lb-v2', + }, + target_group_attributes: { + // http://docs.aws.amazon.com/en_us/elasticloadbalancing/latest/application/load-balancer-target-groups.html#target-group-attributes + 'deregistration_delay.timeout_seconds': '20', + }, + }, + }, + { + // VPC id where the target group is located + vpc_id: 'vpc-WWWWWWWW', + // If you want internal ELB, then use 'scheme'. (ex. internal service that like microservice inside VPC) + // scheme: internal + // Health check path of the target group + health_check_path: '/site/stats', + listeners: [ + { + port: 8888, + protocol: 'HTTP', + }, + ], + subnets: ['subnet-XXXXXXXX', 'subnet-YYYYYYYY'], + security_groups: ['sg-ZZZZZZZZ'], + load_balancer_attributes: { + 'access_logs.s3.enabled': 'true', + 'access_logs.s3.bucket': 'hako-access-logs', + 'access_logs.s3.prefix': 'hako-hello-lb-v2', + }, + target_group_attributes: { + // http://docs.aws.amazon.com/en_us/elasticloadbalancing/latest/application/load-balancer-target-groups.html#target-group-attributes + 'deregistration_delay.timeout_seconds': '20', + }, + }, + }], + app: { + image: 'ryotarai/hello-sinatra', + memory: 128, + cpu: 256, + env: { + PORT: '3000', + }, + secrets: [{ + name: 'MESSAGE', + value_from: 'arn:aws:ssm:ap-northeast-1:012345678901:parameter/hako/hello-lb-v2/secret-message', + }], + }, + sidecars: { + front: { + image_tag: 'hako-nginx', + memory: 32, + cpu: 32, + }, + }, + scripts: [ + (import 'front.libsonnet') + { + backend_port: 3000, + locations: { + '/': { + allow_only_from: ['10.0.0.0/24'], + }, + }, + }, + ], +} diff --git a/lib/hako/schedulers/ecs.rb b/lib/hako/schedulers/ecs.rb index baca29e..f0f795c 100644 --- a/lib/hako/schedulers/ecs.rb +++ b/lib/hako/schedulers/ecs.rb @@ -13,6 +13,7 @@ require 'hako/schedulers/ecs_definition_comparator' require 'hako/schedulers/ecs_elb' require 'hako/schedulers/ecs_elb_v2' +require 'hako/schedulers/ecs_elb_v2s' require 'hako/schedulers/ecs_service_comparator' require 'hako/schedulers/ecs_service_discovery' require 'hako/schedulers/ecs_volume_comparator' @@ -37,7 +38,8 @@ def configure(options) @task_role_arn = options.fetch('task_role_arn', nil) @ecs_elb_options = options.fetch('elb', nil) @ecs_elb_v2_options = options.fetch('elb_v2', nil) - if @ecs_elb_options && @ecs_elb_v2_options + @ecs_elb_v2s_options = options.fetch('elb_v2s', nil) + if @ecs_elb_options && @ecs_elb_v2_options && @ecs_elb_v2s_options validation_error!('Cannot specify both elb and elb_v2') end @network_mode = options.fetch('network_mode', nil) @@ -47,7 +49,7 @@ def configure(options) end @dynamic_port_mapping = options.fetch('dynamic_port_mapping', @ecs_elb_options.nil?) @health_check_grace_period_seconds = options.fetch('health_check_grace_period_seconds') do - @ecs_elb_options || @ecs_elb_v2_options ? 0 : nil + @ecs_elb_options || @ecs_elb_v2_options || @ecs_elb_v2s_options ? 0 : nil end if options.key?('autoscaling') @autoscaling = EcsAutoscaling.new(options.fetch('autoscaling'), @region, ecs_elb_client, dry_run: @dry_run) @@ -253,7 +255,11 @@ def status unless service.load_balancers.empty? puts 'Load balancer:' - ecs_elb_client.show_status(service.load_balancers[0]) + if @elb_v2s_config + ecs_elb_client.show_status(service.load_balancers) + else + ecs_elb_client.show_status(service.load_balancers[0]) + end end puts 'Deployments:' @@ -379,13 +385,15 @@ def ssm_client @ssm_client ||= Aws::SSM::Client.new(region: @region) end - # @return [EcsElb, EcsElbV2] + # @return [EcsElb, EcsElbV2, EcsElbV2s] def ecs_elb_client @ecs_elb_client ||= if @ecs_elb_options EcsElb.new(@app_id, @region, @ecs_elb_options, dry_run: @dry_run) - else + elsif @ecs_elb_v2_options EcsElbV2.new(@app_id, @region, @ecs_elb_v2_options, dry_run: @dry_run) + else + EcsElbV2s.new(@app_id, @region, @ecs_elb_v2s_options, dry_run: @dry_run) end end @@ -891,7 +899,12 @@ def create_initial_service(task_definition_arn, front_port) params[:desired_count] = 0 end if ecs_elb_client.find_or_create_load_balancer(front_port) - params[:load_balancers] = [ecs_elb_client.load_balancer_params_for_service] + params[:load_balancers] = + if @ecs_elb_v2s_options + ecs_elb_client.load_balancer_params_for_services + else + [ecs_elb_client.load_balancer_params_for_service] + end end if @service_discovery @service_discovery.apply diff --git a/lib/hako/schedulers/ecs_elb_v2s.rb b/lib/hako/schedulers/ecs_elb_v2s.rb new file mode 100644 index 0000000..ca8eff9 --- /dev/null +++ b/lib/hako/schedulers/ecs_elb_v2s.rb @@ -0,0 +1,328 @@ +# frozen_string_literal: true + +require 'aws-sdk-elasticloadbalancingv2' +require 'hako' +require 'hako/error' + +module Hako + module Schedulers + class EcsElbV2s + # @param [String] app_id + # @param [String] region + # @param [Hash] elb_v2s_config + # @param [Boolean] dry_run + def initialize(app_id, region, elb_v2s_config, dry_run:) + @app_id = app_id + @region = region + @elb_v2s_config = elb_v2s_config + @dry_run = dry_run + end + + # @param [Aws::ECS::Types::LoadBalancer] ecs_lb + # @return [nil] + def show_status(ecs_lbs) + ecs_lbs.each do |ecs_lb| + elb_config = @elb_v2s_config.find { |c| elb_name(c) == ecs_lb.load_balancer_name } + lb = describe_load_balancer(elb_config) + elb_client.describe_listeners(load_balancer_arn: lb.load_balancer_arn).each do |page| + page.listeners.each do |listener| + puts " #{lb.dns_name}:#{listener.port} -> #{ecs_lb.container_name}:#{ecs_lb.container_port}" + end + end + end + end + + # @return [Aws::ElasticLoadBalancingV2::Types::TargetGroup] + def describe_target_group + cfg = @elb_v2s_config.find { |c| c.fetch('primary_target', false) } + if cfg.nil? + cfg = @elb_v2s_config[0] + end + tg = cfg.fetch('target_group_name', "hako-#{@app_id}") + elb_client.describe_target_groups(names: [tg]).target_groups[0] + rescue Aws::ElasticLoadBalancingV2::Errors::TargetGroupNotFound + nil + end + + # @param [Fixnum] front_port + # @return [Boolean] + def find_or_create_load_balancer(_front_port) + unless @elb_v2s_config + return false + end + + @elb_v2s_config.each do |elb_config| + load_balancer = describe_load_balancer(elb_config) + if load_balancer_given?(elb_config) && !load_balancer + create_load_balancer(elb_config) + end + + target_group = describe_tg(elb_config) + if target_group_given?(elb_config) && !target_group + create_target_group(elb_config) + end + + if load_balancer_given?(elb_config) + listener_ports = elb_client.describe_listeners(load_balancer_arn: load_balancer.load_balancer_arn).flat_map { |page| page.listeners.map(&:port) } + elb_config.fetch('listeners').each do |l| + params = { + load_balancer_arn: load_balancer.load_balancer_arn, + protocol: l.fetch('protocol'), + port: l.fetch('port'), + ssl_policy: l['ssl_policy'], + default_actions: [{ type: 'forward', target_group_arn: target_group.target_group_arn }], + } + certificate_arn = l.fetch('certificate_arn', nil) + if certificate_arn + params[:certificates] = [{ certificate_arn: certificate_arn }] + end + + unless listener_ports.include?(params[:port]) + listener = elb_client.create_listener(params).listeners[0] + Hako.logger.info("Created listener #{listener.listener_arn}") + end + end + end + end + true + end + + # @return [nil] + def modify_attributes + unless @elb_v2s_config + return nil + end + + @elb_v2s_config.each do |elb_config| + unless load_balancer_given?(elb_config) + load_balancer = describe_load_balancer(elb_config) + set_subnets(elb_config, load_balancer) + set_listeners(elb_config, load_balancer) + set_load_balancer_attributes(elb_config, load_balancer) + end + + unless target_group_given?(elb_config) + modify_target_group_attributes(elb_config) + end + end + nil + end + + # @return [Boolean] + def destroy + unless @elb_v2s_config + return false + end + + @elb_v2s_config.each do |elb_config| + unless load_balancer_given? + load_balancer = describe_load_balancer(elb_config) + if load_balancer + if @dry_run + Hako.logger.info("elb_client.delete_load_balancer(load_balancer_arn: #{load_balancer.load_balancer_arn})") + else + elb_client.delete_load_balancer(load_balancer_arn: load_balancer.load_balancer_arn) + Hako.logger.info "Deleted ELBv2 #{load_balancer.load_balancer_arn}" + end + else + Hako.logger.info "ELBv2 #{elb_name(elb_config)} doesn't exist" + end + end + + unless target_group_given?(elb_config) + target_group = describe_tg(elb_config) + if target_group + if @dry_run + Hako.logger.info("elb_client.delete_target_group(target_group_arn: #{target_group.target_group_arn})") + else + deleted = false + 30.times do + begin + elb_client.delete_target_group(target_group_arn: target_group.target_group_arn) + deleted = true + break + rescue Aws::ElasticLoadBalancingV2::Errors::ResourceInUse => e + Hako.logger.warn("#{e.class}: #{e.message}") + end + sleep 1 + end + unless deleted + raise Error.new("Cannot delete target group #{target_group.target_group_arn}") + end + + Hako.logger.info "Deleted target group #{target_group.target_group_arn}" + end + end + end + end + end + + # @return [Hash] + def load_balancer_params_for_services + @elb_v2s_config.map do |elb_config| + { + target_group_arn: describe_tg(elb_config).target_group_arn, + container_name: elb_config.fetch('container_name', 'front'), + container_port: elb_config.fetch('container_port', 80), + } + end + end + + private + + def elb_client + @elb_v2 ||= Aws::ElasticLoadBalancingV2::Client.new(region: @region) + end + + # @return [Boolean] + def load_balancer_given?(elb_config) + elb_config.key?('load_balancer_name') + end + + # @return [String] + def elb_name(elb_config) + elb_config.fetch('load_balancer_name', "hako-#{@app_id}") + end + + # @return [Aws::ElasticLoadBalancingV2::Types::LoadBalancer] + def describe_load_balancer(elb_config) + elb_client.describe_load_balancers(names: [elb_name(elb_config)]).load_balancers[0] + rescue Aws::ElasticLoadBalancingV2::Errors::LoadBalancerNotFound + nil + end + + def describe_tg(elb_config) + elb_client.describe_target_groups(names: [target_group_name(elb_config)]).target_groups[0] + rescue Aws::ElasticLoadBalancingV2::Errors::TargetGroupNotFound + nil + end + + # @return [String] + def target_group_name(elb_config) + elb_config.fetch('target_group_name', "hako-#{@app_id}") + end + + # @return [Boolean] + def target_group_given?(elb_config) + elb_config.key?('target_group_name') + end + + # @return [nil] + def create_load_balancer(elb_config) + tags = elb_config.fetch('tags', {}).map { |k, v| { key: k, value: v.to_s } } + elb_type = elb_config.fetch('type', nil) + if elb_type == 'network' + load_balancer = elb_client.create_load_balancer( + name: elb_name(elb_config), + subnets: elb_config.fetch('subnets'), + scheme: elb_config.fetch('scheme', nil), + type: 'network', + tags: tags.empty? ? nil : tags, + ).load_balancers[0] + Hako.logger.info "Created ELBv2(NLB) #{load_balancer.dns_name}" + else + load_balancer = elb_client.create_load_balancer( + name: elb_name(elb_config), + subnets: elb_config.fetch('subnets'), + security_groups: elb_config.fetch('security_groups'), + scheme: elb_config.fetch('scheme', nil), + type: elb_config.fetch('type', nil), + tags: tags.empty? ? nil : tags, + ).load_balancers[0] + Hako.logger.info "Created ELBv2 #{load_balancer.dns_name}" + end + end + + # @return [nil] + def create_target_group(elb_config) + elb_type = elb_config.fetch('type', nil) + target_group = if elb_type == 'network' + elb_client.create_target_group( + name: target_group_name(elb_config), + port: elb_config.fetch('container_port'), + protocol: 'TCP', + vpc_id: elb_config.fetch('vpc_id'), + target_type: elb_config.fetch('target_type', nil), + ).target_groups[0] + else + elb_client.create_target_group( + name: target_group_name(elb_config), + port: elb_config.fetch('container_port'), + protocol: 'HTTP', + vpc_id: elb_config.fetch('vpc_id'), + health_check_path: elb_config.fetch('health_check_path', nil), + target_type: elb_config.fetch('target_type', 'ip'), + ).target_groups[0] + end + Hako.logger.info "Created target group #{target_group.target_group_arn}" + end + + # @return [nil] + def set_subnets(elb_config, load_balancer) + subnets = elb_config.fetch('subnets').sort + if load_balancer && subnets != load_balancer.availability_zones.map(&:subnet_id).sort + if @dry_run + Hako.logger.info("elb_client.set_subnets(load_balancer_arn: #{load_balancer.load_balancer_arn}, subnets: #{subnets}) (dry-run)") + else + Hako.logger.info("Updating ELBv2 subnets to #{subnets}") + elb_client.set_subnets(load_balancer_arn: load_balancer.load_balancer_arn, subnets: subnets) + end + end + end + + # @return [nil] + def set_listeners(elb_config, load_balancer) + new_listeners = elb_config.fetch('listeners') + if load_balancer + current_listeners = elb_client.describe_listeners(load_balancer_arn: load_balancer.load_balancer_arn).listeners + new_listeners.each do |new_listener| + current_listener = current_listeners.find { |l| l.port == new_listener['port'] } + if current_listener && new_listener['ssl_policy'] && new_listener['ssl_policy'] != current_listener.ssl_policy + if @dry_run + Hako.logger.info("elb_client.modify_listener(listener_arn: #{current_listener.listener_arn}, ssl_policy: #{new_listener['ssl_policy']}) (dry-run)") + else + Hako.logger.info("Updating ELBv2 listener #{new_listener['port']} ssl_policy to #{new_listener['ssl_policy']}") + elb_client.modify_listener(listener_arn: current_listener.listener_arn, ssl_policy: new_listener['ssl_policy']) + end + end + end + end + end + + # @return [nil] + def set_load_balancer_attributes(_elb_config, load_balancer) + if @elb_v2s_config.key?('load_balancer_attributes') + attributes = @elb_v2s_config.fetch('load_balancer_attributes').map { |key, value| { key: key, value: value } } + if @dry_run + if load_balancer + Hako.logger.info("elb_client.modify_load_balancer_attributes(load_balancer_arn: #{load_balancer.load_balancer_arn}, attributes: #{attributes.inspect}) (dry-run)") + else + Hako.logger.info("elb_client.modify_load_balancer_attributes(load_balancer_arn: unknown, attributes: #{attributes.inspect}) (dry-run)") + end + else + Hako.logger.info("Updating ELBv2 attributes to #{attributes.inspect}") + elb_client.modify_load_balancer_attributes(load_balancer_arn: load_balancer.load_balancer_arn, attributes: attributes) + end + end + end + + # @return [nil] + def modify_target_group_attributes(elb_config) + if elb_config.key?('target_group_attributes') + target_group = describe_tg(elb_config) + attributes = elb_config.fetch('target_group_attributes').map { |key, value| { key: key, value: value } } + if @dry_run + if target_group + Hako.logger.info("elb_client.modify_target_group_attributes(target_group_arn: #{target_group.target_group_arn}, attributes: #{attributes.inspect}) (dry-run)") + else + Hako.logger.info("elb_client.modify_target_group_attributes(target_group_arn: unknown, attributes: #{attributes.inspect}) (dry-run)") + end + else + Hako.logger.info("Updating target group attributes to #{attributes.inspect}") + elb_client.modify_target_group_attributes(target_group_arn: target_group.target_group_arn, attributes: attributes) + end + end + end + end + end +end