Merge pull request #1383 from guardian/aa/minInstancesInService

feat: Set `MinInstancesInService` via CFN parameters

magenta-lib/src/main/scala/magenta/deployment_type/AmiCloudFormationParameter.scala

@@ -53,6 +53,7 @@ object AmiCloudFormationParameter
       }
 
       val amiLookupFn = getLatestAmi(pkg, target, reporter, resources.lookup)
+      val minInServiceParameterMap = getMinInServiceTagRequirements(pkg, target)
 
       val unresolvedParameters = new CloudFormationParameters(
         target = target,
@@ -61,7 +62,8 @@ object AmiCloudFormationParameter
         latestImage = amiLookupFn,
 
         // Not expecting any user parameters in this deployment type
-        userParameters = Map.empty
+        userParameters = Map.empty,
+        minInServiceParameterMap = minInServiceParameterMap
       )
 
       List(

magenta-lib/src/main/scala/magenta/deployment_type/CloudFormation.scala

@@ -1,6 +1,6 @@
 package magenta.deployment_type
 
-import magenta.Loggable
+import magenta.{Loggable, Strategy}
 import magenta.Strategy.{Dangerous, MostlyHarmless}
 import magenta.artifact.S3Path
 import magenta.deployment_type.CloudFormationDeploymentTypeParameters._
@@ -148,12 +148,15 @@ class CloudFormation(tagger: BuildTags)
 
       val changeSetName = s"${target.stack.name}-${new DateTime().getMillis}"
 
+      val minInServiceParameterMap = getMinInServiceTagRequirements(pkg, target)
+
       val unresolvedParameters = new CloudFormationParameters(
         target,
         stackTags,
         userParams,
         amiParameterMap,
-        amiLookupFn
+        amiLookupFn,
+        minInServiceParameterMap
       )
 
       val createNewStack = createStackIfAbsent(pkg, target, reporter)

magenta-lib/src/main/scala/magenta/deployment_type/CloudFormationDeploymentTypeParameters.scala

@@ -1,11 +1,15 @@
 package magenta.deployment_type
 
+import magenta.deployment_type.CloudFormationDeploymentTypeParameters.CfnParam
+import magenta.tasks.ASG
+import magenta.tasks.ASG.TagMatch
 import magenta.tasks.UpdateCloudFormationTask.{
   CloudFormationStackLookupStrategy,
   LookupByName,
   LookupByTags
 }
 import magenta.{DeployReporter, DeployTarget, DeploymentPackage, Lookup}
+import software.amazon.awssdk.services.autoscaling.AutoScalingClient
 
 import java.time.Duration
 import java.time.Duration.ofMinutes
@@ -104,6 +108,24 @@ trait CloudFormationDeploymentTypeParameters {
       """.stripMargin
   ).default(false)
 
+  val minInstancesInServiceParameters = Param[Map[CfnParam, TagCriteria]](
+    "minInstancesInServiceParameters",
+    optional = true,
+    documentation = """Mapping between a CloudFormation parameter controlling the MinInstancesInService property of an ASG UpdatePolicy and the ASG.
+        |
+        |For example:
+        |```
+        |  minInstancesInServiceParameters:
+        |    MinInstancesInServiceForApi:
+        |      App: my-api
+        |    MinInstancesInServiceForFrontend:
+        |      App: my-frontend
+        |```
+        |This instructs Riff-Raff that the CFN parameter `MinInstancesInServiceForApi` relates to an ASG tagged `App=my-api`.
+        |Additional requirements of `Stack=<STACK BEING DEPLOYED>`, `Stage=<STAGE BEING DEPLOYED>` and `aws:cloudformation:stack-name=<CFN STACK BEING DEPLOYED>` are automatically added.
+      """.stripMargin
+  )
+
   val secondsToWaitForChangeSetCreation: Param[Duration] = Param
     .waitingSecondsFor(
       "secondsToWaitForChangeSetCreation",
@@ -191,4 +213,25 @@ trait CloudFormationDeploymentTypeParameters {
         }
       }
   }
+
+  def getMinInServiceTagRequirements(
+      pkg: DeploymentPackage,
+      target: DeployTarget
+  ): Map[CfnParam, List[TagMatch]] = {
+    minInstancesInServiceParameters.get(pkg) match {
+      case Some(params) =>
+        val stackStageTags = List(
+          TagMatch("Stack", target.stack.name),
+          TagMatch("Stage", target.parameters.stage.name)
+        )
+        params.map { case (cfnParam, tagRequirements) =>
+          cfnParam -> {
+            tagRequirements.map { case (key, value) =>
+              TagMatch(key, value)
+            }.toList ++ stackStageTags
+          }
+        }
+      case _ => Map.empty
+    }
+  }
 }

magenta-lib/src/main/scala/magenta/tasks/AWS.scala

@@ -490,6 +490,59 @@ object ASG {
     )
   }
 
+  def getMinInstancesInService(
+      tagRequirements: List[TagMatch],
+      client: AutoScalingClient,
+      reporter: DeployReporter
+  ): Int = {
+    groupWithTags(tagRequirements, client, reporter) match {
+      case Some(asg) =>
+        val max = asg.maxSize
+        val desired = asg.desiredCapacity
+        val seventyFivePercent: Int = (max * 0.75).toInt
+        val minInstancesInService = Math.min(
+          seventyFivePercent,
+          desired
+        )
+
+        // Happy path. We have enough headroom to double the instances, then half once healthy.
+        if (2 * desired <= max) {
+          reporter.info(
+            s"Max=$max. Desired=$desired. Setting MinInstancesInService=$minInstancesInService."
+          )
+        }
+
+        // We have to take some running instances out of service, at the cost of possibly reducing service availability.
+        else if (minInstancesInService < desired) {
+          reporter.warning(
+            s"""Max=$max. Desired=$desired. Setting MinInstancesInService=$minInstancesInService.
+               |This deployment will temporarily reduce the number of in-service instances.
+               |The number of instances may go as low as $minInstancesInService, which is less than the current desired capacity of $desired.
+               |To ensure a quick deployment we cannot set "MinInstancesInService" to more than 75% of the maximum number of instances.
+               |You should consider increasing your application's maximum capacity so that we always have at least 25% headroom for deployments.
+               |""".stripMargin
+          )
+        }
+
+        // There isn't enough room to double capacity, but we don't have to take any instances out of service.
+        else {
+          reporter.warning(
+            s"""Max=$max. Desired=$desired. Setting MinInstancesInService=$minInstancesInService.
+               |This deployment will happen more slowly, in multiple steps.
+               |The current number of in-service instances will be preserved and the application will be updated in batches of at most ${max - minInstancesInService}.
+               |You could consider increasing your application's maximum capacity to double of your expected maximum so that deployments can happen in a single step.
+               |""".stripMargin
+          )
+        }
+
+        minInstancesInService
+      case _ =>
+        reporter.fail(
+          s"No autoscaling group found with tags $tagRequirements. Creating a new stack? Initially choose the ${Strategy.Dangerous} strategy."
+        )
+    }
+  }
+
   def groupWithTags(
       tagRequirements: List[TagRequirement],
       client: AutoScalingClient,