feat: make the default stack behavior to use target type IP (#2961)

* fix: make synthetics use additional_hostnames

* refactor skip ports for linkerd

* commit from ci -- ran terraform-docs and pushed

* synthetics

* docs

* follow redirects

* follow redirects

* commit from ci -- ran terraform-docs and pushed

* skip ports

* we don't actually want follow redirect

* reset typical app

* commit

* commit

* always

* try out stickiness

* typo

* commit from ci -- ran terraform-docs and pushed

* always use clusterip

* add service mesh in typical app

* enable service mesh

* add sticky sessions nginx ingress

* print hostname

* fix session stickiness

* variables

* validate

* commit from ci -- ran terraform-docs and pushed

* default false

* sticky on services

* commit from ci -- ran terraform-docs and pushed

* wrong level

* commit from ci -- ran terraform-docs and pushed

* commit from ci -- ran terraform-docs and pushed

* commit from ci -- ran terraform-docs and pushed

* main branch

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

examples/typical_app/.happy/terraform/envs/rdev/main.tf

@@ -1,30 +1,33 @@
 module "stack" {
   source = "[email protected]:chanzuckerberg/happy//terraform/modules/happy-stack-eks?ref=main"
 
-  image_tag        = var.image_tag
-  image_tags       = jsondecode(var.image_tags)
-  app_name         = var.app
-  stack_name       = var.stack_name
-  deployment_stage = "rdev"
-  create_dashboard = true
-
-  stack_prefix  = "/${var.stack_name}"
-  k8s_namespace = var.k8s_namespace
-
+  image_tag           = var.image_tag
+  image_tags          = jsondecode(var.image_tags)
+  app_name            = var.app
+  stack_name          = var.stack_name
+  deployment_stage    = var.env
+  enable_service_mesh = true
+  create_dashboard    = true
+  stack_prefix        = "/${var.stack_name}"
+  k8s_namespace       = var.k8s_namespace
   // this allow these services under the same domain
   // each service is reachable via their path configured below
   routing_method = "CONTEXT"
 
   services = {
     frontend = {
+      sticky_sessions = {
+        enabled = true
+      }
+      synthetics    = true
       name          = "frontend"
       desired_count = 1
       // the maximum number of copies of this service it can autoscale to
       max_count = 50
       // the signal used to trigger autoscaling (ie. 50% of CPU means scale up)
       scaling_cpu_threshold_percentage = 50
       // the port the service is running on
-      port   = 3000
+      port            = 3000
       memory          = "500Mi"
       memory_requests = "300Mi"
       cpu             = "500m"
@@ -48,15 +51,17 @@ module "stack" {
       platform_architecture = "amd64"
     },
     internal-api = {
+
+      synthetics                       = true
       name                             = "internal-api"
       desired_count                    = 1
       max_count                        = 50
       scaling_cpu_threshold_percentage = 80
       port                             = 3000
-      memory          = "500Mi"
-      memory_requests = "300Mi"
-      cpu             = "500m"
-      cpu_requests    = "500m"
+      memory                           = "500Mi"
+      memory_requests                  = "300Mi"
+      cpu                              = "500m"
+      cpu_requests                     = "500m"
       health_check_path                = "/api/health"
       service_type                     = "INTERNAL"
       path                             = "/api/*"

examples/typical_app/.happy/terraform/envs/rdev/variables.tf

@@ -38,3 +38,8 @@ variable "app" {
   type        = string
   description = "Happy Path app name"
 }
+
+variable "env" {
+  type        = string
+  description = "Happy Path environment name"
+}

examples/typical_app/src/api/main.go

@@ -15,6 +15,7 @@ import (
 type Response struct {
 	Status   string
 	Service  string
+	Env      string
 	Complete bool
 }
 
@@ -30,11 +31,13 @@ func main() {
 	}))
 
 	app.Get("/", func(c *fiber.Ctx) error {
-		return c.Status(http.StatusOK).JSON(Response{Status: "OK", Service: "frontend"})
+		env, _ := os.LookupEnv("HOSTNAME")
+		return c.Status(http.StatusOK).JSON(Response{Status: "OK", Service: "frontend", Env: env})
 	})
 
 	app.Get("/health", func(c *fiber.Ctx) error {
-		return c.Status(http.StatusOK).JSON(Response{Status: "Health", Service: "frontend"})
+		env, _ := os.LookupEnv("HOSTNAME")
+		return c.Status(http.StatusOK).JSON(Response{Status: "Health", Service: "frontend", Env: env})
 	})
 
 	app.Get("/proxy", func(c *fiber.Ctx) error {

terraform/modules/happy-ingress-eks/README.md

@@ -31,16 +31,17 @@ No modules.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_additional_annotations"></a> [additional\_annotations](#input\_additional\_annotations) | Additional annotations to apply to the ingress resource | `map(string)` | `{}` | no |
+| <a name="input_additional_target_group_attributes"></a> [additional\_target\_group\_attributes](#input\_additional\_target\_group\_attributes) | Additional attributes to apply to the target group | <pre>list(object({<br>    key   = string<br>    value = string<br>  }))</pre> | `[]` | no |
 | <a name="input_aws_alb_healthcheck_interval_seconds"></a> [aws\_alb\_healthcheck\_interval\_seconds](#input\_aws\_alb\_healthcheck\_interval\_seconds) | The time in seconds to ping the target group for a health check; defaults to a high numbers since k8s also has a healthcheck | `string` | `"300"` | no |
 | <a name="input_certificate_arn"></a> [certificate\_arn](#input\_certificate\_arn) | ACM certificate ARN to attach to the load balancer listener | `string` | n/a | yes |
-| <a name="input_cloud_env"></a> [cloud\_env](#input\_cloud\_env) | Typically data.terraform\_remote\_state.cloud-env.outputs | <pre>object({<br>    public_subnets : list(string),<br>    private_subnets : list(string),<br>    database_subnets : list(string),<br>    database_subnet_group : string,<br>    vpc_id : string,<br>    vpc_cidr_block : string,<br>  })</pre> | n/a | yes |
+| <a name="input_cloud_env"></a> [cloud\_env](#input\_cloud\_env) | Typically data.terraform\_remote\_state.cloud-env.outputs | <pre>object({<br>    public_subnets        = list(string),<br>    private_subnets       = list(string),<br>    database_subnets      = list(string),<br>    database_subnet_group = string,<br>    vpc_id                = string,<br>    vpc_cidr_block        = string,<br>  })</pre> | n/a | yes |
 | <a name="input_health_check_path"></a> [health\_check\_path](#input\_health\_check\_path) | path to use for health checks | `string` | `"/"` | no |
 | <a name="input_ingress_name"></a> [ingress\_name](#input\_ingress\_name) | Name of the ingress resource | `string` | n/a | yes |
 | <a name="input_ingress_security_groups"></a> [ingress\_security\_groups](#input\_ingress\_security\_groups) | A list of security groups that should be allowed to communicate with this ingress. | `list(string)` | `[]` | no |
 | <a name="input_k8s_namespace"></a> [k8s\_namespace](#input\_k8s\_namespace) | K8S namespace for this service | `string` | n/a | yes |
 | <a name="input_labels"></a> [labels](#input\_labels) | Labels to apply to ingress resource | `map(string)` | n/a | yes |
 | <a name="input_regional_wafv2_arn"></a> [regional\_wafv2\_arn](#input\_regional\_wafv2\_arn) | A WAF to protect the EKS Ingress if needed | `string` | `null` | no |
-| <a name="input_routing"></a> [routing](#input\_routing) | Routing configuration for the ingress | <pre>object({<br>    method : optional(string, "DOMAIN")<br>    host_match : string<br>    group_name : string<br>    priority : number<br>    path : optional(string, "/*")<br>    service_name : string<br>    service_port : number<br>    service_scheme : string<br>    service_type : string<br>    alb_idle_timeout : optional(number, 60) // in seconds<br>    oidc_config : optional(object({<br>      issuer : string<br>      authorizationEndpoint : string<br>      tokenEndpoint : string<br>      userInfoEndpoint : string<br>      secretName : string<br>      }), {<br>      issuer                = ""<br>      authorizationEndpoint = ""<br>      tokenEndpoint         = ""<br>      userInfoEndpoint      = ""<br>      secretName            = ""<br>    })<br>    bypasses : optional(map(object({<br>      paths   = optional(set(string), [])<br>      methods = optional(set(string), [])<br>    })))<br>    success_codes : optional(string, "200-499")<br>  })</pre> | n/a | yes |
+| <a name="input_routing"></a> [routing](#input\_routing) | Routing configuration for the ingress | <pre>object({<br>    method           = optional(string, "DOMAIN")<br>    host_match       = string<br>    group_name       = string<br>    priority         = number<br>    path             = optional(string, "/*")<br>    service_name     = string<br>    service_port     = number<br>    service_scheme   = string<br>    service_type     = string<br>    alb_idle_timeout = optional(number, 60) // in seconds<br>    oidc_config = optional(object({<br>      issuer                = string<br>      authorizationEndpoint = string<br>      tokenEndpoint         = string<br>      userInfoEndpoint      = string<br>      secretName            = string<br>      }), {<br>      issuer                = ""<br>      authorizationEndpoint = ""<br>      tokenEndpoint         = ""<br>      userInfoEndpoint      = ""<br>      secretName            = ""<br>    })<br>    bypasses = optional(map(object({<br>      paths   = optional(set(string), [])<br>      methods = optional(set(string), [])<br>    })))<br>    success_codes = optional(string, "200-499")<br>    sticky_sessions = optional(object({<br>      enabled          = optional(bool, false),<br>      duration_seconds = optional(number, 600),<br>      cookie_name      = optional(string, "happy_sticky_session"),<br>    }), {})<br>  })</pre> | n/a | yes |
 | <a name="input_tags_string"></a> [tags\_string](#input\_tags\_string) | Tags to apply to ingress resource, comma delimited key=value pairs | `string` | `""` | no |
 | <a name="input_target_service_name"></a> [target\_service\_name](#input\_target\_service\_name) | Name of destination service that the ingress should route to | `string` | n/a | yes |
 | <a name="input_target_service_port"></a> [target\_service\_port](#input\_target\_service\_port) | Port of destination service that the ingress should route to | `number` | n/a | yes |

terraform/modules/happy-ingress-eks/main.tf

@@ -1,4 +1,23 @@
 locals {
+  base_target_group_attributes = [
+    {
+      key   = "deregistration_delay.timeout_seconds"
+      value = 60
+    },
+    {
+      key   = "stickiness.enabled"
+      value = var.routing.sticky_sessions.enabled
+    },
+    {
+      key   = "stickiness.lb_cookie.duration_seconds"
+      value = var.routing.sticky_sessions.duration_seconds
+    },
+  ]
+  target_group_attributes = concat(
+    local.base_target_group_attributes,
+    var.additional_target_group_attributes,
+  )
+  target_group_attributes_str = join(",", [for attr in local.target_group_attributes : "${attr.key}=${attr.value}"])
   ingress_base_annotations = {
     "kubernetes.io/ingress.class"                            = "alb"
     "alb.ingress.kubernetes.io/healthcheck-interval-seconds" = var.aws_alb_healthcheck_interval_seconds
@@ -7,12 +26,13 @@ locals {
     "alb.ingress.kubernetes.io/healthcheck-protocol"         = var.target_service_scheme
     "alb.ingress.kubernetes.io/listen-ports"                 = jsonencode([{ HTTPS = 443 }, { HTTP = 80 }])
     # All ingresses are "internet-facing". If a service_type was marked "INTERNAL", it will be protected using OIDC.
-    "alb.ingress.kubernetes.io/scheme"                  = var.routing.service_type == "VPC" ? "internal" : "internet-facing"
-    "alb.ingress.kubernetes.io/subnets"                 = join(",", var.cloud_env.public_subnets)
-    "alb.ingress.kubernetes.io/success-codes"           = var.routing.success_codes
-    "alb.ingress.kubernetes.io/tags"                    = var.tags_string
-    "alb.ingress.kubernetes.io/target-group-attributes" = "deregistration_delay.timeout_seconds=60"
-    "alb.ingress.kubernetes.io/target-type"             = "instance"
+    "alb.ingress.kubernetes.io/scheme"        = var.routing.service_type == "VPC" ? "internal" : "internet-facing"
+    "alb.ingress.kubernetes.io/subnets"       = join(",", var.cloud_env.public_subnets)
+    "alb.ingress.kubernetes.io/success-codes" = var.routing.success_codes
+    "alb.ingress.kubernetes.io/tags"          = var.tags_string
+    # IP target type is used to route traffic directly to the pod
+    "alb.ingress.kubernetes.io/target-group-attributes" = local.target_group_attributes_str
+    "alb.ingress.kubernetes.io/target-type"             = "ip"
     "alb.ingress.kubernetes.io/group.name"              = var.routing.group_name
     "alb.ingress.kubernetes.io/group.order"             = var.routing.priority
     "alb.ingress.kubernetes.io/load-balancer-attributes" = join(",", [ // Add any additional load-balancer-attributes here

terraform/modules/happy-ingress-eks/variables.tf

@@ -20,12 +20,12 @@ variable "target_service_scheme" {
 
 variable "cloud_env" {
   type = object({
-    public_subnets : list(string),
-    private_subnets : list(string),
-    database_subnets : list(string),
-    database_subnet_group : string,
-    vpc_id : string,
-    vpc_cidr_block : string,
+    public_subnets        = list(string),
+    private_subnets       = list(string),
+    database_subnets      = list(string),
+    database_subnet_group = string,
+    vpc_id                = string,
+    vpc_cidr_block        = string,
   })
   description = "Typically data.terraform_remote_state.cloud-env.outputs"
 }
@@ -52,36 +52,50 @@ variable "tags_string" {
   default     = ""
 }
 
+variable "additional_target_group_attributes" {
+  type = list(object({
+    key   = string
+    value = string
+  }))
+  description = "Additional attributes to apply to the target group"
+  default     = []
+}
+
 variable "routing" {
   type = object({
-    method : optional(string, "DOMAIN")
-    host_match : string
-    group_name : string
-    priority : number
-    path : optional(string, "/*")
-    service_name : string
-    service_port : number
-    service_scheme : string
-    service_type : string
-    alb_idle_timeout : optional(number, 60) // in seconds
-    oidc_config : optional(object({
-      issuer : string
-      authorizationEndpoint : string
-      tokenEndpoint : string
-      userInfoEndpoint : string
-      secretName : string
+    method           = optional(string, "DOMAIN")
+    host_match       = string
+    group_name       = string
+    priority         = number
+    path             = optional(string, "/*")
+    service_name     = string
+    service_port     = number
+    service_scheme   = string
+    service_type     = string
+    alb_idle_timeout = optional(number, 60) // in seconds
+    oidc_config = optional(object({
+      issuer                = string
+      authorizationEndpoint = string
+      tokenEndpoint         = string
+      userInfoEndpoint      = string
+      secretName            = string
       }), {
       issuer                = ""
       authorizationEndpoint = ""
       tokenEndpoint         = ""
       userInfoEndpoint      = ""
       secretName            = ""
     })
-    bypasses : optional(map(object({
+    bypasses = optional(map(object({
       paths   = optional(set(string), [])
       methods = optional(set(string), [])
     })))
-    success_codes : optional(string, "200-499")
+    success_codes = optional(string, "200-499")
+    sticky_sessions = optional(object({
+      enabled          = optional(bool, false),
+      duration_seconds = optional(number, 600),
+      cookie_name      = optional(string, "happy_sticky_session"),
+    }), {})
   })
   description = "Routing configuration for the ingress"
   validation {

terraform/modules/happy-nginx-ingress-eks/README.md

@@ -31,6 +31,7 @@ No modules.
 | <a name="input_ingress_name"></a> [ingress\_name](#input\_ingress\_name) | Name of the ingress resource | `string` | n/a | yes |
 | <a name="input_k8s_namespace"></a> [k8s\_namespace](#input\_k8s\_namespace) | K8S namespace for this service | `string` | n/a | yes |
 | <a name="input_labels"></a> [labels](#input\_labels) | Labels to apply to ingress resource | `map(string)` | n/a | yes |
+| <a name="input_sticky_sessions"></a> [sticky\_sessions](#input\_sticky\_sessions) | Sticky session configuration | <pre>object({<br>    enabled          = optional(bool, true),<br>    duration_seconds = optional(number, 600),<br>    cookie_name      = optional(string, "happy_sticky_session"),<br>  })</pre> | `{}` | no |
 | <a name="input_target_service_name"></a> [target\_service\_name](#input\_target\_service\_name) | Name of destination service that the ingress should route to | `string` | n/a | yes |
 | <a name="input_target_service_port"></a> [target\_service\_port](#input\_target\_service\_port) | Port of destination service that the ingress should route to | `string` | n/a | yes |
 | <a name="input_timeout"></a> [timeout](#input\_timeout) | Timeout for the ingress resource | `number` | `60` | no |

terraform/modules/happy-nginx-ingress-eks/main.tf

@@ -1,19 +1,28 @@
 
+locals {
+  sticky_annotations = {
+    "nginx.ingress.kubernetes.io/affinity"               = "cookie"
+    "nginx.ingress.kubernetes.io/session-cookie-name"    = var.sticky_sessions.cookie_name
+    "nginx.ingress.kubernetes.io/session-cookie-max-age" = var.sticky_sessions.duration_seconds
+  }
+
+  base_annotations = {
+    "cert-manager.io/cluster-issuer"                    = "nginx-issuer"
+    "linkerd.io/inject"                                 = "enabled"
+    "external-dns.alpha.kubernetes.io/exclude"          = "true"
+    "nginx.ingress.kubernetes.io/proxy-connect-timeout" = var.timeout
+    "nginx.ingress.kubernetes.io/proxy-send-timeout"    = var.timeout
+    "nginx.ingress.kubernetes.io/proxy-read-timeout"    = var.timeout
+  }
 
+  annotations = merge(local.base_annotations, var.sticky_sessions.enabled ? local.sticky_annotations : {})
+}
 resource "kubernetes_ingress_v1" "ingress" {
   metadata {
-    name      = var.ingress_name
-    namespace = var.k8s_namespace
-    annotations = {
-      "cert-manager.io/cluster-issuer"                    = "nginx-issuer"
-      "nginx.ingress.kubernetes.io/service-upstream"      = "true"
-      "linkerd.io/inject"                                 = "enabled"
-      "external-dns.alpha.kubernetes.io/exclude"          = "true"
-      "nginx.ingress.kubernetes.io/proxy-connect-timeout" = var.timeout
-      "nginx.ingress.kubernetes.io/proxy-send-timeout"    = var.timeout
-      "nginx.ingress.kubernetes.io/proxy-read-timeout"    = var.timeout
-    }
-    labels = var.labels
+    name        = var.ingress_name
+    namespace   = var.k8s_namespace
+    annotations = local.annotations
+    labels      = var.labels
   }
 
   spec {

terraform/modules/happy-nginx-ingress-eks/variables.tf

@@ -40,4 +40,14 @@ variable "timeout" {
   type        = number
   description = "Timeout for the ingress resource"
   default     = 60
+}
+
+variable "sticky_sessions" {
+  type = object({
+    enabled          = optional(bool, true),
+    duration_seconds = optional(number, 600),
+    cookie_name      = optional(string, "happy_sticky_session"),
+  })
+  description = "Sticky session configuration"
+  default     = {}
 }