Mesh Gateway Deployment Configuration (#3477)

* Add BATs for Gateway Log Level Configuration * Pass logLevel into init-container and dataplane-container * Add BATs for extraLabels * Test that extraLabels get set on the deployment * BATs for annotations * Use config for log level over gcc if available * Make consulDataplaneContainer an assoc func to builder * Use logLevelForDataplaneContainer func * Test annotations getting set * Add comments for Builder obj * Rename config.go to gateway_config.go * Add comments to gateway_config * Move commands closer to their configuration * Extract some constants * `%s/expected/debug/g`
hashicorp · Jan 18, 2024 · 5d9bd49 · 5d9bd49
1 parent 245a845
commit 5d9bd49
Show file tree

Hide file tree

Showing 10 changed files with 349 additions and 76 deletions.
diff --git a/charts/consul/test/unit/gateway-resources-configmap.bats b/charts/consul/test/unit/gateway-resources-configmap.bats
@@ -73,6 +73,117 @@ target=templates/gateway-resources-configmap.yaml
     [ "$resources" != null ]
 }
 
+#--------------------------------------------------------------------
+# Mesh Gateway logLevel configuration
+
+@test "gateway-resources/ConfigMap: Mesh Gateway logLevel default configuration" {
+    cd `chart_dir`
+    local config=$(helm template \
+        -s $target \
+        --set 'meshGateway.enabled=true' \
+        --set 'global.experiments[0]=resource-apis' \
+        --set 'ui.enabled=false' \
+        . | tee /dev/stderr |
+        yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr)
+
+    local actual=$(echo "$config" | yq -r '.container.consul.logging.level')
+    [ "${actual}" = 'info' ]
+
+    local actual=$(echo "$config" | yq -r '.initContainer.consul.logging.level')
+    [ "${actual}" = 'info' ]
+}
+
+
+@test "gateway-resources/ConfigMap: Mesh Gateway logLevel custom global configuration" {
+    cd `chart_dir`
+    local config=$(helm template \
+        -s $target \
+        --set 'meshGateway.enabled=true' \
+        --set 'global.experiments[0]=resource-apis' \
+        --set 'ui.enabled=false' \
+        --set 'global.logLevel=debug' \
+        . | tee /dev/stderr |
+        yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr)
+
+    local actual=$(echo "$config" | yq -r '.container.consul.logging.level')
+    [ "${actual}" = 'debug' ]
+
+    local actual=$(echo "$config" | yq -r '.initContainer.consul.logging.level')
+    [ "${actual}" = 'debug' ]
+}
+
+@test "gateway-resources/ConfigMap: Mesh Gateway logLevel custom meshGateway configuration" {
+    cd `chart_dir`
+    local config=$(helm template \
+        -s $target \
+        --set 'meshGateway.enabled=true' \
+        --set 'global.experiments[0]=resource-apis' \
+        --set 'ui.enabled=false' \
+        --set 'meshGateway.logLevel=debug' \
+        . | tee /dev/stderr |
+        yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr)
+
+    local actual=$(echo "$config" | yq -r '.container.consul.logging.level')
+    [ "${actual}" = 'debug' ]
+
+    local actual=$(echo "$config" | yq -r '.initContainer.consul.logging.level')
+    [ "${actual}" = 'debug' ]
+}
+
+@test "gateway-resources/ConfigMap: Mesh Gateway logLevel custom meshGateway configuration overrides global configuration" {
+    cd `chart_dir`
+    local config=$(helm template \
+        -s $target \
+        --set 'meshGateway.enabled=true' \
+        --set 'global.experiments[0]=resource-apis' \
+        --set 'ui.enabled=false' \
+        --set 'global.logLevel=error' \
+        --set 'meshGateway.logLevel=debug' \
+        . | tee /dev/stderr |
+        yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment' | tee /dev/stderr)
+
+    local actual=$(echo "$config" | yq -r '.container.consul.logging.level')
+    [ "${actual}" = 'debug' ]
+
+    local actual=$(echo "$config" | yq -r '.initContainer.consul.logging.level')
+    [ "${actual}" = 'debug' ]
+}
+
+#--------------------------------------------------------------------
+# Mesh Gateway Extra Labels configuration
+
+@test "gateway-resources/ConfigMap: Mesh Gateway gets Extra Labels when set" {
+    cd `chart_dir`
+    local actual=$(helm template \
+        -s $target \
+        --set 'connectInject.enabled=true' \
+        --set 'meshGateway.enabled=true' \
+        --set 'global.experiments[0]=resource-apis' \
+        --set 'ui.enabled=false' \
+        --set 'global.extraLabels.foo'='bar' \
+        . | tee /dev/stderr |
+        yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment.labels.set.foo' | tee /dev/stderr
+    )
+    [ "$actual" = 'bar' ]
+}
+
+#--------------------------------------------------------------------
+# Mesh Gateway annotations configuration
+
+@test "gateway-resources/ConfigMap: Mesh Gateway gets annotations when set" {
+    cd `chart_dir`
+    local actual=$(helm template \
+        -s $target \
+        --set 'connectInject.enabled=true' \
+        --set 'meshGateway.enabled=true' \
+        --set 'global.experiments[0]=resource-apis' \
+        --set 'ui.enabled=false' \
+        --set 'meshGateway.annotations.foo'='bar' \
+        . | tee /dev/stderr |
+        yq -r '.data["config.yaml"]' | yq -r '.gatewayClassConfigs[0].spec.deployment.annotations.set.foo' | tee /dev/stderr
+    )
+    [ "$actual" = 'bar' ]
+}
 
 #--------------------------------------------------------------------
 # Mesh Gateway WAN Address configuration

diff --git a/control-plane/gateways/builder.go b/control-plane/gateways/builder.go
@@ -7,12 +7,17 @@ import (
 	meshv2beta1 "github.com/hashicorp/consul-k8s/control-plane/api/mesh/v2beta1"
 )
 
+// meshGatewayBuilder is a helper struct for building the Kubernetes resources for a mesh gateway.
+// This includes Deployment, Role, Service, and ServiceAccount resources.
+// Configuration is combined from the MeshGateway, GatewayConfig, and GatewayClassConfig.
 type meshGatewayBuilder struct {
 	gateway *meshv2beta1.MeshGateway
 	config  GatewayConfig
 	gcc     *meshv2beta1.GatewayClassConfig
 }
 
+// NewMeshGatewayBuilder returns a new meshGatewayBuilder for the given MeshGateway,
+// GatewayConfig, and GatewayClassConfig.
 func NewMeshGatewayBuilder(gateway *meshv2beta1.MeshGateway, gatewayConfig GatewayConfig, gatewayClassConfig *meshv2beta1.GatewayClassConfig) *meshGatewayBuilder {
 	return &meshGatewayBuilder{
 		gateway: gateway,

diff --git a/control-plane/gateways/constants.go b/control-plane/gateways/constants.go
@@ -0,0 +1,33 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+
+package gateways
+
+const (
+	// General environment variables.
+	envPodName      = "POD_NAME"
+	envPodNamespace = "POD_NAMESPACE"
+	envNodeName     = "NODE_NAME"
+	envTmpDir       = "TMPDIR"
+
+	// Dataplane Configuration Environment variables.
+	envDPProxyId             = "DP_PROXY_ID"
+	envDPCredentialLoginMeta = "DP_CREDENTIAL_LOGIN_META"
+	envDPServiceNodeName     = "DP_SERVICE_NODE_NAME"
+
+	// Init Container Configuration Environment variables.
+	envConsulAddresses            = "CONSUL_ADDRESSES"
+	envConsulGRPCPort             = "CONSUL_GRPC_PORT"
+	envConsulHTTPPort             = "CONSUL_HTTP_PORT"
+	envConsulAPITimeout           = "CONSUL_API_TIMEOUT"
+	envConsulNodeName             = "CONSUL_NODE_NAME"
+	envConsulLoginAuthMethod      = "CONSUL_LOGIN_AUTH_METHOD"
+	envConsulLoginBearerTokenFile = "CONSUL_LOGIN_BEARER_TOKEN_FILE"
+	envConsulLoginMeta            = "CONSUL_LOGIN_META"
+	envConsulLoginPartition       = "CONSUL_LOGIN_PARTITION"
+	envConsulNamespace            = "CONSUL_NAMESPACE"
+	envConsulPartition            = "CONSUL_PARTITION"
+
+	// defaultBearerTokenFile is the default location where the init container will store the bearer token for the dataplane container to read.
+	defaultBearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+)
diff --git a/control-plane/gateways/deployment.go b/control-plane/gateways/deployment.go
@@ -51,7 +51,7 @@ func (b *meshGatewayBuilder) deploymentSpec() (*appsv1.DeploymentSpec, error) {
 		return nil, err
 	}
 
-	container, err := consulDataplaneContainer(b.config, containerConfig, b.gateway.Name, b.gateway.Namespace)
+	container, err := b.consulDataplaneContainer(containerConfig)
 	if err != nil {
 		return nil, err
 	}

diff --git a/control-plane/gateways/deployment_dataplane_container.go b/control-plane/gateways/deployment_dataplane_container.go
@@ -27,7 +27,7 @@ const (
 	volumeName                   = "consul-mesh-inject-data"
 )
 
-func consulDataplaneContainer(config GatewayConfig, containerConfig v2beta1.GatewayClassContainerConfig, name, namespace string) (corev1.Container, error) {
+func (b *meshGatewayBuilder) consulDataplaneContainer(containerConfig v2beta1.GatewayClassContainerConfig) (corev1.Container, error) {
 	// Extract the service account token's volume mount.
 	var (
 		err             error
@@ -36,11 +36,11 @@ func consulDataplaneContainer(config GatewayConfig, containerConfig v2beta1.Gate
 
 	resources := containerConfig.Resources
 
-	if config.AuthMethod != "" {
+	if b.config.AuthMethod != "" {
 		bearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token"
 	}
 
-	args, err := getDataplaneArgs(namespace, config, bearerTokenFile, name)
+	args, err := b.dataplaneArgs(bearerTokenFile)
 	if err != nil {
 		return corev1.Container{}, err
 	}
@@ -56,8 +56,8 @@ func consulDataplaneContainer(config GatewayConfig, containerConfig v2beta1.Gate
 	}
 
 	container := corev1.Container{
-		Name:  name,
-		Image: config.ImageDataplane,
+		Name:  b.gateway.Name,
+		Image: b.config.ImageDataplane,
 
 		// We need to set tmp dir to an ephemeral volume that we're mounting so that
 		// consul-dataplane can write files to it. Otherwise, it wouldn't be able to
@@ -66,39 +66,35 @@ func consulDataplaneContainer(config GatewayConfig, containerConfig v2beta1.Gate
 		// TODO(nathancoleman): I don't believe consul-dataplane needs to write anymore, investigate.
 		Env: []corev1.EnvVar{
 			{
-				Name: "DP_PROXY_ID",
+				Name: envDPProxyId,
 				ValueFrom: &corev1.EnvVarSource{
 					FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"},
 				},
 			},
 			{
-				Name: "POD_NAMESPACE",
+				Name: envPodNamespace,
 				ValueFrom: &corev1.EnvVarSource{
 					FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.namespace"},
 				},
 			},
 			{
-				Name:  "TMPDIR",
+				Name:  envTmpDir,
 				Value: constants.MeshV2VolumePath,
 			},
 			{
-				Name: "NODE_NAME",
+				Name: envNodeName,
 				ValueFrom: &corev1.EnvVarSource{
 					FieldRef: &corev1.ObjectFieldSelector{
 						FieldPath: "spec.nodeName",
 					},
 				},
 			},
 			{
-				Name:  "DP_CREDENTIAL_LOGIN_META",
+				Name:  envDPCredentialLoginMeta,
 				Value: "pod=$(POD_NAMESPACE)/$(DP_PROXY_ID)",
 			},
 			{
-				Name:  "DP_CREDENTIAL_LOGIN_META1",
-				Value: "pod=$(POD_NAMESPACE)/$(DP_PROXY_ID)",
-			},
-			{
-				Name:  "DP_SERVICE_NODE_NAME",
+				Name:  envDPServiceNodeName,
 				Value: "$(NODE_NAME)-virtual",
 			},
 		},
@@ -157,36 +153,36 @@ func consulDataplaneContainer(config GatewayConfig, containerConfig v2beta1.Gate
 	return container, nil
 }
 
-func getDataplaneArgs(namespace string, config GatewayConfig, bearerTokenFile string, name string) ([]string, error) {
+func (b *meshGatewayBuilder) dataplaneArgs(bearerTokenFile string) ([]string, error) {
 	args := []string{
-		"-addresses", config.ConsulConfig.Address,
-		"-grpc-port=" + strconv.Itoa(config.ConsulConfig.GRPCPort),
-		"-log-level=" + config.LogLevel,
-		"-log-json=" + strconv.FormatBool(config.LogJSON),
+		"-addresses", b.config.ConsulConfig.Address,
+		"-grpc-port=" + strconv.Itoa(b.config.ConsulConfig.GRPCPort),
+		"-log-level=" + b.logLevelForDataplaneContainer(),
+		"-log-json=" + strconv.FormatBool(b.config.LogJSON),
 		"-envoy-concurrency=" + defaultEnvoyProxyConcurrency,
 	}
 
-	consulNamespace := namespaces.ConsulNamespace(namespace, config.ConsulTenancyConfig.EnableConsulNamespaces, config.ConsulTenancyConfig.ConsulDestinationNamespace, config.ConsulTenancyConfig.EnableConsulNamespaces, config.ConsulTenancyConfig.NSMirroringPrefix)
+	consulNamespace := namespaces.ConsulNamespace(b.gateway.Namespace, b.config.ConsulTenancyConfig.EnableConsulNamespaces, b.config.ConsulTenancyConfig.ConsulDestinationNamespace, b.config.ConsulTenancyConfig.EnableConsulNamespaces, b.config.ConsulTenancyConfig.NSMirroringPrefix)
 
-	if config.AuthMethod != "" {
+	if b.config.AuthMethod != "" {
 		args = append(args,
 			"-credential-type=login",
-			"-login-auth-method="+config.AuthMethod,
+			"-login-auth-method="+b.config.AuthMethod,
 			"-login-bearer-token-path="+bearerTokenFile,
-			"-login-meta="+fmt.Sprintf("gateway=%s/%s", namespace, name),
+			"-login-meta="+fmt.Sprintf("gateway=%s/%s", b.gateway.Namespace, b.gateway.Name),
 		)
-		if config.ConsulTenancyConfig.ConsulPartition != "" {
-			args = append(args, "-login-partition="+config.ConsulTenancyConfig.ConsulPartition)
+		if b.config.ConsulTenancyConfig.ConsulPartition != "" {
+			args = append(args, "-login-partition="+b.config.ConsulTenancyConfig.ConsulPartition)
 		}
 	}
-	if config.ConsulTenancyConfig.EnableConsulNamespaces {
+	if b.config.ConsulTenancyConfig.EnableConsulNamespaces {
 		args = append(args, "-service-namespace="+consulNamespace)
 	}
-	if config.ConsulTenancyConfig.ConsulPartition != "" {
-		args = append(args, "-service-partition="+config.ConsulTenancyConfig.ConsulPartition)
+	if b.config.ConsulTenancyConfig.ConsulPartition != "" {
+		args = append(args, "-service-partition="+b.config.ConsulTenancyConfig.ConsulPartition)
 	}
 
-	args = append(args, buildTLSArgs(config)...)
+	args = append(args, buildTLSArgs(b.config)...)
 
 	// Configure the readiness port on the dataplane sidecar if proxy health checks are enabled.
 	args = append(args, fmt.Sprintf("%s=%d", "-envoy-ready-bind-port", constants.ProxyDefaultHealthPort))