Net 6392- Create MeshGateway Deployment (#3290)

* checkpoint * checkpoint, deployment spec intial skeleton * set up reconcile * checkpoint * checkpoint * checkpoint * working deployment * cleaning up todos, working deployment * cleaned up todos, changed namespaces back from hardcoded default * unit test finished * fix pointer added in rebase * Update control-plane/config-entries/controllersv2/mesh_gateway_controller.go * additional cleanup/linting issues * rename files, clean up configuration to reuse tenacy config * import grouping * responding to code review * gofmt * Update control-plane/config-entries/controllersv2/mesh_gateway_controller.go Co-authored-by: Nathan Coleman <[email protected]> * clean up incorrect comment * add gcc nil test to cover potential nil use cases, add log statment for gcc fetch error * clean up nit picks * checkpoint * fix typing * Update control-plane/gateways/config.go Co-authored-by: Nathan Coleman <[email protected]> * Update control-plane/gateways/deployment_init_container.go Co-authored-by: Nathan Coleman <[email protected]> * Update control-plane/gateways/deployment_init_container.go Co-authored-by: Nathan Coleman <[email protected]> * Update control-plane/config-entries/controllersv2/mesh_gateway_controller.go Co-authored-by: Nathan Coleman <[email protected]> * Add rudimentary Deployment assertions to mesh_gateway_controller_test.go * Fix command assertion whitespace * Use full GCC instead of GCCSpec so we have future access to annotations --------- Co-authored-by: Nathan Coleman <[email protected]>
hashicorp · Dec 5, 2023 · 2785091 · 2785091
1 parent fd6d765
commit 2785091
Show file tree

Hide file tree

Showing 14 changed files with 1,295 additions and 18 deletions.
diff --git a/control-plane/api/common/common.go b/control-plane/api/common/common.go
@@ -4,7 +4,10 @@
 // Package common holds code that isn't tied to a particular CRD version or type.
 package common
 
-import mapset "github.com/deckarep/golang-set"
+import (
+	mapset "github.com/deckarep/golang-set"
+	"time"
+)
 
 const (
 	// V1 config entries.
@@ -79,3 +82,11 @@ type K8sNamespaceConfig struct {
 	// Endpoints in the DenyK8sNamespacesSet are ignored.
 	DenyK8sNamespacesSet mapset.Set
 }
+
+// ConsulConfig manages config to tell a pod where consul is located.
+type ConsulConfig struct {
+	Address    string
+	GRPCPort   int
+	HTTPPort   int
+	APITimeout time.Duration
+}
diff --git a/control-plane/config-entries/controllersv2/mesh_gateway_controller.go b/control-plane/config-entries/controllersv2/mesh_gateway_controller.go
@@ -6,8 +6,10 @@ package controllersv2
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	k8serr "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -16,16 +18,23 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
+	"github.com/hashicorp/consul-k8s/control-plane/api/common"
+
 	meshv2beta1 "github.com/hashicorp/consul-k8s/control-plane/api/mesh/v2beta1"
 	"github.com/hashicorp/consul-k8s/control-plane/gateways"
 )
 
+// errResourceNotOwned indicates that a resource the controller would have
+// updated or deleted does not have an owner reference pointing to the MeshGateway.
+var errResourceNotOwned = errors.New("existing resource not owned by controller")
+
 // MeshGatewayController reconciles a MeshGateway object.
 type MeshGatewayController struct {
 	client.Client
-	Log        logr.Logger
-	Scheme     *runtime.Scheme
-	Controller *ConsulResourceController
+	Log           logr.Logger
+	Scheme        *runtime.Scheme
+	Controller    *ConsulResourceController
+	GatewayConfig gateways.GatewayConfig
 }
 
 // +kubebuilder:rbac:groups=mesh.consul.hashicorp.com,resources=meshgateway,verbs=get;list;watch;create;update;patch;delete
@@ -83,19 +92,54 @@ func (r *MeshGatewayController) SetupWithManager(mgr ctrl.Manager) error {
 //  4. Role
 //  5. RoleBinding
 func (r *MeshGatewayController) onCreateUpdate(ctx context.Context, req ctrl.Request, resource *meshv2beta1.MeshGateway) error {
-	builder := gateways.NewMeshGatewayBuilder(resource)
+	//fetch gatewayclassconfig
+	gcc, err := r.getGatewayClassConfigForGateway(ctx, resource)
+	if err != nil {
+		r.Log.Error(err, "unable to get gatewayclassconfig for gateway: %s gatewayclass: %s", resource.Name, resource.Spec.GatewayClassName)
+		return err
+	}
+
+	builder := gateways.NewMeshGatewayBuilder(resource, r.GatewayConfig, gcc)
 
 	upsertOp := func(ctx context.Context, _, object client.Object) error {
 		_, err := controllerutil.CreateOrUpdate(ctx, r.Client, object, func() error { return nil })
 		return err
 	}
 
-	err := r.opIfNewOrOwned(ctx, resource, &corev1.ServiceAccount{}, builder.ServiceAccount(), upsertOp)
+	err = r.opIfNewOrOwned(ctx, resource, &corev1.ServiceAccount{}, builder.ServiceAccount(), upsertOp)
 	if err != nil {
+		return fmt.Errorf("unable to create service account: %w", err)
+	}
+
+	// TODO NET-6393 NET-6395
+
+	//Create deployment
+
+	mergeDeploymentOp := func(ctx context.Context, existingObject, object client.Object) error {
+		existingDeployment, ok := existingObject.(*appsv1.Deployment)
+		if !ok && existingDeployment != nil {
+			return fmt.Errorf("unable to infer existing deployment type")
+		}
+		builtDeployment, ok := object.(*appsv1.Deployment)
+		if !ok {
+			return fmt.Errorf("unable to infer built deployment type")
+		}
+
+		mergedDeployment := builder.MergeDeployments(gcc, existingDeployment, builtDeployment)
+
+		_, err := controllerutil.CreateOrUpdate(ctx, r.Client, mergedDeployment, func() error { return nil })
 		return err
 	}
 
-	// TODO NET-6392 NET-6393 NET-6395
+	builtDeployment, err := builder.Deployment()
+	if err != nil {
+		return fmt.Errorf("Unable to build deployment: %w", err)
+	}
+
+	err = r.opIfNewOrOwned(ctx, resource, &appsv1.Deployment{}, builtDeployment, mergeDeploymentOp)
+	if err != nil {
+		return fmt.Errorf("Unable to create deployment: %w", err)
+	}
 
 	return nil
 }
@@ -122,6 +166,7 @@ type ownedObjectOp func(ctx context.Context, existingObject client.Object, newOb
 // The purpose of opIfNewOrOwned is to ensure that we aren't updating or deleting a
 // resource that was not created by us. If this scenario is encountered, we error.
 func (r *MeshGatewayController) opIfNewOrOwned(ctx context.Context, gateway *meshv2beta1.MeshGateway, scanTarget, writeSource client.Object, op ownedObjectOp) error {
+
 	// Ensure owner reference is always set on objects that we write
 	if err := ctrl.SetControllerReference(gateway, writeSource, r.Client.Scheme()); err != nil {
 		return err
@@ -157,7 +202,50 @@ func (r *MeshGatewayController) opIfNewOrOwned(ctx context.Context, gateway *mes
 		}
 	}
 	if !owned {
-		return errors.New("existing resource not owned by controller")
+		return errResourceNotOwned
 	}
 	return op(ctx, scanTarget, writeSource)
 }
+
+func (r *MeshGatewayController) getGatewayClassConfigForGateway(ctx context.Context, gateway *meshv2beta1.MeshGateway) (*meshv2beta1.GatewayClassConfig, error) {
+	gatewayClass, err := r.getGatewayClassForGateway(ctx, gateway)
+	if err != nil {
+		return nil, err
+	}
+	gatewayClassConfig, err := r.getConfigForGatewayClass(ctx, gatewayClass)
+	if err != nil {
+		return nil, err
+	}
+
+	return gatewayClassConfig, nil
+
+}
+
+func (r *MeshGatewayController) getConfigForGatewayClass(ctx context.Context, gatewayClassConfig *meshv2beta1.GatewayClass) (*meshv2beta1.GatewayClassConfig, error) {
+	if gatewayClassConfig == nil {
+		// if we don't have a gateway class we can't fetch the corresponding config
+		return nil, nil
+	}
+
+	config := &meshv2beta1.GatewayClassConfig{}
+	if ref := gatewayClassConfig.Spec.ParametersRef; ref != nil {
+		if ref.Group != meshv2beta1.MeshGroup ||
+			ref.Kind != common.GatewayClassConfig {
+			//TODO @Gateway-Management additionally check for controller name when available
+			return nil, nil
+		}
+
+		if err := r.Client.Get(ctx, types.NamespacedName{Name: ref.Name}, config); err != nil {
+			return nil, client.IgnoreNotFound(err)
+		}
+	}
+	return config, nil
+}
+
+func (r *MeshGatewayController) getGatewayClassForGateway(ctx context.Context, gateway *meshv2beta1.MeshGateway) (*meshv2beta1.GatewayClass, error) {
+	var gatewayClass meshv2beta1.GatewayClass
+	if err := r.Client.Get(ctx, types.NamespacedName{Name: string(gateway.Spec.GatewayClassName)}, &gatewayClass); err != nil {
+		return nil, client.IgnoreNotFound(err)
+	}
+	return &gatewayClass, nil
+}
diff --git a/control-plane/config-entries/controllersv2/mesh_gateway_controller_test.go b/control-plane/config-entries/controllersv2/mesh_gateway_controller_test.go
@@ -5,13 +5,13 @@ package controllersv2
 
 import (
 	"context"
-	"errors"
 	"testing"
 
 	logrtest "github.com/go-logr/logr/testr"
 	"github.com/hashicorp/consul/sdk/testutil"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -40,6 +40,7 @@ func TestMeshGatewayController_Reconcile(t *testing.T) {
 		// postReconcile runs some set of assertions on the state of k8s after Reconcile is called
 		postReconcile func(*testing.T, client.Client)
 	}{
+		// ServiceAccount
 		{
 			name: "MeshGateway created with no existing ServiceAccount",
 			k8sObjects: []runtime.Object{
@@ -86,7 +87,7 @@ func TestMeshGatewayController_Reconcile(t *testing.T) {
 				},
 			},
 			expectedResult: ctrl.Result{},
-			expectedErr:    errors.New("existing resource not owned by controller"),
+			expectedErr:    errResourceNotOwned,
 		},
 		{
 			name: "MeshGateway created with existing ServiceAccount owned by gateway",
@@ -120,6 +121,87 @@ func TestMeshGatewayController_Reconcile(t *testing.T) {
 			expectedResult: ctrl.Result{},
 			expectedErr:    nil, // The Reconcile should be a no-op
 		},
+		// Deployment
+		{
+			name: "MeshGateway created with no existing Deployment",
+			k8sObjects: []runtime.Object{
+				&v2beta1.MeshGateway{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "consul",
+						Name:      "mesh-gateway",
+					},
+				},
+			},
+			request: ctrl.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: "consul",
+					Name:      "mesh-gateway",
+				},
+			},
+			expectedResult: ctrl.Result{},
+			postReconcile: func(t *testing.T, c client.Client) {
+				// Verify Deployment was created
+				key := client.ObjectKey{Namespace: "consul", Name: "mesh-gateway"}
+				assert.NoError(t, c.Get(context.Background(), key, &appsv1.Deployment{}))
+			},
+		},
+		{
+			name: "MeshGateway created with existing Deployment not owned by gateway",
+			k8sObjects: []runtime.Object{
+				&v2beta1.MeshGateway{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "default",
+						Name:      "mesh-gateway",
+					},
+				},
+				&appsv1.Deployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "default",
+						Name:      "mesh-gateway",
+					},
+				},
+			},
+			request: ctrl.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: "default",
+					Name:      "mesh-gateway",
+				},
+			},
+			expectedResult: ctrl.Result{},
+			expectedErr:    errResourceNotOwned,
+		},
+		{
+			name: "MeshGateway created with existing Deployment owned by gateway",
+			k8sObjects: []runtime.Object{
+				&v2beta1.MeshGateway{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "default",
+						Name:      "mesh-gateway",
+						UID:       "abc123",
+					},
+				},
+				&appsv1.Deployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "default",
+						Name:      "mesh-gateway",
+						OwnerReferences: []metav1.OwnerReference{
+							{
+								UID:  "abc123",
+								Name: "mesh-gateway",
+							},
+						},
+					},
+				},
+			},
+			request: ctrl.Request{
+				NamespacedName: types.NamespacedName{
+					Namespace: "default",
+					Name:      "mesh-gateway",
+				},
+			},
+			expectedResult: ctrl.Result{},
+			expectedErr:    nil, // The Reconcile should be a no-op
+		},
 	}
 
 	for _, testCase := range testCases {
@@ -130,7 +212,8 @@ func TestMeshGatewayController_Reconcile(t *testing.T) {
 
 			s := runtime.NewScheme()
 			require.NoError(t, corev1.AddToScheme(s))
-			s.AddKnownTypes(v2beta1.MeshGroupVersion, &v2beta1.MeshGateway{})
+			require.NoError(t, appsv1.AddToScheme(s))
+			s.AddKnownTypes(v2beta1.MeshGroupVersion, &v2beta1.MeshGateway{}, &v2beta1.GatewayClass{}, &v2beta1.GatewayClassConfig{})
 			fakeClient := fake.NewClientBuilder().WithScheme(s).
 				WithRuntimeObjects(testCase.k8sObjects...).
 				Build()
@@ -147,7 +230,8 @@ func TestMeshGatewayController_Reconcile(t *testing.T) {
 
 			res, err := controller.Reconcile(context.Background(), testCase.request)
 			if testCase.expectedErr != nil {
-				require.EqualError(t, err, testCase.expectedErr.Error())
+				//require.EqualError(t, err, testCase.expectedErr.Error())
+				require.ErrorIs(t, err, testCase.expectedErr)
 			} else {
 				require.NoError(t, err)
 			}

diff --git a/control-plane/connect-inject/constants/constants.go b/control-plane/connect-inject/constants/constants.go
@@ -58,6 +58,9 @@ const (
 	// DefaultGracefulShutdownPath is the default path that consul-dataplane uses for graceful shutdown.
 	DefaultGracefulShutdownPath = "/graceful_shutdown"
 
+	//DefaultWANPort is the default port that consul-dataplane uses for WAN.
+	DefaultWANPort = 8443
+
 	// ConsulKubernetesCheckType is the type of health check in Consul for Kubernetes readiness status.
 	ConsulKubernetesCheckType = "kubernetes-readiness"
 

diff --git a/control-plane/gateways/builder.go b/control-plane/gateways/builder.go
@@ -3,14 +3,20 @@
 
 package gateways
 
-import meshv2beta1 "github.com/hashicorp/consul-k8s/control-plane/api/mesh/v2beta1"
+import (
+	meshv2beta1 "github.com/hashicorp/consul-k8s/control-plane/api/mesh/v2beta1"
+)
 
 type meshGatewayBuilder struct {
 	gateway *meshv2beta1.MeshGateway
+	config  GatewayConfig
+	gcc     *meshv2beta1.GatewayClassConfig
 }
 
-func NewMeshGatewayBuilder(gateway *meshv2beta1.MeshGateway) *meshGatewayBuilder {
+func NewMeshGatewayBuilder(gateway *meshv2beta1.MeshGateway, gatewayConfig GatewayConfig, gatewayClassConfig *meshv2beta1.GatewayClassConfig) *meshGatewayBuilder {
 	return &meshGatewayBuilder{
 		gateway: gateway,
+		config:  gatewayConfig,
+		gcc:     gatewayClassConfig,
 	}
 }
diff --git a/control-plane/gateways/config.go b/control-plane/gateways/config.go
@@ -0,0 +1,37 @@
+package gateways
+
+import "github.com/hashicorp/consul-k8s/control-plane/api/common"
+
+// GatewayConfig is a combination of settings relevant to Gateways.
+type GatewayConfig struct {
+	// ImageDataplane is the Consul Dataplane image to use in gateway deployments.
+	ImageDataplane string
+	// ImageConsulK8S is the Consul Kubernetes Control Plane image to use in gateway deployments.
+	ImageConsulK8S string
+	// AuthMethod method used to authenticate with Consul Server.
+	AuthMethod string
+
+	ConsulTenancyConfig common.ConsulTenancyConfig
+
+	// LogLevel is the logging level of the deployed Consul Dataplanes.
+	LogLevel string
+	// LogJSON if JSONLogging has been enabled.
+	LogJSON bool
+	// TLSEnabled is the value of whether or not TLS has been enabled in Consul.
+	TLSEnabled bool
+	// PeeringEnabled toggles whether or not Peering is enabled in Consul.
+	PeeringEnabled bool
+	// ConsulTLSServerName the name of the server running TLS.
+	ConsulTLSServerName string
+	// ConsulCACert contains the Consul Certificate Authority.
+	ConsulCACert string
+	// ConsulConfig configuration for the consul server address.
+	ConsulConfig common.ConsulConfig
+
+	// EnableOpenShift indicates whether we're deploying into an OpenShift environment
+	EnableOpenShift bool
+
+	// MapPrivilegedServicePorts is the value which Consul will add to privileged container port values (ports < 1024)
+	// defined on a Gateway.
+	MapPrivilegedServicePorts int
+}