Net 6575- Modify Gateways Cleanup Job to cleanup v2 resources (#3317)

* cleanup existing gatewayclasses and gatewayclassconfigs * add tests for v2 * fixed test to actually run against v2 resources, found issue where loop was failing out early * add TODO with jira ticket * cleanup debug line * delete extra newline
hashicorp · Jan 5, 2024 · 1440631 · 1440631
1 parent 046756d
commit 1440631
Show file tree

Hide file tree

Showing 4 changed files with 328 additions and 13 deletions.
diff --git a/charts/consul/templates/gateway-cleanup-clusterrole.yaml b/charts/consul/templates/gateway-cleanup-clusterrole.yaml
@@ -24,6 +24,15 @@ rules:
     verbs:
     - get
     - delete
+  - apiGroups:
+    - mesh.consul.hashicorp.com
+    resources:
+    - gatewayclassconfigs
+    - gatewayclasses
+    - meshgateways
+    verbs:
+    - get
+    - delete
 {{- if .Values.global.enablePodSecurityPolicies }}
   - apiGroups: ["policy"]
     resources: ["podsecuritypolicies"]

diff --git a/charts/consul/templates/gateway-cleanup-job.yaml b/charts/consul/templates/gateway-cleanup-job.yaml
@@ -52,10 +52,10 @@ spec:
             limits:
               memory: "50Mi"
               cpu: "50m"
-            volumeMounts:
-              - name: config
-                mountPath: /consul/config
-                readOnly: true
+          volumeMounts:
+            - name: config
+              mountPath: /consul/config
+              readOnly: true
       {{- if .Values.global.acls.tolerations }}
       tolerations:
         {{ tpl .Values.global.acls.tolerations . | indent 8 | trim }}

diff --git a/control-plane/subcommand/gateway-cleanup/command.go b/control-plane/subcommand/gateway-cleanup/command.go
@@ -8,6 +8,8 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
+	"os"
 	"sync"
 	"time"
 
@@ -19,29 +21,44 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
+	k8syaml "sigs.k8s.io/yaml"
 
+	"github.com/hashicorp/consul-k8s/control-plane/api/mesh/v2beta1"
 	"github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1"
 	"github.com/hashicorp/consul-k8s/control-plane/subcommand"
 	"github.com/hashicorp/consul-k8s/control-plane/subcommand/flags"
 )
 
+const (
+	gatewayConfigFilename  = "/consul/config/config.yaml"
+	resourceConfigFilename = "/consul/config/resources.json"
+)
+
 type Command struct {
 	UI cli.Ui
 
 	flags *flag.FlagSet
 	k8s   *flags.K8SFlags
 
-	flagGatewayClassName       string
-	flagGatewayClassConfigName string
+	flagGatewayClassName           string
+	flagGatewayClassConfigName     string
+	flagGatewayConfigLocation      string
+	flagResourceConfigFileLocation string
 
 	k8sClient client.Client
 
 	once sync.Once
 	help string
 
+	gatewayConfig gatewayConfig
+
 	ctx context.Context
 }
 
+type gatewayConfig struct {
+	GatewayClassConfigs []*v2beta1.GatewayClassConfig `yaml:"gatewayClassConfigs"`
+}
+
 func (c *Command) init() {
 	c.flags = flag.NewFlagSet("", flag.ContinueOnError)
 
@@ -50,6 +67,12 @@ func (c *Command) init() {
 	c.flags.StringVar(&c.flagGatewayClassConfigName, "gateway-class-config-name", "",
 		"Name of Kubernetes GatewayClassConfig to delete.")
 
+	c.flags.StringVar(&c.flagGatewayConfigLocation, "gateway-config-file-location", gatewayConfigFilename,
+		"specify a different location for where the gateway config file is")
+
+	c.flags.StringVar(&c.flagResourceConfigFileLocation, "resource-config-file-location", resourceConfigFilename,
+		"specify a different location for where the gateway resource config file is")
+
 	c.k8s = &flags.K8SFlags{}
 	flags.Merge(c.flags, c.k8s.Flags())
 	c.help = flags.Usage(help, c.flags)
@@ -93,6 +116,11 @@ func (c *Command) Run(args []string) int {
 			return 1
 		}
 
+		if err := v2beta1.AddMeshToScheme(s); err != nil {
+			c.UI.Error(fmt.Sprintf("Could not add consul-k8s schema: %s", err))
+			return 1
+		}
+
 		c.k8sClient, err = client.New(config, client.Options{Scheme: s})
 		if err != nil {
 			c.UI.Error(fmt.Sprintf("Error initializing Kubernetes client: %s", err))
@@ -102,32 +130,59 @@ func (c *Command) Run(args []string) int {
 
 	// do the cleanup
 
+	//V1 Cleanup
+	err = c.deleteV1GatewayClassAndGatewayClasConfig()
+	if err != nil {
+		c.UI.Error(err.Error())
+		return 1
+	}
+
+	//V2 Cleanup
+	err = c.loadGatewayConfigs()
+	if err != nil {
+
+		c.UI.Error(err.Error())
+		return 1
+	}
+	err = c.deleteV2GatewayClassAndClassConfigs()
+	if err != nil {
+		c.UI.Error(err.Error())
+
+		return 1
+	}
+
+	return 0
+}
+
+func (c *Command) deleteV1GatewayClassAndGatewayClasConfig() error {
 	// find the class config and mark it for deletion first so that we
 	// can do an early return if the gateway class isn't found
 	config := &v1alpha1.GatewayClassConfig{}
-	err = c.k8sClient.Get(context.Background(), types.NamespacedName{Name: c.flagGatewayClassConfigName}, config)
+	err := c.k8sClient.Get(context.Background(), types.NamespacedName{Name: c.flagGatewayClassConfigName}, config)
 	if err != nil {
+
 		if k8serrors.IsNotFound(err) {
 			// no gateway class config, just ignore and return
-			return 0
+			return nil
 		}
 		c.UI.Error(err.Error())
-		return 1
+		return err
 	}
 
 	// ignore any returned errors
 	_ = c.k8sClient.Delete(context.Background(), config)
 
 	// find the gateway class
+
 	gatewayClass := &gwv1beta1.GatewayClass{}
 	err = c.k8sClient.Get(context.Background(), types.NamespacedName{Name: c.flagGatewayClassName}, gatewayClass)
 	if err != nil {
 		if k8serrors.IsNotFound(err) {
 			// no gateway class, just ignore and return
-			return 0
+			return nil
 		}
 		c.UI.Error(err.Error())
-		return 1
+		return err
 	}
 
 	// ignore any returned errors
@@ -151,8 +206,7 @@ func (c *Command) Run(args []string) int {
 		// if we failed, return 0 anyway after logging the error
 		// since we don't want to block someone from uninstallation
 	}
-
-	return 0
+	return nil
 }
 
 func (c *Command) validateFlags() error {
@@ -192,3 +246,89 @@ func exponentialBackoffWithMaxIntervalAndTime() *backoff.ExponentialBackOff {
 	backoff.Reset()
 	return backoff
 }
+
+// loadGatewayConfigs reads and loads the configs from `/consul/config/config.yaml`, if this file does not exist nothing is done.
+func (c *Command) loadGatewayConfigs() error {
+	file, err := os.Open(c.flagGatewayConfigLocation)
+	if err != nil {
+		if os.IsNotExist(err) {
+			c.UI.Warn(fmt.Sprintf("gateway configuration file not found, skipping gateway configuration, filename: %s", c.flagGatewayConfigLocation))
+			return nil
+		}
+		c.UI.Error(fmt.Sprintf("Error opening gateway configuration file %s: %s", c.flagGatewayConfigLocation, err))
+		return err
+	}
+
+	config, err := io.ReadAll(file)
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error reading gateway configuration file %s: %s", c.flagGatewayConfigLocation, err))
+		return err
+	}
+
+	err = k8syaml.Unmarshal(config, &c.gatewayConfig)
+	if err != nil {
+		c.UI.Error(fmt.Sprintf("Error decoding gateway config file: %s", err))
+		return err
+	}
+
+	if err := file.Close(); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *Command) deleteV2GatewayClassAndClassConfigs() error {
+	for _, gcc := range c.gatewayConfig.GatewayClassConfigs {
+
+		// find the class config and mark it for deletion first so that we
+		// can do an early return if the gateway class isn't found
+		config := &v2beta1.GatewayClassConfig{}
+		err := c.k8sClient.Get(context.Background(), types.NamespacedName{Name: gcc.Name, Namespace: gcc.Namespace}, config)
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				// no gateway class config, just ignore and continue
+				continue
+			}
+			return err
+		}
+
+		// ignore any returned errors
+		_ = c.k8sClient.Delete(context.Background(), config)
+
+		// find the gateway class
+		gatewayClass := &v2beta1.GatewayClass{}
+		//TODO: NET-6838 To pull the GatewayClassName from the Configmap
+		err = c.k8sClient.Get(context.Background(), types.NamespacedName{Name: gcc.Name, Namespace: gcc.Namespace}, gatewayClass)
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				// no gateway class, just ignore and continue
+				continue
+			}
+			return err
+		}
+
+		// ignore any returned errors
+		_ = c.k8sClient.Delete(context.Background(), gatewayClass)
+
+		// make sure they're gone
+		if err := backoff.Retry(func() error {
+			err = c.k8sClient.Get(context.Background(), types.NamespacedName{Name: gcc.Name, Namespace: gcc.Namespace}, config)
+			if err == nil || !k8serrors.IsNotFound(err) {
+				return errors.New("gateway class config still exists")
+			}
+
+			err = c.k8sClient.Get(context.Background(), types.NamespacedName{Name: gcc.Name, Namespace: gcc.Namespace}, gatewayClass)
+			if err == nil || !k8serrors.IsNotFound(err) {
+				return errors.New("gateway class still exists")
+			}
+
+			return nil
+		}, exponentialBackoffWithMaxIntervalAndTime()); err != nil {
+			c.UI.Error(err.Error())
+			// if we failed, return 0 anyway after logging the error
+			// since we don't want to block someone from uninstallation
+		}
+	}
+
+	return nil
+}