From bfda29474c6503fd4df5af4aaec1f74c9da55456 Mon Sep 17 00:00:00 2001 From: Sivanantham Chinnaiyan Date: Wed, 22 Jan 2025 18:31:52 +0530 Subject: [PATCH 1/5] fix: AIGatewayRoute controller not watches for HTTPRoute events Signed-off-by: Sivanantham Chinnaiyan --- internal/controller/controller.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 0edc1505..364ea98f 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -70,6 +70,7 @@ func StartControllers(ctx context.Context, config *rest.Config, logger logr.Logg routeC := NewAIGatewayRouteController(c, kubernetes.NewForConfigOrDie(config), logger, sinkChan) if err = ctrl.NewControllerManagedBy(mgr). For(&aigv1a1.AIGatewayRoute{}). + Owns(&gwapiv1.HTTPRoute{}). Complete(routeC); err != nil { return fmt.Errorf("failed to create controller for AIGatewayRoute: %w", err) } From f61d0fc1702f06480a15eb9ef8ab0d597c222695 Mon Sep 17 00:00:00 2001 From: Sivanantham Chinnaiyan Date: Sat, 25 Jan 2025 22:51:53 +0530 Subject: [PATCH 2/5] Set controller reference for HTTPRoute Signed-off-by: Sivanantham Chinnaiyan --- internal/controller/sink.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/controller/sink.go b/internal/controller/sink.go index fa758d8b..b2d51283 100644 --- a/internal/controller/sink.go +++ b/internal/controller/sink.go @@ -14,6 +14,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/yaml" @@ -159,6 +160,10 @@ func (c *configSink) syncAIGatewayRoute(aiGatewayRoute *aigv1a1.AIGatewayRoute) }, Spec: gwapiv1.HTTPRouteSpec{}, } + err := ctrlutil.SetControllerReference(aiGatewayRoute, &httpRoute, c.client.Scheme()) + if err != nil { + c.logger.Error(err, "failed to set controller reference", "namespace", aiGatewayRoute.Namespace, "name", aiGatewayRoute.Name) + } } else if err != nil { c.logger.Error(err, "failed to get HTTPRoute", "namespace", aiGatewayRoute.Namespace, "name", aiGatewayRoute.Name, "error", err) return From cdc9cd5b90346edc4ed841fdaab758ab1d1451f1 Mon Sep 17 00:00:00 2001 From: Sivanantham Chinnaiyan Date: Sat, 25 Jan 2025 22:52:08 +0530 Subject: [PATCH 3/5] Add test Signed-off-by: Sivanantham Chinnaiyan --- .gitignore | 1 + tests/controller/controller_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/.gitignore b/.gitignore index 0c5fd309..0ffadbc8 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ out/ .terraform .idea *for_tests.yaml +.vscode # Files and directories to ignore in the site directory # dependencies diff --git a/tests/controller/controller_test.go b/tests/controller/controller_test.go index a5b82bf7..d23c3ac6 100644 --- a/tests/controller/controller_test.go +++ b/tests/controller/controller_test.go @@ -237,6 +237,30 @@ func TestStartControllers(t *testing.T) { return true }, 30*time.Second, 200*time.Millisecond) }) + + t.Run("verify http route 'route1' is recreated if deleted", func(t *testing.T) { + // When the HTTPRoute resource is deleted, the AIGatewayRoute controller should recreate it. + routeName := "route1" + routeNamespace := "default" + + // Delete the HTTPRoute resource. + err := c.Delete(ctx, &gwapiv1.HTTPRoute{ObjectMeta: metav1.ObjectMeta{Name: routeName, Namespace: routeNamespace}}) + require.NoError(t, err) + + // Verify that the HTTPRoute resource is recreated. + require.Eventually(t, func() bool { + var httpRoute gwapiv1.HTTPRoute + err := c.Get(ctx, client.ObjectKey{Name: routeName, Namespace: routeNamespace}, &httpRoute) + if err != nil { + t.Logf("failed to get http route %s: %v", routeName, err) + return false + } else if httpRoute.DeletionTimestamp != nil { + // Make sure it is not the HTTPRoute resource that is being deleted. + return false + } + return true + }, 30*time.Second, 200*time.Millisecond) + }) } func TestAIGatewayRouteController(t *testing.T) { From c4ebc8fa5d22de67f03b2cdaaeaebd891563d1ed Mon Sep 17 00:00:00 2001 From: Sivanantham Chinnaiyan Date: Mon, 27 Jan 2025 20:12:13 +0530 Subject: [PATCH 4/5] Set controller reference and watch for resources created by controller Signed-off-by: Sivanantham Chinnaiyan --- internal/controller/ai_gateway_route.go | 32 +++++++++----------- internal/controller/ai_gateway_route_test.go | 8 ++--- internal/controller/controller.go | 5 +++ internal/controller/sink.go | 30 +++++++++--------- 4 files changed, 39 insertions(+), 36 deletions(-) diff --git a/internal/controller/ai_gateway_route.go b/internal/controller/ai_gateway_route.go index 0808c769..ef5c72f5 100644 --- a/internal/controller/ai_gateway_route.go +++ b/internal/controller/ai_gateway_route.go @@ -12,6 +12,7 @@ import ( "k8s.io/client-go/kubernetes" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + ctrlutil "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" @@ -88,14 +89,13 @@ func (c *aiGatewayRouteController) Reconcile(ctx context.Context, req reconcile. if !unversioned && len(gvks) == 1 { aiGatewayRoute.SetGroupVersionKind(gvks[0]) } - ownerRef := ownerReferenceForAIGatewayRoute(&aiGatewayRoute) - if err := c.ensuresExtProcConfigMapExists(ctx, &aiGatewayRoute, ownerRef); err != nil { + if err := c.ensuresExtProcConfigMapExists(ctx, &aiGatewayRoute); err != nil { logger.Error(err, "Failed to reconcile extProc config map") return ctrl.Result{}, err } - if err := c.reconcileExtProcExtensionPolicy(ctx, &aiGatewayRoute, ownerRef); err != nil { + if err := c.reconcileExtProcExtensionPolicy(ctx, &aiGatewayRoute); err != nil { logger.Error(err, "Failed to reconcile extension policy") return ctrl.Result{}, err } @@ -106,7 +106,7 @@ func (c *aiGatewayRouteController) Reconcile(ctx context.Context, req reconcile. // reconcileExtProcExtensionPolicy creates or updates the extension policy for the external process. // It only changes the target references. -func (c *aiGatewayRouteController) reconcileExtProcExtensionPolicy(ctx context.Context, aiGatewayRoute *aigv1a1.AIGatewayRoute, ownerRef []metav1.OwnerReference) error { +func (c *aiGatewayRouteController) reconcileExtProcExtensionPolicy(ctx context.Context, aiGatewayRoute *aigv1a1.AIGatewayRoute) error { var existingPolicy egv1a1.EnvoyExtensionPolicy if err := c.client.Get(ctx, client.ObjectKey{Name: extProcName(aiGatewayRoute), Namespace: aiGatewayRoute.Namespace}, &existingPolicy); err == nil { existingPolicy.Spec.PolicyTargetReferences.TargetRefs = aiGatewayRoute.Spec.TargetRefs @@ -121,7 +121,7 @@ func (c *aiGatewayRouteController) reconcileExtProcExtensionPolicy(ctx context.C port := gwapiv1.PortNumber(1063) objNs := gwapiv1.Namespace(aiGatewayRoute.Namespace) extPolicy := &egv1a1.EnvoyExtensionPolicy{ - ObjectMeta: metav1.ObjectMeta{Name: extProcName(aiGatewayRoute), Namespace: aiGatewayRoute.Namespace, OwnerReferences: ownerRef}, + ObjectMeta: metav1.ObjectMeta{Name: extProcName(aiGatewayRoute), Namespace: aiGatewayRoute.Namespace}, Spec: egv1a1.EnvoyExtensionPolicySpec{ PolicyTargetReferences: egv1a1.PolicyTargetReferences{TargetRefs: aiGatewayRoute.Spec.TargetRefs}, ExtProc: []egv1a1.ExtProc{{ @@ -143,6 +143,9 @@ func (c *aiGatewayRouteController) reconcileExtProcExtensionPolicy(ctx context.C }}, }, } + if err := ctrlutil.SetControllerReference(aiGatewayRoute, extPolicy, c.client.Scheme()); err != nil { + c.logger.Error(err, "failed to set controller reference for envoy extension policy", "namespace", extPolicy.Namespace, "name", extPolicy.Name) + } if err := c.client.Create(ctx, extPolicy); client.IgnoreAlreadyExists(err) != nil { return fmt.Errorf("failed to create extension policy: %w", err) } @@ -151,7 +154,7 @@ func (c *aiGatewayRouteController) reconcileExtProcExtensionPolicy(ctx context.C // ensuresExtProcConfigMapExists ensures that a configmap exists for the external process. // This must happen before the external process deployment is created. -func (c *aiGatewayRouteController) ensuresExtProcConfigMapExists(ctx context.Context, aiGatewayRoute *aigv1a1.AIGatewayRoute, ownerRef []metav1.OwnerReference) error { +func (c *aiGatewayRouteController) ensuresExtProcConfigMapExists(ctx context.Context, aiGatewayRoute *aigv1a1.AIGatewayRoute) error { name := extProcName(aiGatewayRoute) // Check if a configmap exists for extproc exists, and if not, create one with the default config. _, err := c.kube.CoreV1().ConfigMaps(aiGatewayRoute.Namespace).Get(ctx, name, metav1.GetOptions{}) @@ -159,12 +162,14 @@ func (c *aiGatewayRouteController) ensuresExtProcConfigMapExists(ctx context.Con if client.IgnoreNotFound(err) == nil { configMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: aiGatewayRoute.Namespace, - OwnerReferences: ownerRef, + Name: name, + Namespace: aiGatewayRoute.Namespace, }, Data: map[string]string{expProcConfigFileName: filterconfig.DefaultConfig}, } + if err := ctrlutil.SetControllerReference(aiGatewayRoute, configMap, c.client.Scheme()); err != nil { + c.logger.Error(err, "failed to set controller reference for service", "namespace", configMap.Namespace, "name", configMap.Name) + } _, err = c.kube.CoreV1().ConfigMaps(aiGatewayRoute.Namespace).Create(ctx, configMap, metav1.CreateOptions{}) if err != nil { return err @@ -178,15 +183,6 @@ func extProcName(route *aigv1a1.AIGatewayRoute) string { return fmt.Sprintf("ai-eg-route-extproc-%s", route.Name) } -func ownerReferenceForAIGatewayRoute(aiGatewayRoute *aigv1a1.AIGatewayRoute) []metav1.OwnerReference { - return []metav1.OwnerReference{{ - APIVersion: aiGatewayRoute.APIVersion, - Kind: aiGatewayRoute.Kind, - Name: aiGatewayRoute.Name, - UID: aiGatewayRoute.UID, - }} -} - func applyExtProcDeploymentConfigUpdate(d *appsv1.DeploymentSpec, filterConfig *aigv1a1.AIGatewayFilterConfig) { if filterConfig == nil || filterConfig.ExternalProcess == nil { return diff --git a/internal/controller/ai_gateway_route_test.go b/internal/controller/ai_gateway_route_test.go index 9c22f25c..082f6408 100644 --- a/internal/controller/ai_gateway_route_test.go +++ b/internal/controller/ai_gateway_route_test.go @@ -37,7 +37,7 @@ func TestAIGatewayRouteController_ensuresExtProcConfigMapExists(t *testing.T) { ownerRef := []metav1.OwnerReference{{APIVersion: "v1", Kind: "Kind", Name: "Name"}} aiGatewayRoute := &aigv1a1.AIGatewayRoute{ObjectMeta: metav1.ObjectMeta{Name: "myroute", Namespace: "default"}} - err := c.ensuresExtProcConfigMapExists(context.Background(), aiGatewayRoute, ownerRef) + err := c.ensuresExtProcConfigMapExists(context.Background(), aiGatewayRoute) require.NoError(t, err) configMap, err := c.kube.CoreV1().ConfigMaps("default").Get(context.Background(), extProcName(aiGatewayRoute), metav1.GetOptions{}) @@ -48,7 +48,7 @@ func TestAIGatewayRouteController_ensuresExtProcConfigMapExists(t *testing.T) { require.Equal(t, filterconfig.DefaultConfig, configMap.Data[expProcConfigFileName]) // Doing it again should not fail. - err = c.ensuresExtProcConfigMapExists(context.Background(), aiGatewayRoute, ownerRef) + err = c.ensuresExtProcConfigMapExists(context.Background(), aiGatewayRoute) require.NoError(t, err) } @@ -67,7 +67,7 @@ func TestAIGatewayRouteController_reconcileExtProcExtensionPolicy(t *testing.T) }, }, } - err := c.reconcileExtProcExtensionPolicy(context.Background(), aiGatewayRoute, ownerRef) + err := c.reconcileExtProcExtensionPolicy(context.Background(), aiGatewayRoute) require.NoError(t, err) var extPolicy egv1a1.EnvoyExtensionPolicy err = c.client.Get(context.Background(), client.ObjectKey{Name: extProcName(aiGatewayRoute), Namespace: "default"}, &extPolicy) @@ -94,7 +94,7 @@ func TestAIGatewayRouteController_reconcileExtProcExtensionPolicy(t *testing.T) {LocalPolicyTargetReference: gwapiv1a2.LocalPolicyTargetReference{Name: "cat"}}, {LocalPolicyTargetReference: gwapiv1a2.LocalPolicyTargetReference{Name: "bird"}}, } - err = c.reconcileExtProcExtensionPolicy(context.Background(), aiGatewayRoute, ownerRef) + err = c.reconcileExtProcExtensionPolicy(context.Background(), aiGatewayRoute) require.NoError(t, err) err = c.client.Get(context.Background(), client.ObjectKey{Name: extProcName(aiGatewayRoute), Namespace: "default"}, &extPolicy) diff --git a/internal/controller/controller.go b/internal/controller/controller.go index 364ea98f..93e49253 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -6,6 +6,8 @@ import ( egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -70,7 +72,10 @@ func StartControllers(ctx context.Context, config *rest.Config, logger logr.Logg routeC := NewAIGatewayRouteController(c, kubernetes.NewForConfigOrDie(config), logger, sinkChan) if err = ctrl.NewControllerManagedBy(mgr). For(&aigv1a1.AIGatewayRoute{}). + Owns(&egv1a1.EnvoyExtensionPolicy{}). Owns(&gwapiv1.HTTPRoute{}). + Owns(&appsv1.Deployment{}). + Owns(&corev1.Service{}). Complete(routeC); err != nil { return fmt.Errorf("failed to create controller for AIGatewayRoute: %w", err) } diff --git a/internal/controller/sink.go b/internal/controller/sink.go index b2d51283..2418df22 100644 --- a/internal/controller/sink.go +++ b/internal/controller/sink.go @@ -154,15 +154,13 @@ func (c *configSink) syncAIGatewayRoute(aiGatewayRoute *aigv1a1.AIGatewayRoute) // This means that this AIGatewayRoute is a new one. httpRoute = gwapiv1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ - Name: aiGatewayRoute.Name, - Namespace: aiGatewayRoute.Namespace, - OwnerReferences: ownerReferenceForAIGatewayRoute(aiGatewayRoute), + Name: aiGatewayRoute.Name, + Namespace: aiGatewayRoute.Namespace, }, Spec: gwapiv1.HTTPRouteSpec{}, } - err := ctrlutil.SetControllerReference(aiGatewayRoute, &httpRoute, c.client.Scheme()) - if err != nil { - c.logger.Error(err, "failed to set controller reference", "namespace", aiGatewayRoute.Namespace, "name", aiGatewayRoute.Name) + if err := ctrlutil.SetControllerReference(aiGatewayRoute, &httpRoute, c.client.Scheme()); err != nil { + c.logger.Error(err, "failed to set controller reference for http route", "namespace", httpRoute.Namespace, "name", httpRoute.Name) } } else if err != nil { c.logger.Error(err, "failed to get HTTPRoute", "namespace", aiGatewayRoute.Namespace, "name", aiGatewayRoute.Name, "error", err) @@ -425,10 +423,9 @@ func (c *configSink) syncExtProcDeployment(ctx context.Context, aiGatewayRoute * if client.IgnoreNotFound(err) == nil { deployment = &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: aiGatewayRoute.Namespace, - OwnerReferences: ownerReferenceForAIGatewayRoute(aiGatewayRoute), - Labels: labels, + Name: name, + Namespace: aiGatewayRoute.Namespace, + Labels: labels, }, Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{MatchLabels: labels}, @@ -464,6 +461,9 @@ func (c *configSink) syncExtProcDeployment(ctx context.Context, aiGatewayRoute * }, }, } + if err := ctrlutil.SetControllerReference(aiGatewayRoute, deployment, c.client.Scheme()); err != nil { + c.logger.Error(err, "failed to set controller reference for deployment", "namespace", deployment.Namespace, "name", deployment.Name) + } updatedSpec, err := c.mountBackendSecurityPolicySecrets(&deployment.Spec.Template.Spec, aiGatewayRoute) if err == nil { deployment.Spec.Template.Spec = *updatedSpec @@ -491,10 +491,9 @@ func (c *configSink) syncExtProcDeployment(ctx context.Context, aiGatewayRoute * // This is static, so we don't need to update it. service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: aiGatewayRoute.Namespace, - OwnerReferences: ownerReferenceForAIGatewayRoute(aiGatewayRoute), - Labels: labels, + Name: name, + Namespace: aiGatewayRoute.Namespace, + Labels: labels, }, Spec: corev1.ServiceSpec{ Selector: labels, @@ -508,6 +507,9 @@ func (c *configSink) syncExtProcDeployment(ctx context.Context, aiGatewayRoute * }, }, } + if err = ctrlutil.SetControllerReference(aiGatewayRoute, service, c.client.Scheme()); err != nil { + c.logger.Error(err, "failed to set controller reference for service", "namespace", service.Namespace, "name", service.Name) + } if _, err = c.kube.CoreV1().Services(aiGatewayRoute.Namespace).Create(ctx, service, metav1.CreateOptions{}); client.IgnoreAlreadyExists(err) != nil { return fmt.Errorf("failed to create Service %s.%s: %w", name, aiGatewayRoute.Namespace, err) } From f58a7b737c6efd8451010e22ab3bcc9123f29c35 Mon Sep 17 00:00:00 2001 From: Sivanantham Chinnaiyan Date: Tue, 28 Jan 2025 16:09:47 +0530 Subject: [PATCH 5/5] Add tests Signed-off-by: Sivanantham Chinnaiyan --- internal/controller/ai_gateway_route_test.go | 15 +++-- tests/controller/controller_test.go | 70 ++++++++++++++++++-- 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/internal/controller/ai_gateway_route_test.go b/internal/controller/ai_gateway_route_test.go index 082f6408..80bc2abe 100644 --- a/internal/controller/ai_gateway_route_test.go +++ b/internal/controller/ai_gateway_route_test.go @@ -33,9 +33,11 @@ func Test_extProcName(t *testing.T) { func TestAIGatewayRouteController_ensuresExtProcConfigMapExists(t *testing.T) { c := &aiGatewayRouteController{client: fake.NewClientBuilder().WithScheme(scheme).Build()} c.kube = fake2.NewClientset() - - ownerRef := []metav1.OwnerReference{{APIVersion: "v1", Kind: "Kind", Name: "Name"}} - aiGatewayRoute := &aigv1a1.AIGatewayRoute{ObjectMeta: metav1.ObjectMeta{Name: "myroute", Namespace: "default"}} + name := "myroute" + ownerRef := []metav1.OwnerReference{ + {APIVersion: "aigateway.envoyproxy.io/v1alpha1", Kind: "AIGatewayRoute", Name: name, Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}, + } + aiGatewayRoute := &aigv1a1.AIGatewayRoute{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "default"}} err := c.ensuresExtProcConfigMapExists(context.Background(), aiGatewayRoute) require.NoError(t, err) @@ -54,10 +56,13 @@ func TestAIGatewayRouteController_ensuresExtProcConfigMapExists(t *testing.T) { func TestAIGatewayRouteController_reconcileExtProcExtensionPolicy(t *testing.T) { c := &aiGatewayRouteController{client: fake.NewClientBuilder().WithScheme(scheme).Build()} - ownerRef := []metav1.OwnerReference{{APIVersion: "v1", Kind: "Kind", Name: "Name"}} + name := "myroute" + ownerRef := []metav1.OwnerReference{ + {APIVersion: "aigateway.envoyproxy.io/v1alpha1", Kind: "AIGatewayRoute", Name: name, Controller: ptr.To(true), BlockOwnerDeletion: ptr.To(true)}, + } aiGatewayRoute := &aigv1a1.AIGatewayRoute{ ObjectMeta: metav1.ObjectMeta{ - Name: "myroute", + Name: name, Namespace: "default", }, Spec: aigv1a1.AIGatewayRouteSpec{ diff --git a/tests/controller/controller_test.go b/tests/controller/controller_test.go index d23c3ac6..711bc1c0 100644 --- a/tests/controller/controller_test.go +++ b/tests/controller/controller_test.go @@ -16,10 +16,11 @@ import ( egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" "github.com/go-logr/logr" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - klog "k8s.io/klog/v2" + "k8s.io/klog/v2" "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -141,6 +142,7 @@ func TestStartControllers(t *testing.T) { require.Len(t, deployment.OwnerReferences, 1) require.Equal(t, aiGatewayRoute.Name, deployment.OwnerReferences[0].Name) require.Equal(t, "AIGatewayRoute", deployment.OwnerReferences[0].Kind) + require.True(t, *deployment.OwnerReferences[0].Controller) require.Equal(t, int32(5), *deployment.Spec.Replicas) require.Equal(t, resourceReq, &deployment.Spec.Template.Spec.Containers[0].Resources) @@ -154,6 +156,7 @@ func TestStartControllers(t *testing.T) { require.Len(t, service.OwnerReferences, 1) require.Equal(t, aiGatewayRoute.Name, service.OwnerReferences[0].Name) require.Equal(t, "AIGatewayRoute", service.OwnerReferences[0].Kind) + require.True(t, *service.OwnerReferences[0].Controller) extPolicy := egv1a1.EnvoyExtensionPolicy{} err = c.Get(ctx, client.ObjectKey{Name: extProcName(route), Namespace: "default"}, &extPolicy) @@ -163,6 +166,7 @@ func TestStartControllers(t *testing.T) { } require.Len(t, extPolicy.OwnerReferences, 1) require.Equal(t, aiGatewayRoute.Name, extPolicy.OwnerReferences[0].Name) + require.True(t, *extPolicy.OwnerReferences[0].Controller) configMap, err := k.CoreV1().ConfigMaps("default").Get(ctx, extProcName(route), metav1.GetOptions{}) if err != nil { @@ -171,6 +175,7 @@ func TestStartControllers(t *testing.T) { } require.Len(t, configMap.OwnerReferences, 1) require.Equal(t, aiGatewayRoute.Name, configMap.OwnerReferences[0].Name) + require.True(t, *configMap.OwnerReferences[0].Controller) require.Contains(t, configMap.Data, "extproc-config.yaml") return true }, 30*time.Second, 200*time.Millisecond) @@ -238,15 +243,32 @@ func TestStartControllers(t *testing.T) { }, 30*time.Second, 200*time.Millisecond) }) - t.Run("verify http route 'route1' is recreated if deleted", func(t *testing.T) { - // When the HTTPRoute resource is deleted, the AIGatewayRoute controller should recreate it. + t.Run("verify resources created by AIGatewayRoute controller are recreated if deleted", func(t *testing.T) { routeName := "route1" routeNamespace := "default" - // Delete the HTTPRoute resource. - err := c.Delete(ctx, &gwapiv1.HTTPRoute{ObjectMeta: metav1.ObjectMeta{Name: routeName, Namespace: routeNamespace}}) + // When the EnvoyExtensionPolicy is deleted, the controller should recreate it. + policyName := extProcName(routeName) + policyNamespace := routeNamespace + err := c.Delete(ctx, &egv1a1.EnvoyExtensionPolicy{ObjectMeta: metav1.ObjectMeta{Name: policyName, Namespace: policyNamespace}}) require.NoError(t, err) + // Verify that the HTTPRoute resource is recreated. + require.Eventually(t, func() bool { + var egExtPolicy egv1a1.EnvoyExtensionPolicy + err := c.Get(ctx, client.ObjectKey{Name: policyName, Namespace: policyNamespace}, &egExtPolicy) + if err != nil { + t.Logf("failed to get envoy extension policy %s: %v", policyName, err) + return false + } else if egExtPolicy.DeletionTimestamp != nil { + // Make sure it is not the EnvoyExtensionPolicy resource that is being deleted. + return false + } + return true + }, 30*time.Second, 200*time.Millisecond) + // When the HTTPRoute resource is deleted, the controller should recreate it. + err = c.Delete(ctx, &gwapiv1.HTTPRoute{ObjectMeta: metav1.ObjectMeta{Name: routeName, Namespace: routeNamespace}}) + require.NoError(t, err) // Verify that the HTTPRoute resource is recreated. require.Eventually(t, func() bool { var httpRoute gwapiv1.HTTPRoute @@ -260,6 +282,44 @@ func TestStartControllers(t *testing.T) { } return true }, 30*time.Second, 200*time.Millisecond) + + // When extproc deployment is deleted, the controller should recreate it. + deployName := extProcName(routeName) + deployNamespace := routeNamespace + err = c.Delete(ctx, &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: deployName, Namespace: deployNamespace}}) + require.NoError(t, err) + // Verify that the deployment is recreated. + require.Eventually(t, func() bool { + var deployment appsv1.Deployment + err := c.Get(ctx, client.ObjectKey{Name: deployName, Namespace: deployNamespace}, &deployment) + if err != nil { + t.Logf("failed to get deployment %s: %v", deployName, err) + return false + } else if deployment.DeletionTimestamp != nil { + // Make sure it is not the deployment resource that is being deleted. + return false + } + return true + }, 30*time.Second, 200*time.Millisecond) + + // When extproc service is deleted, the controller should recreate it. + serviceName := extProcName(routeName) + serviceNamespace := routeNamespace + err = c.Delete(ctx, &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: serviceName, Namespace: serviceNamespace}}) + require.NoError(t, err) + // Verify that the service is recreated. + require.Eventually(t, func() bool { + var service corev1.Service + err := c.Get(ctx, client.ObjectKey{Name: serviceName, Namespace: serviceNamespace}, &service) + if err != nil { + t.Logf("failed to get service %s: %v", serviceName, err) + return false + } else if service.DeletionTimestamp != nil { + // Make sure it is not the service resource that is being deleted. + return false + } + return true + }, 30*time.Second, 200*time.Millisecond) }) }