Merge pull request flux-iac#763 from weaveworks/bbp-support-wildcard-…

…list

feat: support watching all Terraform objects in namespace

cmd/branch-planner/informer.go

@@ -73,15 +73,15 @@ func createProvider(ctx context.Context, clusterClient client.Client, configMapN
 	return gitProvider, nil
 }
 
-func createSharedInformer(ctx context.Context, client client.Client, dynamicClient dynamic.Interface) (cache.SharedIndexInformer, error) {
+func createSharedInformer(_ context.Context, client client.Client, dynamicClient dynamic.Interface) (cache.SharedIndexInformer, error) {
 	restMapper := client.RESTMapper()
 	mapping, err := restMapper.RESTMapping(tfv1alpha2.GroupVersion.WithKind(tfv1alpha2.TerraformKind).GroupKind())
 	if err != nil {
 		return nil, fmt.Errorf("failed to look up mapping for CRD: %w", err)
 	}
 
 	tweakListOptionsFunc := func(options *metav1.ListOptions) {
-		options.LabelSelector = fmt.Sprintf("%s=%s", planner.LabelKey, planner.LabelValue)
+		options.LabelSelector = fmt.Sprintf("%s=%s", config.LabelKey, config.LabelValue)
 	}
 
 	factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, time.Minute, corev1.NamespaceAll, tweakListOptionsFunc)

internal/config/config.go

@@ -11,6 +11,12 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+const (
+	LabelKey            = "infra.weave.works/branch-planner"
+	LabelValue          = "true"
+	LabelPRIDKey string = "infra.weave.works/pr-id"
+)
+
 // Example ConfigMap
 //
 // The secret is a reference to a secret with a 'token' key.

internal/informer/branch-planner/informer.go

@@ -14,6 +14,7 @@ import (
 	"github.com/go-logr/logr"
 	giturl "github.com/kubescape/go-git-url"
 	tfv1alpha2 "github.com/weaveworks/tf-controller/api/v1alpha2"
+	"github.com/weaveworks/tf-controller/internal/config"
 	"github.com/weaveworks/tf-controller/internal/git/provider"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -104,12 +105,6 @@ func (i *Informer) SetDeleteHandler(fn func(interface{})) {
 	i.handlers.DeleteFunc = fn
 }
 
-const (
-	LabelKey            = "infra.weave.works/branch-planner"
-	LabelValue          = "true"
-	LabelPRIDKey string = "infra.weave.works/pr-id"
-)
-
 func (i *Informer) addHandler(obj interface{}) {}
 
 func (i *Informer) updateHandler(oldObj, newObj interface{}) {
@@ -145,14 +140,14 @@ func (i *Informer) updateHandler(oldObj, newObj interface{}) {
 		return
 	}
 
-	if new.Labels[LabelKey] != LabelValue {
+	if new.Labels[config.LabelKey] != config.LabelValue {
 		i.log.Info("Terraform object is not managed by the branch-based planner")
 
 		return
 	}
 
 	if !i.isNewPlan(old, new) {
-		i.log.Info("Plan not updated", "namespace", new.Namespace, "name", new.Name, "pr-id", new.Labels[LabelPRIDKey])
+		i.log.Info("Plan not updated", "namespace", new.Namespace, "name", new.Name, "pr-id", new.Labels[config.LabelPRIDKey])
 
 		return
 	}
@@ -173,17 +168,17 @@ func (i *Informer) updateHandler(oldObj, newObj interface{}) {
 		return
 	}
 
-	i.log.Info("Updated plan", "pr-id", new.Labels[LabelPRIDKey])
+	i.log.Info("Updated plan", "pr-id", new.Labels[config.LabelPRIDKey])
 
 	repo, err := i.getRepo(ctx, new)
 	if err != nil {
 		i.log.Error(err, "failed getting repository")
 		return
 	}
 
-	number, err := strconv.Atoi(new.Labels[LabelPRIDKey])
+	number, err := strconv.Atoi(new.Labels[config.LabelPRIDKey])
 	if err != nil {
-		i.log.Error(err, "failed converting PR id to integer", "pr-id", new.Labels[LabelPRIDKey], "namespace", new.Namespace, "name", new.Name)
+		i.log.Error(err, "failed converting PR id to integer", "pr-id", new.Labels[config.LabelPRIDKey], "namespace", new.Namespace, "name", new.Name)
 	}
 
 	pr := provider.PullRequest{
@@ -192,7 +187,7 @@ func (i *Informer) updateHandler(oldObj, newObj interface{}) {
 	}
 
 	if _, err := i.gitProvider.AddCommentToPullRequest(ctx, pr, formatPlanOutput(planOutput)); err != nil {
-		i.log.Error(err, "failed adding comment to pull request", "pr-id", new.Labels[LabelPRIDKey], "namespace", new.Namespace, "name", new.Name)
+		i.log.Error(err, "failed adding comment to pull request", "pr-id", new.Labels[config.LabelPRIDKey], "namespace", new.Namespace, "name", new.Name)
 	}
 }
 

internal/informer/branch-planner/informer_test.go

@@ -12,6 +12,7 @@ import (
 
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 	infrav1 "github.com/weaveworks/tf-controller/api/v1alpha2"
+	"github.com/weaveworks/tf-controller/internal/config"
 	"github.com/weaveworks/tf-controller/internal/git/provider/providerfakes"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -47,7 +48,7 @@ func TestInformer(t *testing.T) {
 			Name:      "helloworld",
 			Namespace: ns.Name,
 			Labels: map[string]string{
-				LabelKey:                  LabelValue,
+				config.LabelKey:           config.LabelValue,
 				"infra.weave.works/pr-id": "1",
 			},
 		},
@@ -123,7 +124,7 @@ func createSharedInformer(g *WithT, ctx context.Context, client client.Client, d
 	g.Expect(err).NotTo(HaveOccurred())
 
 	tweakListOptionsFunc := func(options *metav1.ListOptions) {
-		options.LabelSelector = fmt.Sprintf("%s=%s", LabelKey, LabelValue)
+		options.LabelSelector = fmt.Sprintf("%s=%s", config.LabelKey, config.LabelValue)
 	}
 
 	factory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, time.Minute, corev1.NamespaceAll, tweakListOptionsFunc)

internal/server/polling/server.go

@@ -6,8 +6,8 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	bpconfig "github.com/weaveworks/tf-controller/internal/config"
 	"github.com/weaveworks/tf-controller/internal/git/provider"
-	planner "github.com/weaveworks/tf-controller/internal/informer/branch-planner"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -65,15 +65,45 @@ func (s *Server) Start(ctx context.Context) error {
 			}
 
 			for _, resource := range config.Resources {
-				if err := s.poll(ctx, resource, secret); err != nil {
-					s.log.Error(err, "failed to check pull request")
+				if resource.Name != "" {
+					if err := s.poll(ctx, resource, secret); err != nil {
+						s.log.Error(err, "failed to check pull request")
+					}
+
+					continue
+				}
+
+				s.log.Info("checking all Terrafrom objects in namespace", "namespace", resource.Namespace)
+
+				resources, err := s.listTerraformObjects(ctx, resource.Namespace, map[string]string{})
+				if err != nil {
+					s.log.Error(err, "failed to list Terraform objects in namespace", "namespace", resource.Namespace)
+
+					continue
+				}
+
+				for _, tf := range resources {
+					if tf.Labels[bpconfig.LabelKey] == bpconfig.LabelValue {
+						continue
+					}
+
+					resource := types.NamespacedName{
+						Namespace: tf.Namespace,
+						Name:      tf.Name,
+					}
+
+					if err := s.poll(ctx, resource, secret); err != nil {
+						s.log.Error(err, "failed to check pull request")
+					}
 				}
 			}
 		}
 	}
 }
 
 func (s *Server) poll(ctx context.Context, resource types.NamespacedName, secret *corev1.Secret) error {
+	s.log.Info("start polling", "namespace", resource.Namespace, "name", resource.Name)
+
 	if secret == nil {
 		return fmt.Errorf("secret is not defined")
 	}
@@ -108,8 +138,8 @@ func (s *Server) poll(ctx context.Context, resource types.NamespacedName, secret
 
 func (s *Server) reconcile(ctx context.Context, original *infrav1.Terraform, source *sourcev1.GitRepository, prs []provider.PullRequest) error {
 	// List Terraform objects, created by the branch planner.
-	tfList, err := s.listTerraformObjects(ctx, original, map[string]string{
-		planner.LabelKey: planner.LabelValue,
+	tfList, err := s.listTerraformObjects(ctx, original.Namespace, map[string]string{
+		bpconfig.LabelKey: bpconfig.LabelValue,
 	})
 	if err != nil {
 		return fmt.Errorf("failed to list Terraform objects: %w", err)
@@ -127,7 +157,7 @@ func (s *Server) reconcile(ctx context.Context, original *infrav1.Terraform, sou
 	}
 
 	for _, branchTf := range tfList {
-		prLabel := branchTf.Labels[planner.LabelPRIDKey]
+		prLabel := branchTf.Labels[bpconfig.LabelPRIDKey]
 
 		if _, ok := prMap[prLabel]; !ok {
 			if err = s.deleteTerraform(ctx, branchTf); err != nil {

internal/server/polling/terraform.go

@@ -7,7 +7,7 @@ import (
 
 	sourcev1b2 "github.com/fluxcd/source-controller/api/v1beta2"
 	infrav1 "github.com/weaveworks/tf-controller/api/v1alpha2"
-	planner "github.com/weaveworks/tf-controller/internal/informer/branch-planner"
+	bpconfig "github.com/weaveworks/tf-controller/internal/config"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sLabels "k8s.io/apimachinery/pkg/labels"
@@ -24,14 +24,14 @@ func (s *Server) getTerraformObject(ctx context.Context, ref client.ObjectKey) (
 	return obj, nil
 }
 
-func (s *Server) listTerraformObjects(ctx context.Context, tf *infrav1.Terraform, labels map[string]string) ([]*infrav1.Terraform, error) {
+func (s *Server) listTerraformObjects(ctx context.Context, namespace string, labels map[string]string) ([]*infrav1.Terraform, error) {
 	tfList := &infrav1.TerraformList{}
 
 	if err := s.clusterClient.List(ctx, tfList,
 		client.MatchingLabelsSelector{
 			Selector: k8sLabels.Set(labels).AsSelector(),
 		},
-		client.InNamespace(tf.Namespace),
+		client.InNamespace(namespace),
 	); err != nil {
 		return nil, fmt.Errorf("unable to list Terraform objects: %w", err)
 	}
@@ -148,8 +148,8 @@ func (s *Server) createLabels(labels map[string]string, branch string, prID stri
 		labels = make(map[string]string)
 	}
 
-	labels[planner.LabelKey] = planner.LabelValue
-	labels[planner.LabelPRIDKey] = prID
+	labels[bpconfig.LabelKey] = bpconfig.LabelValue
+	labels[bpconfig.LabelPRIDKey] = prID
 
 	return labels
 }