Introduce InputProvider interface

Signed-off-by: Stefan Prodan <[email protected]>

api/v1/common_types.go

@@ -3,7 +3,11 @@
 
 package v1
 
-import "fmt"
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
 
 const (
 	EnabledValue  = "enabled"
@@ -23,6 +27,17 @@ var (
 	RevisionAnnotation               = fmt.Sprintf("%s/revision", GroupVersion.Group)
 )
 
+// InputProvider is the interface that the ResourceSet
+// input providers must implement.
+//
+// +k8s:deepcopy-gen=false
+type InputProvider interface {
+	GetInputs() ([]map[string]any, error)
+	GetNamespace() string
+	GetName() string
+	GroupVersionKind() schema.GroupVersionKind
+}
+
 // CommonMetadata defines the common labels and annotations.
 type CommonMetadata struct {
 	// Annotations to be added to the object's metadata.

api/v1/resourceset_types.go

@@ -4,11 +4,13 @@
 package v1
 
 import (
+	"fmt"
 	"strings"
 	"time"
 
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/json"
 
 	"github.com/fluxcd/pkg/apis/meta"
 )
@@ -29,11 +31,11 @@ type ResourceSetSpec struct {
 	// +optional
 	Inputs []ResourceSetInput `json:"inputs,omitempty"`
 
-	// InputsFrom contains the list of references to inputs providers.
-	// When set, the inputs are fetched from the providers and merged with the
-	// inputs defined in the ResourceSet.
+	// InputsFrom contains the list of references to input providers.
+	// When set, the inputs are fetched from the providers and concatenated
+	// with the in-line inputs defined in the ResourceSet.
 	// +optional
-	InputsFrom []InputsProvider `json:"inputsFrom,omitempty"`
+	InputsFrom []InputProviderReference `json:"inputsFrom,omitempty"`
 
 	// Resources contains the list of Kubernetes resources to reconcile.
 	// +optional
@@ -63,18 +65,18 @@ type ResourceSetSpec struct {
 	Wait bool `json:"wait,omitempty"`
 }
 
-type InputsProvider struct {
-	// APIVersion of the inputs provider resource.
+type InputProviderReference struct {
+	// APIVersion of the input provider resource.
 	// When not set, the APIVersion of the ResourceSet is used.
 	// +optional
 	APIVersion string `json:"apiVersion,omitempty"`
 
-	// Kind of the inputs provider resource.
+	// Kind of the input provider resource.
 	// +kubebuilder:validation:Enum=ResourceSetInputProvider
 	// +required
 	Kind string `json:"kind"`
 
-	// Name of the inputs provider resource.
+	// Name of the input provider resource.
 	// +required
 	Name string `json:"name"`
 }
@@ -180,6 +182,23 @@ func (in *ResourceSet) GetTimeout() time.Duration {
 	return timeout
 }
 
+// GetInputs returns the ResourceSet in-line inputs as a list of maps.
+func (in *ResourceSet) GetInputs() ([]map[string]any, error) {
+	inputs := make([]map[string]any, 0, len(in.Spec.Inputs))
+	for i, ji := range in.Spec.Inputs {
+		inp := make(map[string]any, len(ji))
+		for k, v := range ji {
+			var data any
+			if err := json.Unmarshal(v.Raw, &data); err != nil {
+				return nil, fmt.Errorf("failed to unmarshal inputs[%d]: %w", i, err)
+			}
+			inp[k] = data
+		}
+		inputs = append(inputs, inp)
+	}
+	return inputs, nil
+}
+
 // +kubebuilder:storageversion
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status

api/v1/resourcesetinputprovider_types.go

@@ -16,8 +16,8 @@ import (
 
 const (
 	ResourceSetInputProviderKind    = "ResourceSetInputProvider"
-	GitHubPullRequestInputProvider  = "GitHubPullRequest"
-	GitLabMergeRequestInputProvider = "GitLabMergeRequest"
+	InputProviderGitHubPullRequest  = "GitHubPullRequest"
+	InputProviderGitLabMergeRequest = "GitLabMergeRequest"
 )
 
 // ResourceSetInputProviderSpec defines the desired state of ResourceSetInputProvider
@@ -165,6 +165,23 @@ func (in *ResourceSetInputProvider) GetDefaultInputs() (map[string]any, error) {
 	return defaults, nil
 }
 
+// GetInputs returns the exported inputs from ResourceSetInputProvider status.
+func (in *ResourceSetInputProvider) GetInputs() ([]map[string]any, error) {
+	inputs := make([]map[string]any, 0, len(in.Status.ExportedInputs))
+	for i, ji := range in.Status.ExportedInputs {
+		inp := make(map[string]any, len(ji))
+		for k, v := range ji {
+			var data any
+			if err := json.Unmarshal(v.Raw, &data); err != nil {
+				return nil, fmt.Errorf("failed to unmarshal inputs[%d]: %w", i, err)
+			}
+			inp[k] = data
+		}
+		inputs = append(inputs, inp)
+	}
+	return inputs, nil
+}
+
 // +kubebuilder:storageversion
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status

config/crd/bases/fluxcd.controlplane.io_resourcesets.yaml

@@ -115,23 +115,23 @@ spec:
                 type: array
               inputsFrom:
                 description: |-
-                  InputsFrom contains the list of references to inputs providers.
-                  When set, the inputs are fetched from the providers and merged with the
-                  inputs defined in the ResourceSet.
+                  InputsFrom contains the list of references to input providers.
+                  When set, the inputs are fetched from the providers and concatenated
+                  with the in-line inputs defined in the ResourceSet.
                 items:
                   properties:
                     apiVersion:
                       description: |-
-                        APIVersion of the inputs provider resource.
+                        APIVersion of the input provider resource.
                         When not set, the APIVersion of the ResourceSet is used.
                       type: string
                     kind:
-                      description: Kind of the inputs provider resource.
+                      description: Kind of the input provider resource.
                       enum:
                       - ResourceSetInputProvider
                       type: string
                     name:
-                      description: Name of the inputs provider resource.
+                      description: Name of the input provider resource.
                       type: string
                   required:
                   - kind

internal/builder/resourceset.go

@@ -5,7 +5,6 @@ package builder
 
 import (
 	"bytes"
-	"encoding/json"
 	"fmt"
 	"strings"
 	"text/template"
@@ -22,20 +21,7 @@ import (
 
 // BuildResourceSet builds a list of Kubernetes resources
 // from a list of JSON templates using the provided inputs.
-func BuildResourceSet(yamlTemplate string, templates []*apix.JSON, jsonInputs []fluxcdv1.ResourceSetInput) ([]*unstructured.Unstructured, error) {
-	inputs := make([]map[string]any, 0, len(jsonInputs))
-	for i, ji := range jsonInputs {
-		inp := make(map[string]any, len(ji))
-		for k, v := range ji {
-			var data any
-			if err := json.Unmarshal(v.Raw, &data); err != nil {
-				return nil, fmt.Errorf("failed to unmarshal inputs[%d]: %w", i, err)
-			}
-			inp[k] = data
-		}
-		inputs = append(inputs, inp)
-	}
-
+func BuildResourceSet(yamlTemplate string, templates []*apix.JSON, inputs []map[string]any) ([]*unstructured.Unstructured, error) {
 	var objects []*unstructured.Unstructured
 
 	// build resources from JSON templates

internal/builder/resourceset_test.go

@@ -66,7 +66,10 @@ func TestBuildResourceSet(t *testing.T) {
 			err = yaml.Unmarshal(data, &rg)
 			g.Expect(err).ToNot(HaveOccurred())
 
-			objects, err := BuildResourceSet(rg.Spec.ResourcesTemplate, rg.Spec.Resources, rg.Spec.Inputs)
+			inputs, err := rg.GetInputs()
+			g.Expect(err).ToNot(HaveOccurred())
+
+			objects, err := BuildResourceSet(rg.Spec.ResourcesTemplate, rg.Spec.Resources, inputs)
 			g.Expect(err).ToNot(HaveOccurred())
 
 			manifests, err := ssautil.ObjectsToYAML(objects)
@@ -97,7 +100,10 @@ func TestBuildResourceSet_Empty(t *testing.T) {
 	err = yaml.Unmarshal(data, &rg)
 	g.Expect(err).ToNot(HaveOccurred())
 
-	objects, err := BuildResourceSet(rg.Spec.ResourcesTemplate, rg.Spec.Resources, rg.Spec.Inputs)
+	inputs, err := rg.GetInputs()
+	g.Expect(err).ToNot(HaveOccurred())
+
+	objects, err := BuildResourceSet(rg.Spec.ResourcesTemplate, rg.Spec.Resources, inputs)
 	g.Expect(err).ToNot(HaveOccurred())
 	g.Expect(objects).To(BeEmpty())
 }
@@ -133,7 +139,10 @@ func TestBuildResourceSet_Error(t *testing.T) {
 			err = yaml.Unmarshal(data, &rg)
 			g.Expect(err).ToNot(HaveOccurred())
 
-			_, err = BuildResourceSet(rg.Spec.ResourcesTemplate, rg.Spec.Resources, rg.Spec.Inputs)
+			inputs, err := rg.GetInputs()
+			g.Expect(err).ToNot(HaveOccurred())
+
+			_, err = BuildResourceSet(rg.Spec.ResourcesTemplate, rg.Spec.Resources, inputs)
 			g.Expect(err).To(HaveOccurred())
 			g.Expect(err.Error()).To(ContainSubstring(tt.matchErr))
 		})

internal/controller/resourceset_controller.go

@@ -281,23 +281,38 @@ func (r *ResourceSetReconciler) checkDependencies(ctx context.Context,
 }
 
 func (r *ResourceSetReconciler) getInputs(ctx context.Context,
-	obj *fluxcdv1.ResourceSet) ([]fluxcdv1.ResourceSetInput, error) {
-	inputs := make([]fluxcdv1.ResourceSetInput, 0)
+	obj *fluxcdv1.ResourceSet) ([]map[string]any, error) {
+	providers := make([]fluxcdv1.InputProvider, 0)
+	providers = append(providers, obj)
 	for _, inputSource := range obj.Spec.InputsFrom {
-		var provider fluxcdv1.ResourceSetInputProvider
+		var provider fluxcdv1.InputProvider
 		key := client.ObjectKey{
 			Namespace: obj.GetNamespace(),
 			Name:      inputSource.Name,
 		}
 
-		if err := r.Get(ctx, key, &provider); err != nil {
-			return nil, fmt.Errorf("failed to get inputs from %s/%s: %w", key.Namespace, key.Name, err)
+		switch inputSource.Kind {
+		case fluxcdv1.ResourceSetInputProviderKind:
+			var rsip fluxcdv1.ResourceSetInputProvider
+			if err := r.Get(ctx, key, &rsip); err != nil {
+				return nil, fmt.Errorf("failed to get provider %s/%s: %w", key.Namespace, key.Name, err)
+			}
+			provider = &rsip
+		default:
+			return nil, fmt.Errorf("unsupported provider kind %s", inputSource.Kind)
 		}
 
-		inputs = append(inputs, provider.Status.ExportedInputs...)
+		providers = append(providers, provider)
 	}
-	if len(obj.Spec.Inputs) > 0 {
-		inputs = append(inputs, obj.Spec.Inputs...)
+
+	inputs := make([]map[string]any, 0)
+	for _, provider := range providers {
+		exportedInputs, err := provider.GetInputs()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get inputs from %s/%s: %w",
+				provider.GroupVersionKind().Kind, provider.GetName(), err)
+		}
+		inputs = append(inputs, exportedInputs...)
 	}
 
 	return inputs, nil

internal/controller/resourcesetinputprovider_controller.go

@@ -169,7 +169,7 @@ func (r *ResourceSetInputProviderReconciler) reconcile(ctx context.Context,
 func (r *ResourceSetInputProviderReconciler) newGitProvider(ctx context.Context,
 	obj *fluxcdv1.ResourceSetInputProvider,
 	certPool *x509.CertPool,
-	username, password string) (gitprovider.Provider, error) {
+	username, password string) (gitprovider.Interface, error) {
 	switch {
 	case strings.HasPrefix(obj.Spec.Type, "GitHub"):
 		return gitprovider.NewGitHubProvider(ctx, gitprovider.Options{
@@ -210,14 +210,14 @@ func (r *ResourceSetInputProviderReconciler) makeGitOptions(obj *fluxcdv1.Resour
 			if err != nil {
 				return gitprovider.Options{}, fmt.Errorf("invalid includeBranch regex: %w", err)
 			}
-			opts.Filters.IncludeBranchRx = inRx
+			opts.Filters.IncludeBranchRe = inRx
 		}
 		if obj.Spec.Filter.ExcludeBranch != "" {
 			exRx, err := regexp.Compile(obj.Spec.Filter.ExcludeBranch)
 			if err != nil {
 				return gitprovider.Options{}, fmt.Errorf("invalid excludeBranch regex: %w", err)
 			}
-			opts.Filters.ExcludeBranchRx = exRx
+			opts.Filters.ExcludeBranchRe = exRx
 		}
 	}
 
@@ -226,7 +226,7 @@ func (r *ResourceSetInputProviderReconciler) makeGitOptions(obj *fluxcdv1.Resour
 
 func (r *ResourceSetInputProviderReconciler) callProvider(ctx context.Context,
 	obj *fluxcdv1.ResourceSetInputProvider,
-	provider gitprovider.Provider) ([]fluxcdv1.ResourceSetInput, error) {
+	provider gitprovider.Interface) ([]fluxcdv1.ResourceSetInput, error) {
 	var inputs []fluxcdv1.ResourceSetInput
 
 	opts, err := r.makeGitOptions(obj)

internal/gitprovider/github_test.go

@@ -25,8 +25,8 @@ func TestGitHubProvider_ListBranches(t *testing.T) {
 				Token: os.Getenv("GITHUB_TOKEN"),
 				URL:   "https://github.com/fluxcd-testing/pr-testing",
 				Filters: Filters{
-					IncludeBranchRx: regexp.MustCompile(`^stefanprodan-patch-.*`),
-					ExcludeBranchRx: regexp.MustCompile(`^stefanprodan-patch-4`),
+					IncludeBranchRe: regexp.MustCompile(`^stefanprodan-patch-.*`),
+					ExcludeBranchRe: regexp.MustCompile(`^stefanprodan-patch-4`),
 				},
 			},
 			want: []Result{
@@ -59,7 +59,7 @@ func TestGitHubProvider_ListBranches(t *testing.T) {
 				Token: os.Getenv("GITHUB_TOKEN"),
 				URL:   "https://github.com/fluxcd-testing/pr-testing",
 				Filters: Filters{
-					IncludeBranchRx: regexp.MustCompile(`^stefanprodan-patch-.*`),
+					IncludeBranchRe: regexp.MustCompile(`^stefanprodan-patch-.*`),
 					Limit:           1,
 				},
 			},
@@ -179,7 +179,7 @@ func TestGitHubProvider_ListRequests(t *testing.T) {
 				Token: os.Getenv("GITHUB_TOKEN"),
 				URL:   "https://github.com/fluxcd-testing/pr-testing",
 				Filters: Filters{
-					IncludeBranchRx: regexp.MustCompile(`^feat/.*`),
+					IncludeBranchRe: regexp.MustCompile(`^feat/.*`),
 				},
 			},
 			want: []Result{