✨ Support setting manifest work config by addon template

go.mod

@@ -28,7 +28,7 @@ require (
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/kube-aggregator v0.28.1
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
-	open-cluster-management.io/addon-framework v0.8.1-0.20231009020812-e52774032b4c
+	open-cluster-management.io/addon-framework v0.8.1-0.20231102082339-51742bc299f2
 	open-cluster-management.io/api v0.12.1-0.20231114013807-95119cf22df6
 	sigs.k8s.io/controller-runtime v0.15.0
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96

go.sum

@@ -738,8 +738,8 @@ k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5Ohx
 k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI=
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-open-cluster-management.io/addon-framework v0.8.1-0.20231009020812-e52774032b4c h1:9Rvj3UTjVwJWOItlIYx6shFF72f8L3t91T9IwZ8sx6Q=
-open-cluster-management.io/addon-framework v0.8.1-0.20231009020812-e52774032b4c/go.mod h1:r4sQGR9YgLC4hXC695sfinun2WhuigWrEPk2IeIl800=
+open-cluster-management.io/addon-framework v0.8.1-0.20231102082339-51742bc299f2 h1:38vY9paEGugvXfYGJ0oFabL4/8Jxrg+GnxxjUO2DMio=
+open-cluster-management.io/addon-framework v0.8.1-0.20231102082339-51742bc299f2/go.mod h1:aj97pgpGJ0/LpQzBVtU2oDFqqIiZLOPnsjLKG/sVkFw=
 open-cluster-management.io/api v0.12.1-0.20231114013807-95119cf22df6 h1:QwQ3dTtbCGuZ7iGqINuawGh4c3NmGe2qhcdHvEXRsOs=
 open-cluster-management.io/api v0.12.1-0.20231114013807-95119cf22df6/go.mod h1:/I/nFccB0tmF+dZg7pHuzY3SaXOX86MI4vcFtidJ0OM=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

pkg/addon/templateagent/registration.go

@@ -52,16 +52,34 @@ func AddonManagerNamespace() string {
 
 func (a *CRDTemplateAgentAddon) GetDesiredAddOnTemplate(addon *addonapiv1alpha1.ManagedClusterAddOn,
 	clusterName, addonName string) (*addonapiv1alpha1.AddOnTemplate, error) {
+	if addon != nil {
+		return a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences)
+	}
 
-	if addon == nil {
-		var err error
-		addon, err = a.addonLister.ManagedClusterAddOns(clusterName).Get(addonName)
+	if len(clusterName) != 0 {
+		addon, err := a.addonLister.ManagedClusterAddOns(clusterName).Get(addonName)
 		if err != nil {
 			return nil, err
 		}
+
+		return a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences)
 	}
 
-	return a.GetDesiredAddOnTemplateByAddon(addon)
+	// clusterName and addon are both empty, backoff to get the template from the clusterManagementAddOn
+	cma, err := a.cmaLister.Get(addonName)
+	if err != nil {
+		return nil, err
+	}
+
+	// convert the DefaultConfigReference to ConfigReference
+	var configReferences []addonapiv1alpha1.ConfigReference
+	for _, configReference := range cma.Status.DefaultConfigReferences {
+		configReferences = append(configReferences, addonapiv1alpha1.ConfigReference{
+			ConfigGroupResource: configReference.ConfigGroupResource,
+			DesiredConfig:       configReference.DesiredConfig,
+		})
+	}
+	return a.getDesiredAddOnTemplateInner(cma.Name, configReferences)
 }
 
 func (a *CRDTemplateAgentAddon) TemplateCSRConfigurationsFunc() func(cluster *clusterv1.ManagedCluster) []addonapiv1alpha1.RegistrationConfig {

pkg/addon/templateagent/registration_test.go

@@ -4,10 +4,12 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	certificatesv1 "k8s.io/api/certificates/v1"
 	certificates "k8s.io/api/certificates/v1beta1"
 	rbacv1 "k8s.io/api/rbac/v1"
@@ -665,3 +667,45 @@ func TestTemplatePermissionConfigFunc(t *testing.T) {
 		}
 	}
 }
+
+func TestAddonManagerNamespace(t *testing.T) {
+	cases := []struct {
+		name         string
+		podNamespace string
+		envNs        string
+		expected     string
+	}{
+		{
+			name:         "pod namespace is not empty",
+			podNamespace: "test",
+			envNs:        "",
+			expected:     "test",
+		},
+		{
+			name:         "pod namespace is empty, env is not empty",
+			podNamespace: "",
+			envNs:        "test-env",
+			expected:     "test-env",
+		},
+		{
+			name:         "default namespace",
+			podNamespace: "",
+			envNs:        "",
+			expected:     "open-cluster-management-hub",
+		},
+	}
+
+	for _, c := range cases {
+		if c.podNamespace != "" {
+			podNamespace = c.podNamespace
+		}
+		if c.envNs != "" {
+			os.Setenv("POD_NAMESPACE", c.envNs)
+		}
+		ns := AddonManagerNamespace()
+		assert.Equal(t, c.expected, ns)
+		// reset podNamespace and env
+		podNamespace = ""
+		os.Setenv("POD_NAMESPACE", "")
+	}
+}

pkg/addon/templateagent/template_agent.go

@@ -8,6 +8,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
 	rbacv1lister "k8s.io/client-go/listers/rbac/v1"
 	"k8s.io/klog/v2"
@@ -54,6 +55,7 @@ type CRDTemplateAgentAddon struct {
 	addonClient         addonv1alpha1client.Interface
 	addonLister         addonlisterv1alpha1.ManagedClusterAddOnLister
 	addonTemplateLister addonlisterv1alpha1.AddOnTemplateLister
+	cmaLister           addonlisterv1alpha1.ClusterManagementAddOnLister
 	rolebindingLister   rbacv1lister.RoleBindingLister
 	addonName           string
 	agentName           string
@@ -79,6 +81,7 @@ func NewCRDTemplateAgentAddon(
 		addonClient:         addonClient,
 		addonLister:         addonInformers.Addon().V1alpha1().ManagedClusterAddOns().Lister(),
 		addonTemplateLister: addonInformers.Addon().V1alpha1().AddOnTemplates().Lister(),
+		cmaLister:           addonInformers.Addon().V1alpha1().ClusterManagementAddOns().Lister(),
 		rolebindingLister:   rolebindingLister,
 		addonName:           addonName,
 		agentName:           agentName,
@@ -91,7 +94,7 @@ func (a *CRDTemplateAgentAddon) Manifests(
 	cluster *clusterv1.ManagedCluster,
 	addon *addonapiv1alpha1.ManagedClusterAddOn) ([]runtime.Object, error) {
 
-	template, err := a.GetDesiredAddOnTemplateByAddon(addon)
+	template, err := a.getDesiredAddOnTemplateInner(addon.Name, addon.Status.ConfigReferences)
 	if err != nil {
 		return nil, err
 	}
@@ -107,7 +110,7 @@ func (a *CRDTemplateAgentAddon) GetAgentAddonOptions() agent.AgentAddonOptions {
 	for gvr := range utils.BuiltInAddOnConfigGVRs {
 		supportedConfigGVRs = append(supportedConfigGVRs, gvr)
 	}
-	return agent.AgentAddonOptions{
+	agentAddonOptions := agent.AgentAddonOptions{
 		AddonName:       a.addonName,
 		InstallStrategy: nil,
 		HealthProber: &agent.HealthProber{
@@ -124,6 +127,19 @@ func (a *CRDTemplateAgentAddon) GetAgentAddonOptions() agent.AgentAddonOptions {
 		},
 		AgentDeployTriggerClusterFilter: utils.ClusterImageRegistriesAnnotationChanged,
 	}
+
+	template, err := a.GetDesiredAddOnTemplate(nil, "", a.addonName)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("failed to get addon %s template: %v", a.addonName, err))
+		return agentAddonOptions
+	}
+	if template == nil {
+		utilruntime.HandleError(fmt.Errorf("addon %s template is nil", a.addonName))
+		return agentAddonOptions
+	}
+	agentAddonOptions.ManifestConfigs = template.Spec.AgentSpec.ManifestConfigs
+
+	return agentAddonOptions
 }
 
 func (a *CRDTemplateAgentAddon) renderObjects(
@@ -190,19 +206,20 @@ func (a *CRDTemplateAgentAddon) decorateObjects(
 	return objects, nil
 }
 
-// GetDesiredAddOnTemplateByAddon returns the desired template of the addon
-func (a *CRDTemplateAgentAddon) GetDesiredAddOnTemplateByAddon(
-	addon *addonapiv1alpha1.ManagedClusterAddOn) (*addonapiv1alpha1.AddOnTemplate, error) {
-	ok, templateRef := AddonTemplateConfigRef(addon.Status.ConfigReferences)
+// getDesiredAddOnTemplateInner returns the desired template of the addon
+func (a *CRDTemplateAgentAddon) getDesiredAddOnTemplateInner(
+	addonName string, configReferences []addonapiv1alpha1.ConfigReference,
+) (*addonapiv1alpha1.AddOnTemplate, error) {
+	ok, templateRef := AddonTemplateConfigRef(configReferences)
 	if !ok {
-		a.logger.V(4).Info("Addon template config in status is empty", "addonName", addon.Name)
+		a.logger.V(4).Info("Addon template config in status is empty", "addonName", addonName)
 		return nil, nil
 	}
 
 	desiredTemplate := templateRef.DesiredConfig
 	if desiredTemplate == nil || desiredTemplate.SpecHash == "" {
-		a.logger.Info("Addon template spec hash is empty", "addonName", addon.Name)
-		return nil, fmt.Errorf("addon %s template desired spec hash is empty", addon.Name)
+		a.logger.Info("Addon template spec hash is empty", "addonName", addonName)
+		return nil, fmt.Errorf("addon %s template desired spec hash is empty", addonName)
 	}
 
 	template, err := a.addonTemplateLister.Get(desiredTemplate.Name)

pkg/addon/templateagent/template_agent_test.go

@@ -19,12 +19,14 @@ import (
 	"k8s.io/klog/v2/ktesting"
 
 	"open-cluster-management.io/addon-framework/pkg/addonfactory"
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/addontesting"
 	"open-cluster-management.io/addon-framework/pkg/utils"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	fakeaddon "open-cluster-management.io/api/client/addon/clientset/versioned/fake"
 	addoninformers "open-cluster-management.io/api/client/addon/informers/externalversions"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 	clusterv1apha1 "open-cluster-management.io/api/cluster/v1alpha1"
+	workapiv1 "open-cluster-management.io/api/work/v1"
 )
 
 func TestAddonTemplateAgentManifests(t *testing.T) {
@@ -385,6 +387,170 @@ func TestAgentInstallNamespace(t *testing.T) {
 	}
 }
 
+func TestAgentManifestConfigs(t *testing.T) {
+	_, ctx := ktesting.NewTestContext(t)
+	addonName := "hello"
+
+	s := runtime.NewScheme()
+	_ = scheme.AddToScheme(s)
+	_ = clusterv1apha1.Install(s)
+	_ = addonapiv1alpha1.Install(s)
+
+	cases := []struct {
+		name                   string
+		addonTemplate          *addonapiv1alpha1.AddOnTemplate
+		clusterManagementAddOn *addonapiv1alpha1.ClusterManagementAddOn
+		expected               []workapiv1.ManifestConfigOption
+	}{
+		{
+			name:                   "no addontemplate in cma status",
+			clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", "").Build(),
+			expected:               nil,
+		},
+		{
+			name: "no addontemplate",
+			clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", "").
+				WithDefaultConfigReferences(
+					addonapiv1alpha1.DefaultConfigReference{
+						ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{
+							Group:    utils.AddOnTemplateGVR.Group,
+							Resource: utils.AddOnTemplateGVR.Resource,
+						},
+						DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
+							ConfigReferent: addonapiv1alpha1.ConfigReferent{Name: "hello-template"},
+							SpecHash:       "hash",
+						},
+					},
+				).Build(),
+			expected: nil,
+		},
+		{
+			name: "addontemplate does not have manifestconfigs",
+			clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", "").
+				WithDefaultConfigReferences(
+					addonapiv1alpha1.DefaultConfigReference{
+						ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{
+							Group:    utils.AddOnTemplateGVR.Group,
+							Resource: utils.AddOnTemplateGVR.Resource,
+						},
+						DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
+							ConfigReferent: addonapiv1alpha1.ConfigReferent{Name: "hello-template"},
+							SpecHash:       "hash",
+						},
+					},
+				).Build(),
+			addonTemplate: &addonapiv1alpha1.AddOnTemplate{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "hello-template",
+				},
+			},
+			expected: nil,
+		},
+		{
+			name: "addontemplate has manifestconfigs",
+			clusterManagementAddOn: addontesting.NewClusterManagementAddon("hello", "", "").
+				WithDefaultConfigReferences(
+					addonapiv1alpha1.DefaultConfigReference{
+						ConfigGroupResource: addonapiv1alpha1.ConfigGroupResource{
+							Group:    utils.AddOnTemplateGVR.Group,
+							Resource: utils.AddOnTemplateGVR.Resource,
+						},
+						DesiredConfig: &addonapiv1alpha1.ConfigSpecHash{
+							ConfigReferent: addonapiv1alpha1.ConfigReferent{Name: "hello-template"},
+							SpecHash:       "hash",
+						},
+					},
+				).Build(),
+			addonTemplate: &addonapiv1alpha1.AddOnTemplate{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "hello-template",
+				},
+				Spec: addonapiv1alpha1.AddOnTemplateSpec{
+					AddonName: "hello",
+					AgentSpec: workapiv1.ManifestWorkSpec{
+						ManifestConfigs: []workapiv1.ManifestConfigOption{
+							{
+								ResourceIdentifier: workapiv1.ResourceIdentifier{
+									Group:     "apps",
+									Resource:  "deployment",
+									Name:      "hello",
+									Namespace: "default",
+								},
+								FeedbackRules: []workapiv1.FeedbackRule{
+									{
+										Type: workapiv1.WellKnownStatusType,
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: []workapiv1.ManifestConfigOption{
+				{
+					ResourceIdentifier: workapiv1.ResourceIdentifier{
+						Group:     "apps",
+						Resource:  "deployment",
+						Name:      "hello",
+						Namespace: "default",
+					},
+					FeedbackRules: []workapiv1.FeedbackRule{
+						{
+							Type: workapiv1.WellKnownStatusType,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		var objs []runtime.Object
+		if tc.clusterManagementAddOn != nil {
+			objs = append(objs, tc.clusterManagementAddOn)
+		}
+		if tc.addonTemplate != nil {
+			objs = append(objs, tc.addonTemplate)
+		}
+		hubKubeClient := fakekube.NewSimpleClientset()
+		addonClient := fakeaddon.NewSimpleClientset(objs...)
+
+		addonInformerFactory := addoninformers.NewSharedInformerFactory(addonClient, 30*time.Minute)
+		if tc.clusterManagementAddOn != nil {
+			cmaStore := addonInformerFactory.Addon().V1alpha1().ClusterManagementAddOns().Informer().GetStore()
+			if err := cmaStore.Add(tc.clusterManagementAddOn); err != nil {
+				t.Fatal(err)
+			}
+		}
+		if tc.addonTemplate != nil {
+			atStore := addonInformerFactory.Addon().V1alpha1().AddOnTemplates().Informer().GetStore()
+			if err := atStore.Add(tc.addonTemplate); err != nil {
+				t.Fatal(err)
+			}
+		}
+		kubeInformers := kubeinformers.NewSharedInformerFactoryWithOptions(hubKubeClient, 10*time.Minute)
+
+		agentAddon := NewCRDTemplateAgentAddon(
+			ctx,
+			addonName,
+			"test-agent",
+			hubKubeClient,
+			addonClient,
+			addonInformerFactory,
+			kubeInformers.Rbac().V1().RoleBindings().Lister(),
+			addonfactory.GetAddOnDeploymentConfigValues(
+				addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
+				addonfactory.ToAddOnCustomizedVariableValues,
+				ToAddOnNodePlacementPrivateValues,
+				ToAddOnRegistriesPrivateValues,
+			),
+		)
+
+		agentManifestConfigs := agentAddon.GetAgentAddonOptions().ManifestConfigs
+		assert.Equal(t, tc.expected, agentManifestConfigs)
+	}
+}
+
 type testManagedClusterAddOnBuilder struct {
 	managedClusterAddOn   *addonapiv1alpha1.ManagedClusterAddOn
 	addonTemplate         *addonapiv1alpha1.AddOnTemplate