Merge pull request #501 from gianlucam76/main

Merge dev to main

.github/workflows/main.yaml

@@ -98,3 +98,31 @@ jobs:
       run: make create-cluster fv-sharding
       env:
         FV: true
+  FV_AGENTLESS:
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+    - name: Set up Go
+      uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # tag=v5.0.0
+      with:
+        go-version: 1.21.5
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
+
+        # all of these default to true, but feel free to set to
+        # "false" if necessary for your workflow
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        docker-images: true
+        swap-storage: true
+    - name: fv-agentless
+      run: make create-cluster fv-agentless
+      env:
+        FV: true

Dockerfile

@@ -1,6 +1,9 @@
 # Build the manager binary
 FROM golang:1.21 as builder
 
+ARG BUILDOS
+ARG TARGETARCH
+
 WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod
@@ -17,7 +20,7 @@ COPY pkg/ pkg/
 COPY internal/ internal/
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager cmd/main.go
+RUN CGO_ENABLED=0 GOOS=$BUILDOS GOARCH=$TARGETARCH go build -a -o manager cmd/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

Makefile

@@ -62,7 +62,6 @@ CONTROLLER_GEN := $(TOOLS_BIN_DIR)/controller-gen
 ENVSUBST := $(TOOLS_BIN_DIR)/envsubst
 GOIMPORTS := $(TOOLS_BIN_DIR)/goimports
 GOLANGCI_LINT := $(TOOLS_BIN_DIR)/golangci-lint
-KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
 GINKGO := $(TOOLS_BIN_DIR)/ginkgo
 SETUP_ENVTEST := $(TOOLS_BIN_DIR)/setup_envs
 CLUSTERCTL := $(TOOLS_BIN_DIR)/clusterctl
@@ -72,6 +71,25 @@ KUBECTL := $(TOOLS_BIN_DIR)/kubectl
 GOLANGCI_LINT_VERSION := "v1.55.2"
 CLUSTERCTL_VERSION := "v1.6.3"
 
+KUSTOMIZE_VER := v4.5.2
+KUSTOMIZE_BIN := kustomize
+KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER))
+KUSTOMIZE_PKG := sigs.k8s.io/kustomize/kustomize/v4
+$(KUSTOMIZE): # Build kustomize from tools folder.
+	CGO_ENABLED=0 GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(KUSTOMIZE_PKG) $(KUSTOMIZE_BIN) $(KUSTOMIZE_VER)
+
+SETUP_ENVTEST_VER := v0.0.0-20240215143116-d0396a3d6f9f
+SETUP_ENVTEST_BIN := setup-envtest
+SETUP_ENVTEST := $(abspath $(TOOLS_BIN_DIR)/$(SETUP_ENVTEST_BIN)-$(SETUP_ENVTEST_VER))
+SETUP_ENVTEST_PKG := sigs.k8s.io/controller-runtime/tools/setup-envtest
+setup-envtest: $(SETUP_ENVTEST) ## Set up envtest (download kubebuilder assets)
+	@echo KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS)
+
+$(SETUP_ENVTEST_BIN): $(SETUP_ENVTEST) ## Build a local copy of setup-envtest.
+
+$(SETUP_ENVTEST): # Build setup-envtest from tools folder.
+	GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(SETUP_ENVTEST_PKG) $(SETUP_ENVTEST_BIN) $(SETUP_ENVTEST_VER)
+
 $(CONTROLLER_GEN): $(TOOLS_DIR)/go.mod # Build controller-gen from tools folder.
 	cd $(TOOLS_DIR); $(GOBUILD) -tags=tools -o $(subst $(TOOLS_DIR)/hack/tools/,,$@) sigs.k8s.io/controller-tools/cmd/controller-gen
 
@@ -81,9 +99,6 @@ $(ENVSUBST): $(TOOLS_DIR)/go.mod # Build envsubst from tools folder.
 $(GOLANGCI_LINT): # Build golangci-lint from tools folder.
 	cd $(TOOLS_DIR); ./get-golangci-lint.sh $(GOLANGCI_LINT_VERSION)
 
-$(SETUP_ENVTEST): $(TOOLS_DIR)/go.mod # Build setup-envtest from tools folder.
-	cd $(TOOLS_DIR); $(GOBUILD) -tags=tools -o $(subst $(TOOLS_DIR)/hack/tools/,,$@) sigs.k8s.io/controller-runtime/tools/setup-envtest
-
 $(GOIMPORTS):
 	cd $(TOOLS_DIR); $(GOBUILD) -tags=tools -o $(subst $(TOOLS_DIR)/hack/tools/,,$@) golang.org/x/tools/cmd/goimports
 
@@ -101,13 +116,6 @@ $(KUBECTL):
 	curl -L https://storage.googleapis.com/kubernetes-release/release/$(K8S_LATEST_VER)/bin/$(OS)/$(ARCH)/kubectl -o $@
 	chmod +x $@
 
-KUSTOMIZE_VER := v4.5.2
-KUSTOMIZE_BIN := kustomize
-KUSTOMIZE := $(abspath $(TOOLS_BIN_DIR)/$(KUSTOMIZE_BIN)-$(KUSTOMIZE_VER))
-KUSTOMIZE_PKG := sigs.k8s.io/kustomize/kustomize/v4
-$(KUSTOMIZE): # Build kustomize from tools folder.
-	CGO_ENABLED=0 GOBIN=$(TOOLS_BIN_DIR) $(GO_INSTALL) $(KUSTOMIZE_PKG) $(KUSTOMIZE_BIN) $(KUSTOMIZE_VER)
-
 .PHONY: tools
 tools: $(CONTROLLER_GEN) $(ENVSUBST) $(KUSTOMIZE) $(SETUP_ENVTEST) $(GOLANGCI_LINT) $(GOIMPORTS) $(GINKGO) $(CLUSTERCTL) $(KIND) $(KUBECTL) ## build all tools
 
@@ -121,9 +129,10 @@ clean: ## Remove all built tools
 .PHONY: manifests
 manifests: $(CONTROLLER_GEN) $(KUSTOMIZE) $(ENVSUBST) fmt generate ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	$(CONTROLLER_GEN) rbac:roleName=controller-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-	MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image
+	MANIFEST_IMG=$(CONTROLLER_IMG) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image
 	$(KUSTOMIZE) build config/default | $(ENVSUBST) > manifest/manifest.yaml
-	./scripts/extract_deployment.sh manifest/manifest.yaml manifest/deployment-shard.yaml
+	./scripts/extract_deployment-shard.sh manifest/manifest.yaml manifest/deployment-shard.yaml
+	./scripts/extract_deployment-agentless.sh manifest/manifest.yaml manifest/deployment-agentless.yaml
 
 .PHONY: generate
 generate: $(CONTROLLER_GEN) ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
@@ -206,6 +215,17 @@ fv-sharding: $(KUBECTL) $(GINKGO) ## Run Sveltos Controller tests using existing
 	rm -f test/addon-controller-deployment-shard.yaml
 	cd test/fv; $(GINKGO) -nodes $(NUM_NODES) --label-filter='FV' --v --trace --randomize-all
 
+.PHONY: fv-agentless
+fv-agentless: $(KUBECTL) $(GINKGO) ## Run Sveltos Controller tests using existing cluster
+	$(KUBECTL) apply -f https://raw.githubusercontent.com/projectsveltos/drift-detection-manager/$(TAG)/manifest/mgmt_cluster_common_manifest.yaml
+	$(KUBECTL) apply -f manifest/drift_detection_manager_rbac.yaml 
+	cp manifest/deployment-agentless.yaml test/addon-controller-deployment-agentless.yaml
+	$(KUBECTL) apply -f test/addon-controller-deployment-agentless.yaml
+	rm -f test/addon-controller-deployment-agentless.yaml
+	@echo "Waiting for projectsveltos addon-controller to be available..."
+	$(KUBECTL) wait --for=condition=Available deployment/addon-controller -n projectsveltos --timeout=$(TIMEOUT)
+	cd test/fv; $(GINKGO) -nodes $(NUM_NODES) --label-filter='FV' --v --trace --randomize-all
+
 .PHONY: create-cluster
 create-cluster: $(KIND) $(CLUSTERCTL) $(KUBECTL) $(ENVSUBST) ## Create a new kind cluster designed for development
 	$(MAKE) create-control-cluster
@@ -247,7 +267,7 @@ create-control-cluster: $(KIND) $(CLUSTERCTL) $(KUBECTL)
 	sed -e "s/K8S_VERSION/$(K8S_VERSION)/g"  test/$(KIND_CONFIG) > test/$(KIND_CONFIG).tmp
 	$(KIND) create cluster --name=$(CONTROL_CLUSTER_NAME) --config test/$(KIND_CONFIG).tmp
 	@echo "Create control cluster with docker as infrastructure provider"
-	CLUSTER_TOPOLOGY=true $(CLUSTERCTL) init --infrastructure docker
+	CLUSTER_TOPOLOGY=true $(CLUSTERCTL) init --core cluster-api --bootstrap kubeadm --control-plane kubeadm --infrastructure docker
 
 	@echo wait for capd-system pod
 	$(KUBECTL) wait --for=condition=Available deployment/capd-controller-manager -n capd-system --timeout=$(TIMEOUT)
@@ -331,17 +351,22 @@ run: manifests generate fmt vet ## Run a controller from your host.
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager.
 	go generate
-	docker build -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) .
-	MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image
+	docker build --build-arg BUILDOS=linux --build-arg TARGETARCH=amd64 -t $(CONTROLLER_IMG):$(TAG) .
+	MANIFEST_IMG=$(CONTROLLER_IMG) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image
 	$(MAKE) set-manifest-pull-policy
 
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
-	docker push $(CONTROLLER_IMG)-$(ARCH):$(TAG)
+	docker push $(CONTROLLER_IMG):$(TAG)
+
+.PHONY: docker-buildx
+docker-buildx: ## docker build for multiple arch and push to docker hub
+	docker buildx build --push --platform linux/amd64,linux/arm64 -t $(CONTROLLER_IMG):$(TAG) .
+
 
 .PHONY: load-image
 load-image: docker-build $(KIND)
-	$(KIND) load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name $(CONTROL_CLUSTER_NAME)
+	$(KIND) load docker-image $(CONTROLLER_IMG):$(TAG) --name $(CONTROL_CLUSTER_NAME)
 
 ##@ Deployment
 

api/v1alpha1/spec.go

@@ -189,6 +189,11 @@ type HelmOptions struct {
 	// Labels that would be added to release metadata.
 	// +optional
 	Labels map[string]string `json:"labels,omitempty"`
+
+	// EnableClientCache is a flag to enable Helm client cache. If it is not specified, it will be set to false.
+	// +kubebuilder:default=false
+	// +optional
+	EnableClientCache bool `json:"enableClientCache,omitempty"`
 }
 
 type HelmChart struct {
@@ -352,7 +357,7 @@ type Spec struct {
 	// - ClusterProfile can reference ClusterSet;
 	// - Profile can reference Set;
 	// +optional
-	SetRefs []corev1.ObjectReference `json:"setRefs,omitempty"`
+	SetRefs []string `json:"setRefs,omitempty"`
 
 	// SyncMode specifies how features are synced in a matching workload cluster.
 	// - OneTime means, first time a workload cluster matches the ClusterProfile,

cmd/main.go

@@ -20,7 +20,11 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"net/http"
+	"net/http/pprof"
 	"os"
+	"runtime"
+	"runtime/debug"
 	"sync"
 	"syscall"
 	"time"
@@ -56,6 +60,7 @@ import (
 	"github.com/projectsveltos/libsveltos/lib/crd"
 	"github.com/projectsveltos/libsveltos/lib/deployer"
 	"github.com/projectsveltos/libsveltos/lib/logsettings"
+	logs "github.com/projectsveltos/libsveltos/lib/logsettings"
 	libsveltosset "github.com/projectsveltos/libsveltos/lib/set"
 	//+kubebuilder:scaffold:imports
 )
@@ -84,6 +89,8 @@ const (
 	defaultReconcilers   = 10
 	defaultWorkers       = 20
 	defaulReportMode     = int(controllers.CollectFromManagementCluster)
+	mebibytes_bytes      = 1 << 20
+	gibibytes_per_bytes  = 1 << 30
 )
 
 // Add RBAC for the authorized diagnostics endpoint.
@@ -106,7 +113,6 @@ func main() {
 	reportMode = controllers.ReportMode(tmpReportMode)
 
 	ctrl.SetLogger(klog.Background())
-
 	ctrlOptions := ctrl.Options{
 		Scheme:                 scheme,
 		Metrics:                getDiagnosticsOptions(),
@@ -133,13 +139,15 @@ func main() {
 
 	// Setup the context that's going to be used in controllers and for the manager.
 	ctx := ctrl.SetupSignalHandler()
-
 	controllers.SetManagementClusterAccess(mgr.GetClient(), mgr.GetConfig())
 
-	logsettings.RegisterForLogSettings(ctx,
+	logs.RegisterForLogSettings(ctx,
 		libsveltosv1alpha1.ComponentAddonManager, ctrl.Log.WithName("log-setter"),
 		ctrl.GetConfigOrDie())
 
+	debug.SetMemoryLimit(gibibytes_per_bytes)
+	go printMemUsage(ctrl.Log.WithName("memory-usage"))
+
 	startControllersAndWatchers(ctx, mgr)
 
 	setupChecks(mgr)
@@ -168,7 +176,7 @@ func initFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&diagnosticsAddress, "diagnostics-address", ":8443",
 		"The address the diagnostics endpoint binds to. Per default metrics are served via https and with"+
 			"authentication/authorization. To serve via http and without authentication/authorization set --insecure-diagnostics."+
-			"If --insecure-diagnostics is not set the diagnostics endpoint also serves pprof endpoints and an endpoint to change the log level.")
+			"If --insecure-diagnostics is not set the diagnostics endpoint also serves pprof endpoints")
 
 	fs.BoolVar(&insecureDiagnostics, "insecure-diagnostics", false,
 		"Enable insecure diagnostics serving. For more details see the description of --diagnostics-address.")
@@ -374,11 +382,11 @@ func getProfileReconciler(mgr manager.Manager) *controllers.ProfileReconciler {
 		Scheme:               mgr.GetScheme(),
 		SetMap:               make(map[corev1.ObjectReference]*libsveltosset.Set),
 		ClusterMap:           make(map[corev1.ObjectReference]*libsveltosset.Set),
-		ProfileMap:           make(map[corev1.ObjectReference]*libsveltosset.Set),
 		Profiles:             make(map[corev1.ObjectReference]libsveltosv1alpha1.Selector),
 		ClusterLabels:        make(map[corev1.ObjectReference]map[string]string),
 		Mux:                  sync.Mutex{},
 		ConcurrentReconciles: concurrentReconciles,
+		Logger:               ctrl.Log.WithName("profilereconciler"),
 	}
 }
 
@@ -388,11 +396,11 @@ func getClusterProfileReconciler(mgr manager.Manager) *controllers.ClusterProfil
 		Scheme:               mgr.GetScheme(),
 		ClusterSetMap:        make(map[corev1.ObjectReference]*libsveltosset.Set),
 		ClusterMap:           make(map[corev1.ObjectReference]*libsveltosset.Set),
-		ClusterProfileMap:    make(map[corev1.ObjectReference]*libsveltosset.Set),
 		ClusterProfiles:      make(map[corev1.ObjectReference]libsveltosv1alpha1.Selector),
 		ClusterLabels:        make(map[corev1.ObjectReference]map[string]string),
 		Mux:                  sync.Mutex{},
 		ConcurrentReconciles: concurrentReconciles,
+		Logger:               ctrl.Log.WithName("clusterprofilereconciler"),
 	}
 }
 
@@ -410,9 +418,9 @@ func getClusterSummaryReconciler(ctx context.Context, mgr manager.Manager) *cont
 		Deployer:             d,
 		ClusterMap:           make(map[corev1.ObjectReference]*libsveltosset.Set),
 		ReferenceMap:         make(map[corev1.ObjectReference]*libsveltosset.Set),
-		ClusterSummaryMap:    make(map[types.NamespacedName]*libsveltosset.Set),
 		PolicyMux:            sync.Mutex{},
 		ConcurrentReconciles: concurrentReconciles,
+		Logger:               ctrl.Log.WithName("clustersummaryreconciler"),
 	}
 }
 
@@ -426,6 +434,7 @@ func getSetReconciler(mgr manager.Manager) *controllers.SetReconciler {
 		SetMap:               make(map[corev1.ObjectReference]*libsveltosset.Set),
 		Sets:                 make(map[corev1.ObjectReference]libsveltosv1alpha1.Selector),
 		ClusterLabels:        make(map[corev1.ObjectReference]map[string]string),
+		Logger:               ctrl.Log.WithName("setreconciler"),
 	}
 }
 
@@ -439,6 +448,7 @@ func getClusterSetReconciler(mgr manager.Manager) *controllers.ClusterSetReconci
 		ClusterSetMap:        make(map[corev1.ObjectReference]*libsveltosset.Set),
 		ClusterSets:          make(map[corev1.ObjectReference]libsveltosv1alpha1.Selector),
 		ClusterLabels:        make(map[corev1.ObjectReference]map[string]string),
+		Logger:               ctrl.Log.WithName("clustersetreconciler"),
 	}
 }
 
@@ -460,6 +470,15 @@ func getDiagnosticsOptions() metricsserver.Options {
 		BindAddress:    diagnosticsAddress,
 		SecureServing:  true,
 		FilterProvider: filters.WithAuthenticationAndAuthorization,
+		ExtraHandlers: map[string]http.Handler{
+			// Add pprof handler.
+			"/debug/pprof/":        http.HandlerFunc(pprof.Index),
+			"/debug/pprof/cmdline": http.HandlerFunc(pprof.Cmdline),
+			"/debug/pprof/profile": http.HandlerFunc(pprof.Profile),
+			"/debug/pprof/symbol":  http.HandlerFunc(pprof.Symbol),
+			"/debug/pprof/trace":   http.HandlerFunc(pprof.Trace),
+			"/debug/pprof/heap":    pprof.Handler("heap"),
+		},
 	}
 }
 
@@ -532,3 +551,23 @@ func startControllersAndWatchers(ctx context.Context, mgr manager.Manager) {
 
 	startWatchers(ctx, mgr, watchersForCAPI, watchersForFlux)
 }
+
+// printMemUsage memory stats. Call GC
+func printMemUsage(logger logr.Logger) {
+	for {
+		time.Sleep(time.Minute)
+		var m runtime.MemStats
+		runtime.ReadMemStats(&m)
+		// For info on each, see: /pkg/runtime/#MemStats
+		l := logger.WithValues("Alloc (MiB)", bToMb(m.Alloc)).
+			WithValues("TotalAlloc (MiB)", bToMb(m.TotalAlloc)).
+			WithValues("Sys (MiB)", bToMb(m.Sys)).
+			WithValues("NumGC", m.NumGC)
+		l.V(logs.LogInfo).Info("memory stats")
+		runtime.GC()
+	}
+}
+
+func bToMb(b uint64) uint64 {
+	return b / mebibytes_bytes
+}