Report manifestworks first applied timestamp (#215)

* Report work applied timestamp per generation Signed-off-by: zhujian <[email protected]> * Update vendor Signed-off-by: zhujian <[email protected]> * Start a controller to add work applied timestamp annotation Signed-off-by: zhujian <[email protected]> * Exclude vendor from sonar scanning Signed-off-by: zhujian <[email protected]> * Add tests Signed-off-by: zhujian <[email protected]> * Fix unit tests Signed-off-by: zhujian <[email protected]> * Update vendor Signed-off-by: zhujian <[email protected]> * Address code review comments Signed-off-by: zhujian <[email protected]> --------- Signed-off-by: zhujian <[email protected]>
stolostron · Jan 9, 2025 · 4192e97 · 4192e97
1 parent 9f821ed
commit 4192e97
Show file tree

Hide file tree

Showing 2,995 changed files with 239,251 additions and 107,943 deletions.
diff --git a/cmd/clusterlifecycle-state-metrics/main.go b/cmd/clusterlifecycle-state-metrics/main.go
@@ -17,20 +17,29 @@ import (
 	"os"
 	"strconv"
 	"strings"
-
-	logf "sigs.k8s.io/controller-runtime/pkg/log"
-	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes"
+	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
-
 	koptions "k8s.io/kube-state-metrics/pkg/options"
 	"k8s.io/kube-state-metrics/pkg/whiteblacklist"
+	workv1 "open-cluster-management.io/api/work/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+	logf "sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 
 	"github.com/stolostron/clusterlifecycle-state-metrics/pkg/collectors"
-	ocollectors "github.com/stolostron/clusterlifecycle-state-metrics/pkg/collectors"
+	"github.com/stolostron/clusterlifecycle-state-metrics/pkg/controllers"
 	"github.com/stolostron/clusterlifecycle-state-metrics/pkg/options"
 	"github.com/stolostron/clusterlifecycle-state-metrics/pkg/version"
 )
@@ -44,9 +53,14 @@ const (
 	hubTypeACM              = "acm"
 	hubTypeStolostronEngine = "stolostron-engine"
 	hubTypeStolostron       = "stolostron"
+
+	configMapName = "clusterlifecycle-state-metrics-config"
 )
 
-var opts *options.Options
+var (
+	opts   *options.Options
+	scheme = runtime.NewScheme()
+)
 
 // promLogger implements promhttp.Logger
 type promLogger struct{}
@@ -62,6 +76,10 @@ func init() {
 
 	opts = options.NewOptions()
 	opts.AddFlags()
+
+	// init schema
+	_ = clientgoscheme.AddToScheme(scheme)
+	_ = workv1.AddToScheme(scheme)
 }
 
 func (pl promLogger) Println(v ...interface{}) {
@@ -84,8 +102,29 @@ func main() {
 }
 
 func start(opts *options.Options) {
-	collectorBuilder := ocollectors.NewBuilder(context.TODO())
-	collectorBuilder.WithApiserver(opts.Apiserver).WithKubeConfig(opts.Kubeconfig)
+	ctx := context.TODO()
+	config, err := clientcmd.BuildConfigFromFlags(opts.Apiserver, opts.Kubeconfig)
+	if err != nil {
+		klog.Fatalf("cannot create config: %v", err)
+	}
+	kubeClient, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		klog.Fatalf("cannot create kubeClient: %v", err)
+	}
+
+	timestampMetricsEnabled, err := isTimestampMetricsEnabled(ctx, kubeClient)
+	if err != nil {
+		klog.Fatalf("cannot determine if timestamp metrics should be enabled: %v", err)
+	}
+
+	if timestampMetricsEnabled {
+		go startControllers(opts)
+	}
+
+	collectorBuilder := collectors.NewBuilder(ctx)
+	collectorBuilder.WithRestConfig(config).
+		WithKubeclient(kubeClient).
+		WithTimestampMetricsEnabled(timestampMetricsEnabled)
 	if len(opts.Collectors) == 0 {
 		klog.Info("Using default collectors")
 		collectorBuilder.WithEnabledCollectors(options.DefaultCollectors.AsSlice())
@@ -127,10 +166,10 @@ func start(opts *options.Options) {
 	collectorBuilder.WithWhiteBlackList(whiteBlackList)
 
 	ocmMetricsRegistry := prometheus.NewRegistry()
-	if err := ocmMetricsRegistry.Register(ocollectors.ResourcesPerScrapeMetric); err != nil {
+	if err := ocmMetricsRegistry.Register(collectors.ResourcesPerScrapeMetric); err != nil {
 		panic(err)
 	}
-	if err := ocmMetricsRegistry.Register(ocollectors.ScrapeErrorTotalMetric); err != nil {
+	if err := ocmMetricsRegistry.Register(collectors.ScrapeErrorTotalMetric); err != nil {
 		panic(err)
 	}
 	if err := ocmMetricsRegistry.Register(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{})); err != nil {
@@ -146,6 +185,70 @@ func start(opts *options.Options) {
 	serveMetrics(collectors, opts.Host, opts.HTTPPort, opts.HTTPSPort, opts.TLSCrtFile, opts.TLSKeyFile, opts.EnableGZIPEncoding)
 }
 
+func startControllers(opts *options.Options) {
+	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
+		Scheme: scheme,
+		Metrics: metricsserver.Options{
+			BindAddress: opts.ControllerMetricsAddress,
+		},
+		// WebhookServer:          webhookServer,
+		// HealthProbeBindAddress: probeAddr,
+		LeaderElection:   opts.EnableLeaderElection,
+		LeaderElectionID: "clusterlifecycle-state-metrics-controller",
+		// LeaderElectionReleaseOnCancel defines if the leader should step down voluntarily
+		// when the Manager ends. This requires the binary to immediately end when the
+		// Manager is stopped, otherwise, this setting is unsafe. Setting this significantly
+		// LeaseDuration time first.
+		//
+		// In the default scaffold provided, the program ends immediately after
+		// the manager stops, so would be fine to enable this option. However,
+		// if you are doing or is intended to do any operation such as perform cleanups
+		// after the manager stops then its usage might be unsafe.
+		LeaderElectionReleaseOnCancel: true,
+		Cache: cache.Options{
+			DefaultTransform: func(obj interface{}) (interface{}, error) {
+				mw, ok := obj.(*workv1.ManifestWork)
+				if !ok {
+					return obj, nil
+				}
+
+				// Transform the object to only include metadata and conditions
+				return &workv1.ManifestWork{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:              mw.Name,
+						Namespace:         mw.Namespace,
+						Annotations:       mw.Annotations,
+						Labels:            mw.Labels,
+						CreationTimestamp: mw.CreationTimestamp,
+						UID:               mw.UID,
+					},
+					Status: workv1.ManifestWorkStatus{
+						Conditions: mw.Status.Conditions,
+					},
+				}, nil
+			},
+		},
+	})
+	if err != nil {
+		logf.Log.Error(err, "unable to start manager")
+		os.Exit(1)
+	} // speeds up voluntary leader transitions as the new leader don't have to wait
+
+	now := time.Now()
+	if err = controllers.NewManifestworkReconciler(mgr.GetClient(), now).SetupWithManager(mgr); err != nil {
+		logf.Log.Error(err, "unable to create controller", "controller", "ManifestWork")
+		os.Exit(1)
+	}
+	logf.Log.Info("The manifestwork controller start time has been set", "startTime", now)
+
+	// +kubebuilder:scaffold:builder
+
+	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+		logf.Log.Error(err, "problem running manager")
+		os.Exit(1)
+	}
+}
+
 func telemetryServer(
 	registry prometheus.Gatherer,
 	host string,
@@ -310,3 +413,40 @@ func (m *metricHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 }
+
+// Check the ConfigMap to see if the timestamp-metrics should be enabled
+func isTimestampMetricsEnabled(ctx context.Context, clientset *kubernetes.Clientset) (bool, error) {
+	namespace, err := GetComponentNamespace()
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return false, fmt.Errorf("failed to get namespace: %v", err)
+		}
+		// the local test will run into here
+		klog.Info("serviceaccount namespace file does not exist, enable timestamp metrics")
+		return true, nil
+	}
+
+	// Get the ConfigMap
+	cm, err := clientset.CoreV1().ConfigMaps(namespace).Get(context.TODO(), configMapName, metav1.GetOptions{})
+	if errors.IsNotFound(err) {
+		return false, nil
+	} else if err != nil {
+		return false, fmt.Errorf("failed to get ConfigMap: %v", err)
+	}
+
+	// Check if collect-timestamp-metrics is "enable"
+	value, exists := cm.Data["collect-timestamp-metrics"]
+	if !exists {
+		return false, nil
+	}
+
+	return value == "Enable", nil
+}
+
+func GetComponentNamespace() (string, error) {
+	nsBytes, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+	if err != nil {
+		return "multicluster-engine", err
+	}
+	return string(nsBytes), nil
+}
diff --git a/cmd/clusterlifecycle-state-metrics/main_test.go b/cmd/clusterlifecycle-state-metrics/main_test.go
@@ -1,7 +1,7 @@
 // Copyright (c) 2020 Red Hat, Inc.
 // Copyright Contributors to the Open Cluster Management project
 
-
+//go:build testrunmain
 // +build testrunmain
 
 package main

diff --git a/deploy/clusterrole.yaml b/deploy/clusterrole.yaml
@@ -53,3 +53,13 @@ rules:
 - apiGroups: ["work.open-cluster-management.io"]
   resources: ["manifestworks"]
   verbs: ["get","list","watch"]
+# Allow hub to patch manifestwroks annotation
+- apiGroups: ["work.open-cluster-management.io"]
+  resources: ["manifestworks"]
+  verbs: ["patch"]
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get","create","update","patch","delete"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["get","create","update","patch","delete"]
diff --git a/go.mod b/go.mod
@@ -3,86 +3,89 @@ module github.com/stolostron/clusterlifecycle-state-metrics
 go 1.22.5
 
 require (
-	github.com/onsi/ginkgo/v2 v2.9.1
-	github.com/onsi/gomega v1.27.4
+	github.com/onsi/ginkgo/v2 v2.19.0
+	github.com/onsi/gomega v1.33.1
 	github.com/openshift/api v3.9.1-0.20191111211345-a27ff30ebf09+incompatible
 	github.com/openshift/build-machinery-go v0.0.0-20220720161851-9b4f0386f6b0
 	github.com/openshift/client-go v0.0.0-20210916133943-9acee1a0fb83
-	github.com/prometheus/client_golang v1.12.2
+	github.com/prometheus/client_golang v1.19.1
 	github.com/stolostron/applier v0.0.0-20220328110401-26b0ea4f8e1e
 	github.com/stolostron/cluster-lifecycle-api v0.0.0-20220714081119-eae2fe1f05fd
 	github.com/stolostron/library-go v0.0.0-20220328023725-63d77a3ad428
 	golang.org/x/net v0.34.0
-	k8s.io/api v0.26.7
-	k8s.io/apimachinery v0.26.7
-	k8s.io/client-go v0.26.7
+	k8s.io/api v0.31.0
+	k8s.io/apimachinery v0.31.0
+	k8s.io/client-go v0.31.0
 	k8s.io/klog v1.0.0
-	k8s.io/klog/v2 v2.90.1
+	k8s.io/klog/v2 v2.130.1
 	k8s.io/kube-state-metrics v1.9.8
 	open-cluster-management.io/api v0.8.0
-	sigs.k8s.io/controller-runtime v0.12.3
+	sigs.k8s.io/controller-runtime v0.19.3
 )
 
 require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.1.1 // indirect
 	github.com/Masterminds/sprig/v3 v3.2.2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
-	github.com/evanphx/json-patch v4.12.0+incompatible // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.9.0 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect
-	github.com/go-logr/logr v1.2.3 // indirect
-	github.com/go-logr/zapr v1.2.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.19.6 // indirect
-	github.com/go-openapi/jsonreference v0.20.1 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
-	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
+	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3 // indirect
-	github.com/google/uuid v1.3.0 // indirect
+	github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.2.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.7.3 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/shopspring/decimal v1.2.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
-	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.19.1 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.26.0 // indirect
 	golang.org/x/crypto v0.32.0 // indirect
-	golang.org/x/oauth2 v0.0.0-20220628200809-02e64fa58f26 // indirect
+	golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
+	golang.org/x/oauth2 v0.21.0 // indirect
 	golang.org/x/sys v0.29.0 // indirect
 	golang.org/x/term v0.28.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
-	golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect
+	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
-	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiextensions-apiserver v0.24.2 // indirect
-	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
-	k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
+	k8s.io/apiextensions-apiserver v0.31.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
+	sigs.k8s.io/yaml v1.4.0 // indirect
 )