feat(controller): allow enabling pprof (#2838)

Signed-off-by: Hidde Beydals <[email protected]>

cmd/controlplane/controller.go

@@ -41,6 +41,8 @@ type controllerOptions struct {
 	ArgoCDKubeConfig    string
 	ArgoCDNamespaceOnly bool
 
+	PprofBindAddress string
+
 	Logger *logging.Logger
 }
 
@@ -72,6 +74,7 @@ func (o *controllerOptions) complete() {
 	o.ArgoCDEnabled = types.MustParseBool(os.GetEnv("ARGOCD_INTEGRATION_ENABLED", "true"))
 	o.ArgoCDKubeConfig = os.GetEnv("ARGOCD_KUBECONFIG", "")
 	o.ArgoCDNamespaceOnly = types.MustParseBool(os.GetEnv("ARGOCD_WATCH_ARGOCD_NAMESPACE_ONLY", "false"))
+	o.PprofBindAddress = os.GetEnv("PPROF_BIND_ADDRESS", "")
 }
 
 func (o *controllerOptions) run(ctx context.Context) error {
@@ -183,6 +186,7 @@ func (o *controllerOptions) setupKargoManager(
 			Metrics: server.Options{
 				BindAddress: "0",
 			},
+			PprofBindAddress: o.PprofBindAddress,
 			Client: client.Options{
 				Cache: &client.CacheOptions{
 					// The controller does not have cluster-wide permissions, to

cmd/controlplane/garbage_collector.go

@@ -24,6 +24,8 @@ import (
 type garbageCollectorOptions struct {
 	KubeConfig string
 
+	PprofBindAddress string
+
 	Logger *logging.Logger
 }
 
@@ -51,6 +53,7 @@ func newGarbageCollectorCommand() *cobra.Command {
 
 func (o *garbageCollectorOptions) complete() {
 	o.KubeConfig = os.GetEnv("KUBECONFIG", "")
+	o.PprofBindAddress = os.GetEnv("PPROF_BIND_ADDRESS", "")
 }
 
 func (o *garbageCollectorOptions) run(ctx context.Context) error {
@@ -105,6 +108,7 @@ func (o *garbageCollectorOptions) setupManager(ctx context.Context) (manager.Man
 			Metrics: server.Options{
 				BindAddress: "0",
 			},
+			PprofBindAddress: o.PprofBindAddress,
 		},
 	)
 	if err != nil {

cmd/controlplane/management_controller.go

@@ -21,12 +21,18 @@ import (
 	"github.com/akuity/kargo/internal/controller/management/serviceaccounts"
 	"github.com/akuity/kargo/internal/logging"
 	"github.com/akuity/kargo/internal/os"
+	"github.com/akuity/kargo/internal/types"
 	versionpkg "github.com/akuity/kargo/internal/version"
 )
 
 type managementControllerOptions struct {
 	KubeConfig string
 
+	KargoNamespace               string
+	ManageControllerRoleBindings bool
+
+	PprofBindAddress string
+
 	Logger *logging.Logger
 }
 
@@ -54,6 +60,9 @@ func newManagementControllerCommand() *cobra.Command {
 
 func (o *managementControllerOptions) complete() {
 	o.KubeConfig = os.GetEnv("KUBECONFIG", "")
+	o.KargoNamespace = os.GetEnv("KARGO_NAMESPACE", "kargo")
+	o.ManageControllerRoleBindings = types.MustParseBool(os.GetEnv("MANAGE_CONTROLLER_ROLE_BINDINGS", "true"))
+	o.PprofBindAddress = os.GetEnv("PPROF_BIND_ADDRESS", "")
 }
 
 func (o *managementControllerOptions) run(ctx context.Context) error {
@@ -81,7 +90,7 @@ func (o *managementControllerOptions) run(ctx context.Context) error {
 		return fmt.Errorf("error setting up Projects reconciler: %w", err)
 	}
 
-	if os.GetEnv("MANAGE_CONTROLLER_ROLE_BINDINGS", "true") == "true" {
+	if o.ManageControllerRoleBindings {
 		if err := serviceaccounts.SetupReconcilerWithManager(
 			kargoMgr,
 			serviceaccounts.ReconcilerConfigFromEnv(),
@@ -130,11 +139,12 @@ func (o *managementControllerOptions) setupManager(ctx context.Context) (manager
 			Metrics: server.Options{
 				BindAddress: "0",
 			},
+			PprofBindAddress: o.PprofBindAddress,
 			Cache: cache.Options{
 				ByObject: map[client.Object]cache.ByObject{
 					&corev1.ServiceAccount{}: {
 						Namespaces: map[string]cache.Config{
-							os.GetEnv("KARGO_NAMESPACE", "kargo"): {},
+							o.KargoNamespace: {},
 						},
 					},
 				},

cmd/controlplane/webhooks.go

@@ -30,6 +30,8 @@ import (
 type webhooksServerOptions struct {
 	KubeConfig string
 
+	PprofBindAddress string
+
 	Logger *logging.Logger
 }
 
@@ -57,6 +59,7 @@ func newWebhooksServerCommand() *cobra.Command {
 
 func (o *webhooksServerOptions) complete() {
 	o.KubeConfig = os.GetEnv("KUBECONFIG", "")
+	o.PprofBindAddress = os.GetEnv("PPROF_BIND_ADDRESS", "")
 }
 
 func (o *webhooksServerOptions) run(ctx context.Context) error {
@@ -100,6 +103,7 @@ func (o *webhooksServerOptions) run(ctx context.Context) error {
 			Metrics: server.Options{
 				BindAddress: "0",
 			},
+			PprofBindAddress: o.PprofBindAddress,
 		},
 	)
 	if err != nil {

docs/docs/40-contributor-guide/5-debugging-kargo.md

@@ -0,0 +1,63 @@
+---
+description: Steps to help you debug Kargo when you run into issues.
+sidebar_label: Debugging Kargo
+---
+
+# Debugging Kargo
+
+From time to time, you may need to debug Kargo. This document outlines options
+to help you achieve a better understanding of what Kargo (internally) is doing,
+especially when you are running into issues which are not immediately obvious
+by just reading the code.
+
+As a Kargo user, you may not need to debug Kargo itself, but this document may
+serve you when a Kargo maintainer asks you to provide additional information
+about an issue you are facing.
+
+## Enabling pprof endpoints
+
+Kargo components can be configured to expose [`pprof` endpoints](https://golang.org/pkg/net/http/pprof/).
+These endpoints can be used to profile the components when they are running,
+and can be useful to understand what the components are doing and where they
+are spending time.
+
+To enable the `pprof` endpoint on a component, you can set the
+`PPROF_BIND_ADDRESS` environment variable to the address where the component
+should listen for `pprof` requests. For example, to enable the `pprof` endpoint
+on port `6060` of the controller, you can set the `PPROF_BIND_ADDRESS`
+environment variable to `:6060`.
+
+```yaml
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kargo-controller
+spec:
+  # ...omitted for brevity
+  template:
+    spec:
+      containers:
+      - name: kargo-controller
+        env:
+        - name: PPROF_BIND_ADDRESS
+          value: ":6060"
+```
+
+After setting the `PPROF_BIND_ADDRESS` environment variable, the `pprof`
+endpoints will be available at `http://<controller-ip>:6060/debug/pprof/`.
+
+### Collecting a profile
+
+To collect a profile, you can port-forward the `pprof` address to your local
+machine and collect the data from an endpoint of choice. For example, to
+collect a heap profile, you can run:
+
+```console
+$ kubectl port-forward -n <namespace> deployment/<component> 6060
+$ curl -Sk -v http://localhost:6060/debug/pprof/heap > heap.out
+```
+
+This will collect a heap profile in the `heap.out` file, which you can then
+[analyze using `go`](https://go.dev/blog/pprof), or share with a Kargo
+maintainer.