feat: Configurable cgroup ID regex for process discovery component (#…

…1557) * feat: Configurable cgroup ID regex for process discovery component * Add relevant unit tests and update docs Signed-off-by: Mahendra Paipuri <[email protected]> * docs: Address PR comments Co-authored-by: Clayton Cornell <[email protected]> * docs: Improve docs based on PR comments * refactor: Update regexp during args update Signed-off-by: Mahendra Paipuri <[email protected]> * test: Add a unit test to verify cgroup regex updating Signed-off-by: Mahendra Paipuri <[email protected]> * test: Add missing import Signed-off-by: Mahendra Paipuri <[email protected]> * refactor: Export cgroup paths in targets * This is a simplified approach to the original idea. Here we export cgroup paths as one of the labels and users can use relabel component to retrieve the relevant cgroup IDs. * In the case of cgroups v1, we export all the controllers paths delimited by `|` where as in cgroups v2, there is always one path Signed-off-by: Mahendra Paipuri <[email protected]> * docs: Add an example on how to use cgroup path Disable cgroup_path meta data by default * docs: Correct default value of `cgroup_path` in docs Co-authored-by: Christian Simon <[email protected]> * Update changelog * docs: Update based on review comment Co-authored-by: Clayton Cornell <[email protected]> --------- Signed-off-by: Mahendra Paipuri <[email protected]> Co-authored-by: Clayton Cornell <[email protected]> Co-authored-by: Christian Simon <[email protected]>
grafana · Dec 3, 2024 · cbc8723 · cbc8723
1 parent 2b7003d
commit cbc8723
Show file tree

Hide file tree

Showing 7 changed files with 153 additions and 15 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -286,6 +286,8 @@ v1.4.0
 
 - Add the label `alloy_cluster` in the metric `alloy_config_hash` when the flag `cluster.name` is set to help differentiate between
   configs from the same alloy cluster or different alloy clusters. (@wildum)
+
+- Add support for discovering the cgroup path(s) of a process in `process.discovery`. (@mahendrapaipuri)
 
 ### Bugfixes
 

diff --git a/docs/sources/reference/components/discovery/discovery.process.md b/docs/sources/reference/components/discovery/discovery.process.md
@@ -111,6 +111,7 @@ The following arguments are supported:
 | `commandline`  | `bool` | A flag to enable discovering `__meta_process_commandline` label. | true    | no       |
 | `uid`          | `bool` | A flag to enable discovering `__meta_process_uid`: label.        | true    | no       |
 | `username`     | `bool` | A flag to enable discovering `__meta_process_username`: label.   | true    | no       |
+| `cgroup_path`  | `bool` | A flag to enable discovering `__meta_cgroup_path__` label.       | false    | no       |
 | `container_id` | `bool` | A flag to enable discovering `__container_id__` label.           | true    | no       |
 
 ## Exported fields
@@ -129,6 +130,7 @@ Each target includes the following labels:
 * `__meta_process_commandline`: The process command line. Taken from `/proc/<pid>/cmdline`.
 * `__meta_process_uid`: The process UID. Taken from `/proc/<pid>/status`.
 * `__meta_process_username`: The process username. Taken from `__meta_process_uid` and `os/user/LookupID`.
+* `__meta_cgroup_path`: The cgroup path under which the process is running. In the case of cgroups v1, this label includes all the controllers paths delimited by `|`.
 * `__container_id__`: The container ID. Taken from `/proc/<pid>/cgroup`. If the process is not running in a container, this label is not set.
 
 ## Component health
@@ -157,6 +159,7 @@ discovery.process "all" {
     commandline = true
     username = true
     uid = true
+    cgroup_path = true
     container_id = true
   }
 }
@@ -187,6 +190,34 @@ discovery.process "all" {
   }
 }
 
+### Example discovering processes on the local host based on `cgroups` path
+
+The following example configuration shows you how to discover processes running under systemd services on the local host.
+
+```alloy
+discovery.process "all" {
+  refresh_interval = "60s"
+  discover_config {
+    cwd = true
+    exe = true
+    commandline = true
+    username = true
+    uid = true
+    cgroup_path = true
+    container_id = true
+  }
+}
+
+discovery.relabel "systemd_services" {
+  targets = discovery.process.all.targets
+  // Only keep the targets that correspond to systemd services
+  rule {
+    action = "keep"
+    regex = "^.*/([a-zA-Z0-9-_]+).service(?:.*$)"
+    source_labels = ["__meta_cgroup_id"]
+  }
+}
+
 ```
 <!-- START GENERATED COMPATIBLE COMPONENTS -->
 

diff --git a/internal/component/discovery/process/args.go b/internal/component/discovery/process/args.go
@@ -19,6 +19,7 @@ type DiscoverConfig struct {
 	Username    bool `alloy:"username,attr,optional"`
 	UID         bool `alloy:"uid,attr,optional"`
 	ContainerID bool `alloy:"container_id,attr,optional"`
+	CgroupPath  bool `alloy:"cgroup_path,attr,optional"`
 }
 
 var DefaultConfig = Arguments{
@@ -29,6 +30,7 @@ var DefaultConfig = Arguments{
 		Exe:         true,
 		Commandline: true,
 		ContainerID: true,
+		CgroupPath:  false,
 	},
 }
 

diff --git a/internal/component/discovery/process/cgroup.go b/internal/component/discovery/process/cgroup.go
@@ -0,0 +1,26 @@
+//go:build linux
+
+package process
+
+import (
+	"bufio"
+	"io"
+	"strings"
+)
+
+// getPathFromCGroup fetches cgroup path(s) from process.
+// In the case of cgroups v2 (unified), there will be only
+// one path and function returns that path. In the case
+// cgroups v1, there will be one path for each controller.
+// The function will join all the paths using `|` and
+// returns as one string. Users can use relabel component
+// to retrieve the path that they are interested.
+func getPathFromCGroup(cgroup io.Reader) string {
+	var paths []string
+	scanner := bufio.NewScanner(cgroup)
+	for scanner.Scan() {
+		line := scanner.Bytes()
+		paths = append(paths, string(line))
+	}
+	return strings.Join(paths, "|")
+}
diff --git a/internal/component/discovery/process/cgroup_test.go b/internal/component/discovery/process/cgroup_test.go
@@ -0,0 +1,52 @@
+//go:build linux
+
+package process
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGenericCGroupMatching(t *testing.T) {
+	type testcase = struct {
+		name, cgroup, expectedPath string
+	}
+	testcases := []testcase{
+		{
+			name:         "cgroups v2",
+			cgroup:       `0::/system.slice/slurmstepd.scope/job_1446354/step_batch/user/task_0`, // cgroups v2
+			expectedPath: `0::/system.slice/slurmstepd.scope/job_1446354/step_batch/user/task_0`,
+		},
+		{
+			name: "cgroups v1",
+			cgroup: `12:rdma:/
+11:devices:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator
+10:cpuset:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator
+9:blkio:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator
+8:pids:/user.slice/user-118.slice/session-5.scope
+7:memory:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator
+6:hugetlb:/
+5:net_cls,net_prio:/
+4:perf_event:/
+3:cpu,cpuacct:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator
+2:freezer:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator
+1:name=systemd:/user.slice/user-118.slice/session-5.scope`, // cgroups v1
+			expectedPath: "12:rdma:/|11:devices:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator|10:cpuset:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator|9:blkio:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator|8:pids:/user.slice/user-118.slice/session-5.scope|7:memory:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator|6:hugetlb:/|5:net_cls,net_prio:/|4:perf_event:/|3:cpu,cpuacct:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator|2:freezer:/machine/qemu-1-instance-00000025.libvirt-qemu/emulator|1:name=systemd:/user.slice/user-118.slice/session-5.scope",
+		},
+		{
+			name:         "empty cgroups path", // Should not happen in real cases
+			cgroup:       "",
+			expectedPath: "",
+		},
+	}
+	for i, tc := range testcases {
+		t.Run(fmt.Sprintf("testcase %d %s", i, tc.name), func(t *testing.T) {
+			cgroupID := getPathFromCGroup(bytes.NewReader([]byte(tc.cgroup)))
+			expected := tc.expectedPath
+			require.Equal(t, expected, cgroupID)
+		})
+	}
+}
diff --git a/internal/component/discovery/process/discover.go b/internal/component/discovery/process/discover.go
@@ -8,7 +8,6 @@ import (
 	"os"
 	"os/user"
 	"path"
-	"runtime"
 
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
@@ -24,6 +23,7 @@ const (
 	labelProcessCommandline = "__meta_process_commandline"
 	labelProcessUsername    = "__meta_process_username"
 	labelProcessUID         = "__meta_process_uid"
+	labelProcessCgroupPath  = "__meta_process_cgroup_path"
 	labelProcessContainerID = "__container_id__"
 )
 
@@ -33,12 +33,13 @@ type process struct {
 	cwd         string
 	commandline string
 	containerID string
+	cgroupPath  string
 	username    string
 	uid         string
 }
 
 func (p process) String() string {
-	return fmt.Sprintf("pid=%s exe=%s cwd=%s commandline=%s containerID=%s", p.pid, p.exe, p.cwd, p.commandline, p.containerID)
+	return fmt.Sprintf("pid=%s exe=%s cwd=%s commandline=%s cgrouppath=%s containerID=%s", p.pid, p.exe, p.cwd, p.commandline, p.cgroupPath, p.containerID)
 }
 
 func convertProcesses(ps []process) []discovery.Target {
@@ -51,7 +52,7 @@ func convertProcesses(ps []process) []discovery.Target {
 }
 
 func convertProcess(p process) discovery.Target {
-	t := make(discovery.Target, 5)
+	t := make(discovery.Target, 8)
 	t[labelProcessID] = p.pid
 	if p.exe != "" {
 		t[labelProcessExe] = p.exe
@@ -71,6 +72,9 @@ func convertProcess(p process) discovery.Target {
 	if p.uid != "" {
 		t[labelProcessUID] = p.uid
 	}
+	if p.cgroupPath != "" {
+		t[labelProcessCgroupPath] = p.cgroupPath
+	}
 	return t
 }
 
@@ -92,7 +96,7 @@ func discover(l log.Logger, cfg *DiscoverConfig) ([]process, error) {
 	for _, p := range processes {
 		spid := fmt.Sprintf("%d", p.Pid)
 		var (
-			exe, cwd, commandline, containerID, username, uid string
+			exe, cwd, commandline, containerID, cgroupPath, username, uid string
 		)
 		if cfg.Exe {
 			exe, err = p.Exe()
@@ -131,20 +135,27 @@ func discover(l log.Logger, cfg *DiscoverConfig) ([]process, error) {
 				uid = fmt.Sprintf("%d", uids[0])
 			}
 		}
-
 		if cfg.ContainerID {
 			containerID, err = getLinuxProcessContainerID(spid)
 			if err != nil {
 				loge(int(p.Pid), err)
 				continue
 			}
 		}
+		if cfg.CgroupPath {
+			cgroupPath, err = getLinuxProcessCgroupPath(spid)
+			if err != nil {
+				loge(int(p.Pid), err)
+				continue
+			}
+		}
 		res = append(res, process{
 			pid:         spid,
 			exe:         exe,
 			cwd:         cwd,
 			commandline: commandline,
 			containerID: containerID,
+			cgroupPath:  cgroupPath,
 			username:    username,
 			uid:         uid,
 		})
@@ -154,16 +165,28 @@ func discover(l log.Logger, cfg *DiscoverConfig) ([]process, error) {
 }
 
 func getLinuxProcessContainerID(pid string) (string, error) {
-	if runtime.GOOS == "linux" {
-		cgroup, err := os.Open(path.Join("/proc", pid, "cgroup"))
-		if err != nil {
-			return "", err
-		}
-		defer cgroup.Close()
-		cid := getContainerIDFromCGroup(cgroup)
-		if cid != "" {
-			return cid, nil
-		}
+	cgroup, err := os.Open(path.Join("/proc", pid, "cgroup"))
+	if err != nil {
+		return "", err
 	}
+	defer cgroup.Close()
+	cid := getContainerIDFromCGroup(cgroup)
+	if cid != "" {
+		return cid, nil
+	}
+
+	return "", nil
+}
+
+func getLinuxProcessCgroupPath(pid string) (string, error) {
+	cgroup, err := os.Open(path.Join("/proc", pid, "cgroup"))
+	if err != nil {
+		return "", err
+	}
+	defer cgroup.Close()
+	if cgroupPath := getPathFromCGroup(cgroup); cgroupPath != "" {
+		return cgroupPath, nil
+	}
+
 	return "", nil
 }
diff --git a/internal/component/discovery/process/process.go b/internal/component/discovery/process/process.go
@@ -32,6 +32,7 @@ func New(opts component.Options, args Arguments) (*Component, error) {
 		argsUpdates:   make(chan Arguments),
 		args:          args,
 	}
+
 	return c, nil
 }
 
@@ -51,6 +52,7 @@ func (c *Component) Run(ctx context.Context) error {
 		}
 		c.processes = convertProcesses(processes)
 		c.changed()
+
 		return nil
 	}
 	if err := doDiscover(); err != nil {