Merge pull request #157 from coroot/fix_resource_usage_metrics

cgroup: don't report CPU, Memory, and IO usage for `/` cgroup
coroot · Dec 16, 2024 · 7f46467 · 7f46467
2 parents 6dcf8fe + 638facd
commit 7f46467
Show file tree

Hide file tree

Showing 8 changed files with 69 additions and 37 deletions.
diff --git a/cgroup/cpu.go b/cgroup/cpu.go
@@ -16,14 +16,19 @@ type CPUStat struct {
 	LimitCores           float64
 }
 
-func (cg Cgroup) CpuStat() (*CPUStat, error) {
+func (cg Cgroup) CpuStat() *CPUStat {
 	if cg.Version == V1 {
-		return cg.cpuStatV1()
+		st, _ := cg.cpuStatV1()
+		return st
 	}
-	return cg.cpuStatV2()
+	st, _ := cg.cpuStatV2()
+	return st
 }
 
 func (cg Cgroup) cpuStatV1() (*CPUStat, error) {
+	if cg.subsystems["cpu"] == "/" || cg.subsystems["cpuacct"] == "/" {
+		return nil, nil
+	}
 	throttling, err := readVariablesFromFile(path.Join(cgRoot, "cpu", cg.subsystems["cpu"], "cpu.stat"))
 	if err != nil {
 		return nil, err

diff --git a/cgroup/cpu_test.go b/cgroup/cpu_test.go
@@ -11,28 +11,29 @@ func TestCgroup_CpuStat(t *testing.T) {
 	cgRoot = "fixtures/cgroup"
 
 	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/100/cgroup"))
-	s, err := cg.CpuStat()
-	assert.Nil(t, err)
+	s := cg.CpuStat()
 	assert.Equal(t, 0., s.LimitCores)
 	assert.Equal(t, 26778.913419246, s.UsageSeconds)
 
 	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/200/cgroup"))
-	s, err = cg.CpuStat()
-	assert.Nil(t, err)
+	s = cg.CpuStat()
 	assert.Equal(t, 1.5, s.LimitCores)
 	assert.Equal(t, 254005.032764376, s.ThrottledTimeSeconds)
 
 	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/400/cgroup"))
-	s, err = cg.CpuStat()
-	assert.Nil(t, err)
+	s = cg.CpuStat()
 	assert.Equal(t, 0.1, s.LimitCores)
 	assert.Equal(t, 0.363166, s.ThrottledTimeSeconds)
 	assert.Equal(t, 3795.681254, s.UsageSeconds)
 
 	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/500/cgroup"))
-	s, err = cg.CpuStat()
-	assert.Nil(t, err)
+	s = cg.CpuStat()
 	assert.Equal(t, 0., s.LimitCores)
 	assert.Equal(t, 0., s.ThrottledTimeSeconds)
 	assert.Equal(t, 5531.521992, s.UsageSeconds)
+
+	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/1000/cgroup"))
+	s, err := cg.cpuStatV1()
+	assert.NoError(t, err)
+	assert.Nil(t, s)
 }
diff --git a/cgroup/fixtures/proc/1000/cgroup b/cgroup/fixtures/proc/1000/cgroup
@@ -0,0 +1,11 @@
+11:pids:/
+10:blkio:/
+9:net_cls,net_prio:/
+8:perf_event:/
+7:freezer:/
+6:memory:/
+5:cpu,cpuacct:/
+4:hugetlb:/
+3:cpuset:/
+2:devices:/system.slice/system-getty.slice
+1:name=systemd:/system.slice/system-getty.slice/[email protected]
diff --git a/cgroup/io.go b/cgroup/io.go
@@ -16,14 +16,19 @@ type IOStat struct {
 	WrittenBytes uint64
 }
 
-func (cg *Cgroup) IOStat() (map[string]IOStat, error) {
+func (cg *Cgroup) IOStat() map[string]IOStat {
 	if cg.Version == V1 {
-		return cg.ioStatV1()
+		st, _ := cg.ioStatV1()
+		return st
 	}
-	return cg.ioStatV2()
+	st, _ := cg.ioStatV2()
+	return st
 }
 
 func (cg *Cgroup) ioStatV1() (map[string]IOStat, error) {
+	if cg.subsystems["blkio"] == "/" {
+		return nil, nil
+	}
 	ops, err := readBlkioStatFile(path.Join(cgRoot, "blkio", cg.subsystems["blkio"], "blkio.throttle.io_serviced"))
 	if err != nil {
 		return nil, err

diff --git a/cgroup/io_test.go b/cgroup/io_test.go
@@ -11,8 +11,7 @@ func TestCgroup_IOStat(t *testing.T) {
 	cgRoot = "fixtures/cgroup"
 
 	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/200/cgroup"))
-	stat, err := cg.IOStat()
-	assert.Nil(t, err)
+	stat := cg.IOStat()
 	assert.Equal(t,
 		map[string]IOStat{
 			"8:0":  {ReadOps: 0, WriteOps: 281, ReadBytes: 0, WrittenBytes: 4603904},
@@ -24,13 +23,16 @@ func TestCgroup_IOStat(t *testing.T) {
 		stat)
 
 	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/400/cgroup"))
-	stat, err = cg.IOStat()
-	assert.Nil(t, err)
+	stat = cg.IOStat()
 	assert.Equal(t,
 		map[string]IOStat{
 			"252:0": {ReadOps: 22, WriteOps: 57111, ReadBytes: 11, WrittenBytes: 630538240},
 			"253:0": {ReadOps: 44, WriteOps: 57056, ReadBytes: 33, WrittenBytes: 630538241},
 		},
 		stat)
 
+	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/1000/cgroup"))
+	stat, err := cg.ioStatV1()
+	assert.NoError(t, err)
+	assert.Nil(t, stat)
 }
diff --git a/cgroup/memory.go b/cgroup/memory.go
@@ -14,14 +14,19 @@ type MemoryStat struct {
 	Limit uint64
 }
 
-func (cg *Cgroup) MemoryStat() (*MemoryStat, error) {
+func (cg *Cgroup) MemoryStat() *MemoryStat {
 	if cg.Version == V1 {
-		return cg.memoryStatV1()
+		st, _ := cg.memoryStatV1()
+		return st
 	}
-	return cg.memoryStatV2()
+	st, _ := cg.memoryStatV2()
+	return st
 }
 
 func (cg *Cgroup) memoryStatV1() (*MemoryStat, error) {
+	if cg.subsystems["memory"] == "/" {
+		return nil, nil
+	}
 	vars, err := readVariablesFromFile(path.Join(cgRoot, "memory", cg.subsystems["memory"], "memory.stat"))
 	if err != nil {
 		return nil, err

diff --git a/cgroup/memory_test.go b/cgroup/memory_test.go
@@ -11,29 +11,30 @@ func TestCgroup_MemoryStat(t *testing.T) {
 	cgRoot = "fixtures/cgroup"
 
 	cg, _ := NewFromProcessCgroupFile(path.Join("fixtures/proc/100/cgroup"))
-	stat, err := cg.MemoryStat()
-	assert.Nil(t, err)
+	stat := cg.MemoryStat()
 	assert.Equal(t, uint64(0), stat.Limit)
 
 	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/200/cgroup"))
-	stat, err = cg.MemoryStat()
-	assert.Nil(t, err)
+	stat = cg.MemoryStat()
 	assert.Equal(t, uint64(14775123968), stat.RSS)
 	assert.Equal(t, uint64(3206844416), stat.Cache)
 	assert.Equal(t, uint64(21474836480), stat.Limit)
 
 	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/400/cgroup"))
-	stat, err = cg.MemoryStat()
-	assert.Nil(t, err)
+	stat = cg.MemoryStat()
 	assert.Equal(t, uint64(44892160+0), stat.RSS)
 	assert.Equal(t, uint64(1044480), stat.Cache)
 	assert.Equal(t, uint64(0), stat.Limit)
 
 	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/500/cgroup"))
-	stat, err = cg.MemoryStat()
-	assert.Nil(t, err)
+	stat = cg.MemoryStat()
 	assert.Equal(t, uint64(75247616+4038656), stat.RSS)
 	assert.Equal(t, uint64(50835456), stat.Cache)
 	assert.Equal(t, uint64(4294967296), stat.Limit)
 
+	cg, _ = NewFromProcessCgroupFile(path.Join("fixtures/proc/1000/cgroup"))
+	stat, err := cg.memoryStatV1()
+	assert.NoError(t, err)
+	assert.Nil(t, stat)
+
 }
diff --git a/containers/container.go b/containers/container.go
@@ -230,7 +230,7 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 
 	ch <- counter(metrics.Restarts, float64(c.restarts))
 
-	if cpu, err := c.cgroup.CpuStat(); err == nil {
+	if cpu := c.cgroup.CpuStat(); cpu != nil {
 		if cpu.LimitCores > 0 {
 			ch <- gauge(metrics.CPULimit, cpu.LimitCores)
 		}
@@ -244,7 +244,7 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 		ch <- counter(metrics.DiskDelay, float64(c.delays.disk)/float64(time.Second))
 	}
 
-	if s, err := c.cgroup.MemoryStat(); err == nil {
+	if s := c.cgroup.MemoryStat(); s != nil {
 		ch <- gauge(metrics.MemoryRss, float64(s.RSS))
 		ch <- gauge(metrics.MemoryCache, float64(s.Cache))
 		if s.Limit > 0 {
@@ -257,7 +257,7 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 	}
 
 	if disks, err := node.GetDisks(); err == nil {
-		ioStat, _ := c.cgroup.IOStat()
+		ioStat := c.cgroup.IOStat()
 		for majorMinor, mounts := range c.getMounts() {
 			dev := disks.GetParentBlockDevice(majorMinor)
 			if dev == nil {
@@ -268,11 +268,13 @@ func (c *Container) Collect(ch chan<- prometheus.Metric) {
 				ch <- gauge(metrics.DiskSize, float64(fsStat.CapacityBytes), dls...)
 				ch <- gauge(metrics.DiskUsed, float64(fsStat.UsedBytes), dls...)
 				ch <- gauge(metrics.DiskReserved, float64(fsStat.ReservedBytes), dls...)
-				if io, ok := ioStat[majorMinor]; ok {
-					ch <- counter(metrics.DiskReadOps, float64(io.ReadOps), dls...)
-					ch <- counter(metrics.DiskReadBytes, float64(io.ReadBytes), dls...)
-					ch <- counter(metrics.DiskWriteOps, float64(io.WriteOps), dls...)
-					ch <- counter(metrics.DiskWriteBytes, float64(io.WrittenBytes), dls...)
+				if ioStat != nil {
+					if io, ok := ioStat[majorMinor]; ok {
+						ch <- counter(metrics.DiskReadOps, float64(io.ReadOps), dls...)
+						ch <- counter(metrics.DiskReadBytes, float64(io.ReadBytes), dls...)
+						ch <- counter(metrics.DiskWriteOps, float64(io.WriteOps), dls...)
+						ch <- counter(metrics.DiskWriteBytes, float64(io.WrittenBytes), dls...)
+					}
 				}
 			}
 		}