Add vm prometheus metrics (#756)

redhat-performance · Jan 10, 2024 · 0f5d63a · 0f5d63a
1 parent f7f2a3d
commit 0f5d63a
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 5 deletions.
diff --git a/benchmark_runner/common/prometheus/metrics-default.yaml b/benchmark_runner/common/prometheus/metrics-default.yaml
@@ -1,5 +1,3 @@
-
-
 # container CPU/Memory
 
 - query: (sum(irate(container_cpu_usage_seconds_total{container!="",namespace="benchmark-runner", mode!="idle"}[2m])) by (node)) > 0
@@ -17,3 +15,38 @@
 - query: container_memory_max_usage_bytes{pod=~"virt-launcher.*", container="compute"}
   metricName: VM Memory Max Usage Bytes
 
+- query: container_memory_cache{container=~"virt-handler", namespace="openshift-cnv"}
+  metricName: virt handler Cache
+
+- query: container_memory_working_set_bytes{container=~"virt-handler", namespace="openshift-cnv"}
+  metricName: virt handler working set bytes
+
+- query: container_memory_cache{container=~"virt-controller", namespace="openshift-cnv"}
+  metricName: virt controller Cache
+
+- query: container_memory_working_set_bytes{container=~"virt-controller", namespace="openshift-cnv"}
+  metricName: virt controller working set bytes
+
+- query: container_memory_cache{container=~"virt-operator", namespace="openshift-cnv"}
+  metricName: virt operator Cache
+
+- query: container_memory_working_set_bytes{container=~"virt-operator", namespace="openshift-cnv"}
+  metricName: virt operator working set bytes
+
+- query: container_memory_cache{container=~"virt-api", namespace="openshift-cnv"}
+  metricName: virt api Cache
+
+- query: container_memory_working_set_bytes{container=~"virt-api", namespace="openshift-cnv"}
+  metricName: virt api working set bytes
+
+- query: min ((kube_pod_container_resource_requests{namespace="openshift-cnv",container=~"virt-handler",resource="memory"}) - on(pod) group_left(node) container_memory_working_set_bytes{container="",namespace="openshift-cnv"})
+  metricName: virt handler exceeds requested memory
+
+- query: min ((kube_pod_container_resource_requests{namespace="openshift-cnv",container=~"virt-controller",resource="memory"}) - on(pod) group_left(node) container_memory_working_set_bytes{container="",namespace="openshift-cnv"})
+  metricName: virt controller exceeds requested memory
+
+- query: min ((kube_pod_container_resource_requests{namespace="openshift-cnv",container=~"virt-operator",resource="memory"}) - on(pod) group_left(node) container_memory_working_set_bytes{container="",namespace="openshift-cnv"})
+  metricName: virt operator exceeds requested memory
+
+- query: min ((kube_pod_container_resource_requests{namespace="openshift-cnv",container=~"virt-api",resource="memory"}) - on(pod) group_left(node) container_memory_working_set_bytes{container="",namespace="openshift-cnv"})
+  metricName: virt api exceeds requested memory
diff --git a/benchmark_runner/common/prometheus/prometheus_metrics_operations.py b/benchmark_runner/common/prometheus/prometheus_metrics_operations.py
@@ -180,15 +180,15 @@ def parse_prometheus_metrics(data: dict):
             elif 'containerMemory-benchmark-runner' in query:
                 suffix = 'Memory'
             else:
-                suffix = ''
+                suffix = None
             total = 0
             max = 0
             for item in data_list:
                 for val in item['values']:
                     if float(val[1]) > max:
                         max = round(float(val[1]), 3)
                 total = total + max
-                if 'VM Memory' in query:
+                if not suffix:
                     result_dict[f'{query}'] = max
                 else:
                     result_dict[f"{item['metric']['node']}_{suffix}"] = max

diff --git a/benchmark_runner/grafana/perf/jsonnet/main.libsonnet b/benchmark_runner/grafana/perf/jsonnet/main.libsonnet
@@ -5570,7 +5570,7 @@ g.dashboard.new('PerfCI-Regression-Summary')
             ]),
             ////////////////////////
 
-            g.panel.stateTimeline.new('120 Windows VM Memory')
+            g.panel.stateTimeline.new('111 Windows VM Memory')
             + stateTimeline.queryOptions.withDatasource('Elasticsearch-windows-results')
             + g.panel.stateTimeline.withDescription('Time till VM Login - Lower is better')