Add alert rules to kserve-controller based on the KF093 spec (#265) (#…

…274) * Add alert rules to kserve-controller based on the KF093 spec * add cos integration tests for alert rules Co-authored-by: Robert Gildein <[email protected]>
canonical · Oct 11, 2024 · c9d6747 · c9d6747
1 parent 92c8a79
commit c9d6747
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 0 deletions.
diff --git a/charms/kserve-controller/src/prometheus_alert_rules/KubeflowKserveControllerServices.rules b/charms/kserve-controller/src/prometheus_alert_rules/KubeflowKserveControllerServices.rules
@@ -0,0 +1,24 @@
+groups:
+- name: KubeflowKserveControllerServices
+  rules:
+  - alert: KubeflowServiceDown
+    expr: up{} < 1
+    for: 5m
+    labels:
+      severity: critical
+    annotations:
+      summary: "{{ $labels.juju_charm }} service is Down ({{ $labels.juju_model }}/{{ $labels.juju_unit }})"
+      description: |
+       One or more targets of {{ $labels.juju_charm }} charm are down on unit {{ $labels.juju_model }}/{{ $labels.juju_unit }}.
+       LABELS = {{ $labels }}
+
+  - alert: KubeflowServiceIsNotStable
+    expr: avg_over_time(up{}[10m]) < 0.5
+    for: 0m
+    labels:
+      severity: warning
+    annotations:
+      summary: "{{ $labels.juju_charm }} service is not stable ({{ $labels.juju_model }}/{{ $labels.juju_unit }})"
+      description: |
+        {{ $labels.juju_charm }} unit {{ $labels.juju_model }}/{{ $labels.juju_unit }} has been unreachable at least 50% of the time over the last 10 minutes.
+        LABELS = {{ $labels }}
diff --git a/charms/kserve-controller/tests/integration/test_charm.py b/charms/kserve-controller/tests/integration/test_charm.py
@@ -16,9 +16,11 @@
 import yaml
 from charmed_kubeflow_chisme.kubernetes import KubernetesResourceHandler
 from charmed_kubeflow_chisme.testing import (
+    assert_alert_rules,
     assert_logging,
     assert_metrics_endpoint,
     deploy_and_assert_grafana_agent,
+    get_alert_rules,
 )
 from lightkube.core.exceptions import ApiError
 from lightkube.models.meta_v1 import ObjectMeta
@@ -339,6 +341,14 @@ async def test_metrics_enpoint(ops_test):
     await assert_metrics_endpoint(app, metrics_port=8080, metrics_path="/metrics")
 
 
+async def test_alert_rules(ops_test):
+    """Test check charm alert rules and rules defined in relation data bag."""
+    app = ops_test.model.applications[APP_NAME]
+    alert_rules = get_alert_rules()
+    logger.info("found alert_rules: %s", alert_rules)
+    await assert_alert_rules(app, alert_rules)
+
+
 #    # Remove the InferenceService deployed in RawDeployment mode
 #    lightkube_client.delete(
 #        inference_service_resource, name=inf_svc_name, namespace=rawdeployment_mode_namespace