base/longhorn-system: improve alerts about backup jobs

thaum-xyz · Oct 14, 2023 · 984c81c · 984c81c
1 parent add67f1
commit 984c81c
Showing 1 changed file with 10 additions and 2 deletions.
diff --git a/base/longhorn-system/prometheusrule.yaml b/base/longhorn-system/prometheusrule.yaml
@@ -86,10 +86,18 @@ spec:
       labels:
         issue: Longhorn node {{$labels.node}} experiences high CPU pressure.
         severity: warning
+    - alert: LonghornVolumeBackupStuck
+      expr: count by (volume) (longhorn_backup_state < 2)
+      for: 8h
+      labels:
+        severity: warning
+      annotations:
+        description: There are {{$value}} longhorn backups of a volume {{$labels.volume}} stuck for at least 8h.
+        summary: Longhorn backups stuck.
     - alert: LonghornVolumeBackupError
-      expr: count by (backup, volume) (longhorn_backup_state > 2)
+      expr: count by (volume) (longhorn_backup_state > 3)
       labels:
         severity: warning
       annotations:
-        description: Longhorn backup {{$labels.backup}} of a volume {{$labels.volume}} failed.
+        description: There are {{$value}} longhorn backups of a volume {{$labels.volume}} which failed to complete.
         summary: Longhorn backups failed.