added warning for failed services

files/core_alerts.yml

@@ -171,5 +171,13 @@ groups:
     annotations:
       summary: "Unusual disk read latency (instance {{ $labels.instance }})"
       description: "Disk latency is growing (read operations > 100ms)\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
+  - alert: ServiceFailed
+    expr: node_systemd_unit_state{state='failed'} > 0 
+    for: 30m
+    labels:
+      severity: warning
+    annotations:
+      summary: "Service failed (instance {{ $labels.instance }})"
+      description: "Service Failed. Check host to ensure functioning.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"