combine ssl alerts to use medians/averages

files/core_alerts.yml

@@ -180,15 +180,15 @@ groups:
       summary: "Service failed (instance {{ $labels.instance }})"
       description: "Service Failed. Check host to ensure functioning.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
   - alert: CertificateExpirationWarning
-    expr: ssl_cert_not_after - time() AND (((ssl_cert_not_after - time()) /  (ssl_cert_not_after - ssl_cert_not_before) ) < 0.33) - time()
+    expr: ((quantile by (serial_no,cn,instance) (0.5, ssl_cert_not_after)) and ((quantile by (serial_no,cn,instance) (0.5, ((ssl_cert_not_after - time()) /  (ssl_cert_not_after - ssl_cert_not_before))) ) < 0.3)) - time()
     for: 2m
     labels:
       severity: warning
     annotations:
       summary: "Certificate (instance {{ $labels.instance }})"
-      description: "Certificate more than 66% expired.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
+      description: "Certificate more than 70% expired.\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
   - alert: CertificateExpirationCritical
-    expr: ssl_cert_not_after - time() AND (((ssl_cert_not_after - time()) /  (ssl_cert_not_after - ssl_cert_not_before) ) < 0.2) - time()
+    expr: ((quantile by (serial_no,cn,instance) (0.5, ssl_cert_not_after)) and ((quantile by (serial_no,cn,instance) (0.5, ((ssl_cert_not_after - time()) /  (ssl_cert_not_after - ssl_cert_not_before))) ) < 0.2)) - time()
     for: 2m
     labels:
       severity: critical
@@ -196,7 +196,7 @@ groups:
       summary: "Certificate (instance {{ $labels.instance }})"
       description: "Certificate more than 80% expired!\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}"
   - alert: CertificateTestFailed
-    expr: ssl_probe_success < 1
+    expr: avg by (instance) (ssl_probe_success) < 1
     for: 15m
     labels:
       severity: critical