Prepare monitoring alert rules

2026-05-27 06:38:57 +02:00
parent 85a0eb4c3a
commit 8e111d1e04
7 changed files with 225 additions and 3 deletions
@@ -28,6 +28,24 @@ groups:
          summary: "{{ $labels.instance }} is slow"
          description: "Blackbox probe duration is above 5 seconds for {{ $labels.instance }}."

+      - alert: HomelabCertificateExpiresSoon
+        expr: (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) < 21 * 24 * 3600 and (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) > 7 * 24 * 3600
+        for: 30m
+        labels:
+          severity: warning
+        annotations:
+          summary: "TLS certificate expires soon for {{ $labels.instance }}"
+          description: "The earliest certificate expiry for {{ $labels.instance }} is below 21 days."
+
+      - alert: HomelabCertificateExpiresCritical
+        expr: (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) <= 7 * 24 * 3600
+        for: 15m
+        labels:
+          severity: critical
+        annotations:
+          summary: "TLS certificate is close to expiry for {{ $labels.instance }}"
+          description: "The earliest certificate expiry for {{ $labels.instance }} is at or below 7 days, or already expired."
+
  - name: homelab-host
    rules:
      - alert: HomelabDiskAlmostFull
@@ -56,3 +74,59 @@ groups:
        annotations:
          summary: "Traefik 5xx responses for {{ $labels.service }}"
          description: "Traefik reports at least 5 5xx responses for {{ $labels.service }} within 5 minutes."
+
+  - name: homelab-backup-and-containers
+    rules:
+      - alert: HomelabTextfileExporterStale
+        expr: time() - homelab_textfile_exporter_last_run_timestamp_seconds > 2 * 60 * 60
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Homelab textfile metrics are stale"
+          description: "The host textfile exporter has not refreshed metrics for more than 2 hours."
+
+      - alert: HomelabBorgMetricsMissing
+        expr: absent(homelab_borg_last_completed_timestamp_seconds)
+        for: 15m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Borg backup metrics are missing"
+          description: "Prometheus cannot see the homelab_borg_last_completed_timestamp_seconds metric."
+
+      - alert: HomelabBorgBackupStale
+        expr: time() - homelab_borg_last_completed_timestamp_seconds > 30 * 60 * 60
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Borg backup is stale"
+          description: "The latest completed Borg backup is older than 30 hours."
+
+      - alert: HomelabBorgLastJobFailed
+        expr: homelab_borg_last_success != 1
+        for: 15m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Latest Borg backup did not complete successfully"
+          description: "The latest Borg UI job status is {{ $labels.status }} for archive {{ $labels.archive }}."
+
+      - alert: HomelabBorgLastJobCompletedWithWarnings
+        expr: homelab_borg_last_job_warning == 1
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Latest Borg backup completed with warnings"
+          description: "The latest Borg UI job completed with warnings for archive {{ $labels.archive }}."
+
+      - alert: HomelabCriticalContainerDown
+        expr: homelab_critical_container_running == 0
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Critical container is down: {{ $labels.name }}"
+          description: "The host textfile exporter reports that critical container {{ $labels.name }} is not running."