groups: - name: homelab-availability rules: - alert: HomelabExternalConnectivityDown expr: sum(probe_success{job="blackbox-http"} == 0) >= 5 for: 8m labels: severity: warning annotations: summary: "External connectivity appears down" description: "At least 5 public homelab endpoints are unreachable. Likely WAN, DNS, or provider issue." - alert: HomelabEndpointDown expr: (probe_success{job="blackbox-http"} == 0) unless on() (sum(probe_success{job="blackbox-http"} == 0) >= 5) for: 8m labels: severity: critical annotations: summary: "{{ $labels.instance }} is not reachable" description: "Blackbox probe failed for {{ $labels.instance }}." - alert: HomelabEndpointSlow expr: probe_duration_seconds{job="blackbox-http"} > 5 for: 5m labels: severity: warning annotations: summary: "{{ $labels.instance }} is slow" description: "Blackbox probe duration is above 5 seconds for {{ $labels.instance }}." - alert: HomelabCertificateExpiresSoon expr: (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) < 21 * 24 * 3600 and (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) > 7 * 24 * 3600 for: 30m labels: severity: warning annotations: summary: "TLS certificate expires soon for {{ $labels.instance }}" description: "The earliest certificate expiry for {{ $labels.instance }} is below 21 days." - alert: HomelabCertificateExpiresCritical expr: (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) <= 7 * 24 * 3600 for: 15m labels: severity: critical annotations: summary: "TLS certificate is close to expiry for {{ $labels.instance }}" description: "The earliest certificate expiry for {{ $labels.instance }} is at or below 7 days, or already expired." - name: homelab-host rules: - alert: HomelabDiskAlmostFull expr: 100 * (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) > 85 for: 10m labels: severity: warning annotations: summary: "Disk usage high on {{ $labels.mountpoint }}" description: "{{ $labels.mountpoint }} is above 85% used." - alert: HomelabDiskCritical expr: 100 * (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) > 95 for: 5m labels: severity: critical annotations: summary: "Disk critically full on {{ $labels.mountpoint }}" description: "{{ $labels.mountpoint }} is above 95% used. Writes may start to fail (DB, appdata, cache)." - alert: HomelabHighMemoryUsage expr: 100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 90 for: 10m labels: severity: warning annotations: summary: "Memory usage high" description: "Host memory usage is above 90%." - alert: HomelabTraefik5xx expr: sum(increase(traefik_service_requests_total{code=~"5.."}[5m])) by (service) >= 5 for: 2m labels: severity: warning annotations: summary: "Traefik 5xx responses for {{ $labels.service }}" description: "Traefik reports at least 5 5xx responses for {{ $labels.service }} within 5 minutes." - name: homelab-backup-and-containers rules: - alert: HomelabTextfileExporterStale expr: time() - homelab_textfile_exporter_last_run_timestamp_seconds > 2 * 60 * 60 for: 15m labels: severity: warning annotations: summary: "Homelab textfile metrics are stale" description: "The host textfile exporter has not refreshed metrics for more than 2 hours." - alert: HomelabBorgMetricsMissing expr: absent(homelab_borg_last_completed_timestamp_seconds) for: 15m labels: severity: critical annotations: summary: "Borg backup metrics are missing" description: "Prometheus cannot see the homelab_borg_last_completed_timestamp_seconds metric." - alert: HomelabBorgBackupStale expr: time() - homelab_borg_last_completed_timestamp_seconds > 30 * 60 * 60 for: 15m labels: severity: warning annotations: summary: "Borg backup is stale" description: "The latest completed Borg backup is older than 30 hours." - alert: HomelabBorgLastJobFailed expr: homelab_borg_last_success != 1 for: 15m labels: severity: critical annotations: summary: "Latest Borg backup did not complete successfully" description: "The latest Borg UI job status is {{ $labels.status }} for archive {{ $labels.archive }}." - alert: HomelabBorgLastJobCompletedWithWarnings expr: homelab_borg_last_job_warning == 1 for: 15m labels: severity: warning annotations: summary: "Latest Borg backup completed with warnings" description: "The latest Borg UI job completed with warnings for archive {{ $labels.archive }}." - alert: HomelabBorgScopeSourceListMissing expr: homelab_borg_scope_expected_file_present != 1 for: 15m labels: severity: critical annotations: summary: "Borg expected source list is not visible" description: "Borg UI cannot see the repo source list used for drift checks." - alert: HomelabBorgScopeMissingSources expr: homelab_borg_scope_missing_sources_total > 0 for: 15m labels: severity: critical annotations: summary: "Borg UI is missing expected backup sources" description: "Borg UI is missing {{ $value }} source path(s) from ops/borg-ui/all-important-sources.txt." - alert: HomelabBorgScopeExtraSources expr: homelab_borg_scope_extra_sources_total > 0 for: 30m labels: severity: warning annotations: summary: "Borg UI has sources not tracked in the repo" description: "Borg UI has {{ $value }} source path(s) that are not listed in ops/borg-ui/all-important-sources.txt." - alert: HomelabBorgDumpMissing expr: homelab_borg_dump_present == 0 for: 15m labels: severity: critical annotations: summary: "Borg pre-backup dump is missing: {{ $labels.dump }}" description: "Expected dump artifact {{ $labels.dump }} is not present in the latest dump set. The pre-backup dump job may have failed or stopped." - alert: HomelabBorgDumpStale expr: homelab_borg_dump_age_seconds > 30 * 60 * 60 for: 15m labels: severity: critical annotations: summary: "Borg pre-backup dump is stale: {{ $labels.dump }}" description: "Dump artifact {{ $labels.dump }} is older than 30 hours. pre-backup-dumps.sh may have stopped; Borg would keep archiving stale database content without a job failure." - alert: HomelabBorgRepositoryCheckStale expr: time() - homelab_borg_repository_last_check_timestamp_seconds > 14 * 24 * 60 * 60 for: 30m labels: severity: warning annotations: summary: "Borg repository check is stale" description: "Borg repository {{ $labels.repository }} has not had a recorded check for more than 14 days." - alert: HomelabBorgRetentionDisabled expr: homelab_borg_schedule_prune_after_enabled != 1 for: 30m labels: severity: warning annotations: summary: "Borg retention pruning is disabled" description: "Scheduled Borg job {{ $labels.schedule }} does not run prune after backup." - alert: HomelabBorgCompactDisabled expr: homelab_borg_schedule_compact_after_enabled != 1 for: 30m labels: severity: warning annotations: summary: "Borg compaction is disabled" description: "Scheduled Borg job {{ $labels.schedule }} does not run compact after backup." - alert: HomelabCriticalContainerDown expr: homelab_critical_container_running == 0 for: 5m labels: severity: critical annotations: summary: "Critical container is down: {{ $labels.name }}" description: "The host textfile exporter reports that critical container {{ $labels.name }} is not running." - alert: HomelabGitOpsRuntimeImageDrift expr: homelab_gitops_runtime_image_match == 0 for: 10m labels: severity: warning annotations: summary: "Runtime image drift: {{ $labels.name }}" description: "Container {{ $labels.name }} is not running the image declared by its Compose config in project {{ $labels.project }}." - name: homelab-meta rules: - alert: HomelabPrometheusTargetDown expr: up == 0 for: 5m labels: severity: critical annotations: summary: "Prometheus target down: {{ $labels.job }} / {{ $labels.instance }}" description: "Scrape target {{ $labels.instance }} (job {{ $labels.job }}) is unreachable. Metrics from this target are silent — alerts built on them will not fire."