homelab-infra/monitoring/prometheus/alerts.yml

groups:
  - name: homelab-availability
    rules:
      - alert: HomelabExternalConnectivityDown
        expr: sum(probe_success{job="blackbox-http"} == 0) >= 5
        for: 8m
        labels:
          severity: warning
        annotations:
          summary: "External connectivity appears down"
          description: "At least 5 public homelab endpoints are unreachable. Likely WAN, DNS, or provider issue."

      - alert: HomelabEndpointDown
        expr: (probe_success{job="blackbox-http"} == 0) unless on() (sum(probe_success{job="blackbox-http"} == 0) >= 5)
        for: 8m
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.instance }} is not reachable"
          description: "Blackbox probe failed for {{ $labels.instance }}."

      - alert: HomelabEndpointSlow
        expr: probe_duration_seconds{job="blackbox-http"} > 5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.instance }} is slow"
          description: "Blackbox probe duration is above 5 seconds for {{ $labels.instance }}."

      - alert: HomelabCertificateExpiresSoon
        expr: (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) < 21 * 24 * 3600 and (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) > 7 * 24 * 3600
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "TLS certificate expires soon for {{ $labels.instance }}"
          description: "The earliest certificate expiry for {{ $labels.instance }} is below 21 days."

      - alert: HomelabCertificateExpiresCritical
        expr: (probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time()) <= 7 * 24 * 3600
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "TLS certificate is close to expiry for {{ $labels.instance }}"
          description: "The earliest certificate expiry for {{ $labels.instance }} is at or below 7 days, or already expired."

  - name: homelab-host
    rules:
      - alert: HomelabDiskAlmostFull
        expr: 100 * (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) > 85
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Disk usage high on {{ $labels.mountpoint }}"
          description: "{{ $labels.mountpoint }} is above 85% used."

      - alert: HomelabDiskCritical
        expr: 100 * (1 - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"}) > 95
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Disk critically full on {{ $labels.mountpoint }}"
          description: "{{ $labels.mountpoint }} is above 95% used. Writes may start to fail (DB, appdata, cache)."

      - alert: HomelabHighMemoryUsage
        expr: 100 * (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) > 90
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Memory usage high"
          description: "Host memory usage is above 90%."

      - alert: HomelabTraefik5xx
        expr: sum(increase(traefik_service_requests_total{code=~"5.."}[5m])) by (service) >= 5
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Traefik 5xx responses for {{ $labels.service }}"
          description: "Traefik reports at least 5 5xx responses for {{ $labels.service }} within 5 minutes."

  - name: homelab-backup-and-containers
    rules:
      - alert: HomelabTextfileExporterStale
        expr: time() - homelab_textfile_exporter_last_run_timestamp_seconds > 2 * 60 * 60
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Homelab textfile metrics are stale"
          description: "The host textfile exporter has not refreshed metrics for more than 2 hours."

      - alert: HomelabBorgMetricsMissing
        expr: absent(homelab_borg_last_completed_timestamp_seconds)
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Borg backup metrics are missing"
          description: "Prometheus cannot see the homelab_borg_last_completed_timestamp_seconds metric."

      - alert: HomelabBorgBackupStale
        expr: time() - homelab_borg_last_completed_timestamp_seconds > 30 * 60 * 60
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Borg backup is stale"
          description: "The latest completed Borg backup is older than 30 hours."

      - alert: HomelabBorgLastJobFailed
        expr: homelab_borg_last_success != 1
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Latest Borg backup did not complete successfully"
          description: "The latest Borg UI job status is {{ $labels.status }} for archive {{ $labels.archive }}."

      - alert: HomelabBorgLastJobCompletedWithWarnings
        expr: homelab_borg_last_job_warning == 1
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: "Latest Borg backup completed with warnings"
          description: "The latest Borg UI job completed with warnings for archive {{ $labels.archive }}."

      - alert: HomelabBorgScopeSourceListMissing
        expr: homelab_borg_scope_expected_file_present != 1
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Borg expected source list is not visible"
          description: "Borg UI cannot see the repo source list used for drift checks."

      - alert: HomelabBorgScopeMissingSources
        expr: homelab_borg_scope_missing_sources_total > 0
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Borg UI is missing expected backup sources"
          description: "Borg UI is missing {{ $value }} source path(s) from ops/borg-ui/all-important-sources.txt."

      - alert: HomelabBorgScopeExtraSources
        expr: homelab_borg_scope_extra_sources_total > 0
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Borg UI has sources not tracked in the repo"
          description: "Borg UI has {{ $value }} source path(s) that are not listed in ops/borg-ui/all-important-sources.txt."

      - alert: HomelabBorgDumpMissing
        expr: homelab_borg_dump_present == 0
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Borg pre-backup dump is missing: {{ $labels.dump }}"
          description: "Expected dump artifact {{ $labels.dump }} is not present in the latest dump set. The pre-backup dump job may have failed or stopped."

      - alert: HomelabBorgDumpStale
        expr: homelab_borg_dump_age_seconds > 30 * 60 * 60
        for: 15m
        labels:
          severity: critical
        annotations:
          summary: "Borg pre-backup dump is stale: {{ $labels.dump }}"
          description: "Dump artifact {{ $labels.dump }} is older than 30 hours. pre-backup-dumps.sh may have stopped; Borg would keep archiving stale database content without a job failure."

      - alert: HomelabBorgRepositoryCheckStale
        expr: time() - homelab_borg_repository_last_check_timestamp_seconds > 14 * 24 * 60 * 60
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Borg repository check is stale"
          description: "Borg repository {{ $labels.repository }} has not had a recorded check for more than 14 days."

      - alert: HomelabBorgRetentionDisabled
        expr: homelab_borg_schedule_prune_after_enabled != 1
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Borg retention pruning is disabled"
          description: "Scheduled Borg job {{ $labels.schedule }} does not run prune after backup."

      - alert: HomelabBorgCompactDisabled
        expr: homelab_borg_schedule_compact_after_enabled != 1
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Borg compaction is disabled"
          description: "Scheduled Borg job {{ $labels.schedule }} does not run compact after backup."

      - alert: HomelabCriticalContainerDown
        expr: homelab_critical_container_running == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Critical container is down: {{ $labels.name }}"
          description: "The host textfile exporter reports that critical container {{ $labels.name }} is not running."

      - alert: HomelabGitOpsRuntimeImageDrift
        expr: homelab_gitops_runtime_image_match == 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Runtime image drift: {{ $labels.name }}"
          description: "Container {{ $labels.name }} is not running the image declared by its Compose config in project {{ $labels.project }}."

  - name: homelab-meta
    rules:
      - alert: HomelabPrometheusTargetDown
        expr: up == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Prometheus target down: {{ $labels.job }} / {{ $labels.instance }}"
          description: "Scrape target {{ $labels.instance }} (job {{ $labels.job }}) is unreachable. Metrics from this target are silent — alerts built on them will not fire."