#!/usr/bin/env bash set -euo pipefail TEXTFILE_DIR="${TEXTFILE_DIR:-/mnt/user/services/posture-check/textfile}" OUTPUT_FILE="${OUTPUT_FILE:-$TEXTFILE_DIR/homelab.prom}" BORG_CONTAINER="${BORG_CONTAINER:-borg-ui}" BORG_EXPECTED_SOURCES_FILE="${BORG_EXPECTED_SOURCES_FILE:-/local/services/homelab-infra/ops/borg-ui/all-important-sources.txt}" # Host-Pfad der aktuellen Dump-Artefakte (pre-backup-dumps.sh schreibt hierhin). # Wird host-seitig gestattet; der Exporter laeuft als Unraid User Script. BORG_DUMP_DIR="${BORG_DUMP_DIR:-/mnt/user/backups/borg/dumps/latest}" CRITICAL_CONTAINERS="${CRITICAL_CONTAINERS:-traefik authelia postgresql17 gitea komodo-core komodo-mongo komodo-periphery vaultwarden borg-ui ntfy adguard unbound monitoring-alertmanager monitoring-alertmanager-ntfy-bridge monitoring-blackbox-exporter monitoring-cadvisor monitoring-grafana monitoring-loki monitoring-node-exporter monitoring-promtail immich_server immich_postgres immich_redis paperless-ngx nextcloud nextcloud-postgres nextcloud-redis mealie mealie-postgres mail-archiver n8n homeassistant smarthome-mosquitto}" # Hinweis: Tailscale laeuft als natives Unraid-Plugin (kein Docker-Container) und # wird daher hier bewusst NICHT als kritischer Container gefuehrt (Stand 2026-06-06). mkdir -p "$TEXTFILE_DIR" tmp="$(mktemp "$TEXTFILE_DIR/homelab.prom.XXXXXX")" cleanup() { rm -f "$tmp" } trap cleanup EXIT now="$(date +%s)" emit_gitops_runtime_image_metrics() { cat <<'EOF' # HELP homelab_gitops_runtime_image_match Whether a running Compose container uses the image currently declared by its Compose config. # TYPE homelab_gitops_runtime_image_match gauge EOF if ! command -v jq >/dev/null 2>&1; then return fi docker ps \ --filter label=com.docker.compose.project \ --format '{{.Names}}\t{{.Label "com.docker.compose.project"}}\t{{.Label "com.docker.compose.service"}}\t{{.Label "com.docker.compose.project.config_files"}}\t{{.Label "com.docker.compose.project.environment_file"}}' | while IFS="$(printf '\t')" read -r container project service config_files env_file; do [ -n "$container" ] || continue [ -n "$service" ] || continue [ -n "$config_files" ] || continue config_file="${config_files%%,*}" [ -f "$config_file" ] || continue compose_args=(-f "$config_file") if [ -n "$env_file" ] && [ -f "$env_file" ]; then compose_args+=(--env-file "$env_file") fi expected="$( docker compose "${compose_args[@]}" config --format json 2>/dev/null | jq -r --arg service "$service" '.services[$service].image // empty' 2>/dev/null || true )" [ -n "$expected" ] || continue running="$(docker inspect -f '{{.Config.Image}}' "$container" 2>/dev/null || true)" [ -n "$running" ] || continue match="0" if [ "$running" = "$expected" ]; then match="1" fi printf 'homelab_gitops_runtime_image_match{name="%s",project="%s",service="%s"} %s\n' \ "$container" "$project" "$service" "$match" done } { cat <<'EOF' # HELP homelab_textfile_exporter_last_run_timestamp_seconds Unix timestamp of the last successful homelab textfile exporter run. # TYPE homelab_textfile_exporter_last_run_timestamp_seconds gauge EOF printf 'homelab_textfile_exporter_last_run_timestamp_seconds %s\n' "$now" cat <<'EOF' # HELP homelab_critical_container_running Whether a critical container is currently running according to docker inspect. # TYPE homelab_critical_container_running gauge EOF for container in $CRITICAL_CONTAINERS; do running="0" if docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null | grep -qx true; then running="1" fi printf 'homelab_critical_container_running{name="%s"} %s\n' "$container" "$running" done emit_gitops_runtime_image_metrics cat <<'EOF' # HELP homelab_borg_last_completed_timestamp_seconds Unix timestamp of the most recent completed Borg backup job known to Borg UI. # TYPE homelab_borg_last_completed_timestamp_seconds gauge # HELP homelab_borg_last_success Whether the most recent Borg backup job completed successfully. # TYPE homelab_borg_last_success gauge # HELP homelab_borg_last_job_warning Whether the most recent Borg backup job completed with warnings. # TYPE homelab_borg_last_job_warning gauge # HELP homelab_borg_repository_last_check_timestamp_seconds Unix timestamp of the latest Borg repository check known to Borg UI. # TYPE homelab_borg_repository_last_check_timestamp_seconds gauge # HELP homelab_borg_scope_expected_file_present Whether the expected Borg source list file is visible inside Borg UI. # TYPE homelab_borg_scope_expected_file_present gauge # HELP homelab_borg_scope_expected_sources_total Number of expected Borg source paths from the repo source list. # TYPE homelab_borg_scope_expected_sources_total gauge # HELP homelab_borg_scope_configured_sources_total Number of Borg source paths configured in Borg UI. # TYPE homelab_borg_scope_configured_sources_total gauge # HELP homelab_borg_scope_missing_sources_total Number of expected Borg source paths missing from Borg UI. # TYPE homelab_borg_scope_missing_sources_total gauge # HELP homelab_borg_scope_extra_sources_total Number of Borg UI source paths not present in the repo source list. # TYPE homelab_borg_scope_extra_sources_total gauge # HELP homelab_borg_scope_source_configured Whether an expected Borg source path is configured in Borg UI. # TYPE homelab_borg_scope_source_configured gauge # HELP homelab_borg_schedule_prune_after_enabled Whether a Borg scheduled job runs prune after backup. # TYPE homelab_borg_schedule_prune_after_enabled gauge # HELP homelab_borg_schedule_compact_after_enabled Whether a Borg scheduled job runs compact after backup. # TYPE homelab_borg_schedule_compact_after_enabled gauge EOF if docker inspect "$BORG_CONTAINER" >/dev/null 2>&1; then docker exec -i -e BORG_EXPECTED_SOURCES_FILE="$BORG_EXPECTED_SOURCES_FILE" "$BORG_CONTAINER" python3 - <<'PY' import datetime as dt import json import os from pathlib import Path import sqlite3 conn = sqlite3.connect("/data/borg.db") conn.row_factory = sqlite3.Row cur = conn.cursor() latest = cur.execute(""" select status, completed_at, archive_name from backup_jobs order by coalesce(started_at, created_at) desc limit 1 """).fetchone() completed = cur.execute(""" select completed_at, archive_name from backup_jobs where status in ('completed', 'completed_with_warnings') and completed_at is not null order by completed_at desc limit 1 """).fetchone() def parse_ts(value): if not value: return 0 value = value.replace("Z", "+00:00") try: parsed = dt.datetime.fromisoformat(value) except ValueError: try: parsed = dt.datetime.strptime(value, "%Y-%m-%d %H:%M:%S") except ValueError: return 0 if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=dt.timezone.utc) return int(parsed.timestamp()) def escape_label(value): return (value or "").replace("\\", "\\\\").replace('"', '\\"') def bool_metric(value): return 1 if value else 0 latest_status = latest["status"] if latest else "missing" latest_success = 1 if latest_status in ("completed", "completed_with_warnings") else 0 latest_warning = 1 if latest_status == "completed_with_warnings" else 0 completed_ts = parse_ts(completed["completed_at"]) if completed else 0 latest_archive = escape_label(latest["archive_name"] if latest else "") completed_archive = escape_label(completed["archive_name"] if completed else "") print(f'homelab_borg_last_success{{status="{latest_status}",archive="{latest_archive}"}} {latest_success}') print(f'homelab_borg_last_job_warning{{status="{latest_status}",archive="{latest_archive}"}} {latest_warning}') print(f'homelab_borg_last_completed_timestamp_seconds{{archive="{completed_archive}"}} {completed_ts}') repo = cur.execute(""" select id, name, source_directories, last_check from repositories order by id limit 1 """).fetchone() if repo: repo_name = escape_label(repo["name"] or str(repo["id"])) print(f'homelab_borg_repository_last_check_timestamp_seconds{{repository="{repo_name}"}} {parse_ts(repo["last_check"])}') try: configured_sources = json.loads(repo["source_directories"] or "[]") except json.JSONDecodeError: configured_sources = [] else: configured_sources = [] expected_path = Path(os.environ.get("BORG_EXPECTED_SOURCES_FILE", "")) expected_file_present = expected_path.is_file() if expected_file_present: expected_sources = [ line.strip() for line in expected_path.read_text(encoding="utf-8").splitlines() if line.strip() and not line.lstrip().startswith("#") ] else: expected_sources = [] configured_set = set(configured_sources) expected_set = set(expected_sources) missing_sources = [source for source in expected_sources if source not in configured_set] extra_sources = [source for source in configured_sources if source not in expected_set] print(f"homelab_borg_scope_expected_file_present {bool_metric(expected_file_present)}") print(f"homelab_borg_scope_expected_sources_total {len(expected_sources)}") print(f"homelab_borg_scope_configured_sources_total {len(configured_sources)}") print(f"homelab_borg_scope_missing_sources_total {len(missing_sources)}") print(f"homelab_borg_scope_extra_sources_total {len(extra_sources)}") for source in expected_sources: value = 1 if source in configured_set else 0 print(f'homelab_borg_scope_source_configured{{source="{escape_label(source)}"}} {value}') for source in extra_sources: print(f'homelab_borg_scope_source_configured{{source="{escape_label(source)}",state="extra"}} 0') for schedule in cur.execute(""" select id, name, run_prune_after, run_compact_after from scheduled_jobs where enabled = 1 order by id """): schedule_name = escape_label(schedule["name"] or str(schedule["id"])) print(f'homelab_borg_schedule_prune_after_enabled{{schedule="{schedule_name}"}} {bool_metric(schedule["run_prune_after"])}') print(f'homelab_borg_schedule_compact_after_enabled{{schedule="{schedule_name}"}} {bool_metric(schedule["run_compact_after"])}') PY else printf 'homelab_borg_last_success{status="container_missing",archive=""} 0\n' printf 'homelab_borg_last_job_warning{status="container_missing",archive=""} 0\n' printf 'homelab_borg_last_completed_timestamp_seconds{archive=""} 0\n' printf 'homelab_borg_repository_last_check_timestamp_seconds{repository=""} 0\n' printf 'homelab_borg_scope_expected_file_present 0\n' printf 'homelab_borg_scope_expected_sources_total 0\n' printf 'homelab_borg_scope_configured_sources_total 0\n' printf 'homelab_borg_scope_missing_sources_total 0\n' printf 'homelab_borg_scope_extra_sources_total 0\n' fi # Dump-Frische host-seitig messen. Schliesst den Blindfleck, dass Borg # weiterlaeuft und stale Dumps archiviert, ohne dass ein Job-Fehler entsteht # (pre-backup-dumps.sh gestoppt). Laeuft ausserhalb des borg-ui-Containers, # weil die Dumps host-seitig unter $BORG_DUMP_DIR liegen. cat <<'EOF' # HELP homelab_borg_dump_present Whether an expected Borg pre-backup dump artifact exists in the latest dump set. # TYPE homelab_borg_dump_present gauge # HELP homelab_borg_dump_age_seconds Age in seconds of an expected Borg pre-backup dump artifact. # TYPE homelab_borg_dump_age_seconds gauge EOF for dump in \ postgresql17-globals.sql \ postgresql17-mailarchiver.dump \ postgresql17-paperless.dump \ mealie.dump \ immich.dump \ nextcloud.dump \ gitea.sqlite.dump \ vaultwarden.sqlite.dump \ n8n.sqlite.dump \ unraid-flash-config.tar.gz \ komodo-mongo.archive.gz; do dump_path="$BORG_DUMP_DIR/$dump" if [ -f "$dump_path" ]; then dump_mtime="$(stat -c %Y "$dump_path" 2>/dev/null || echo 0)" printf 'homelab_borg_dump_present{dump="%s"} 1\n' "$dump" printf 'homelab_borg_dump_age_seconds{dump="%s"} %s\n' "$dump" "$(( now - dump_mtime ))" else printf 'homelab_borg_dump_present{dump="%s"} 0\n' "$dump" fi done } > "$tmp" # 0644 statt mktemp-default 0600, damit der node-exporter-Textfile-Collector # (laeuft als nobody:65534) die Datei lesen kann. chmod 644 "$tmp" mv "$tmp" "$OUTPUT_FILE" trap - EXIT printf '%s\n' "$OUTPUT_FILE"