diff --git a/docs/DISASTER_RECOVERY.md b/docs/DISASTER_RECOVERY.md index e732560..ae3f123 100644 --- a/docs/DISASTER_RECOVERY.md +++ b/docs/DISASTER_RECOVERY.md @@ -61,7 +61,7 @@ Diese Punkte sollten **vor** einem echten Ausfall geklaert sein: | Thema | Sollzustand | |---|---| | Repo-Zugang ausserhalb von Gitea | privater GitHub-Push-Mirror `michaelkaleschke-spec/homelab-infra` und lokaler aktueller Clone vorhanden | -| Unraid USB-/Flash-Backup | eingerichtet und wiederherstellbar | +| Unraid USB-/Flash-Backup | `unraid-flash-config.tar.gz` wird vor Borg unter `/mnt/user/backups/borg/dumps/latest` erzeugt und nach Hetzner/Borg gesichert; Unraid-Connect-Cloud-Backup optional zusaetzlich | | Borg-Ziel | nicht nur lokal auf demselben Ausfallpfad | | Borg-Passphrase | Host-Secret-Datei vorhanden und fuer Borg-Zugriff verifiziert; externe analoge Hinterlegung bleibt Operator-Aufgabe | | Secrets-Dateien | ueber Borg bzw. Restore-Quellen abgedeckt | @@ -100,6 +100,8 @@ Wenn **weder GitHub-Mirror noch lokaler Repo-Clone** verfuegbar sind, ist `servi 3. Array wieder zuweisen und starten 4. Grundlegende Shares pruefen +Primaere lokale/off-site Restore-Quelle fuer die bestehende Flash-Konfiguration ist das Borg-Artefakt `unraid-flash-config.tar.gz` aus `/mnt/user/backups/borg/dumps/latest`. Dieses Archiv enthaelt `/boot/config` und muss wie Secret-Material behandelt werden. + ### 5.2 Erwartete Shares / Pfade Mindestens diese Pfade muessen wieder verfuegbar sein: @@ -207,6 +209,7 @@ Besonders kritisch: - `/mnt/user/appdata/komodo/core` - `/mnt/user/appdata/komodo/periphery` - `/mnt/user/backups/borg/dumps/latest` +- `/mnt/user/backups/borg/dumps/latest/unraid-flash-config.tar.gz` - dienstspezifische App- und Nutzdatenpfade **Nicht blind alles extrahieren**, wenn nur einzelne Pfade oder Dienste betroffen sind. @@ -368,6 +371,7 @@ Relevant: - Dump-Ziel: `/mnt/user/backups/borg/dumps/latest` - Skript: `ops/borg-ui/scripts/pre-backup-dumps.sh` +- Unraid-Flash-Artefakt: `unraid-flash-config.tar.gz` plus `.sha256` und Manifest im selben Zielpfad ### Hermes Agent @@ -390,7 +394,7 @@ Smoke-Test: `hermes-gateway` healthcheck ist gruen, `hermes.kaleschke.info` leit ## 11. Offene Vorbereitungs-To-dos -- Unraid-USB-/Flash-Backup pruefen +- Unraid-USB-/Flash-Backup regelmaessig ueber `unraid-flash-config.tar.gz` und optional Unraid Connect pruefen - Borg-Passphrase aus `/mnt/user/appdata/secrets/borg_repo_passphrase.txt` extern analog sicher hinterlegen - Komodo Stack-ENV-Werte zentral ausserhalb von Komodo dokumentieren - regelmaessige automatisierte Restore-Smoke-Tests fuer Vaultwarden, Gitea und Paperless etablieren diff --git a/docs/MIGRATION_LOG.md b/docs/MIGRATION_LOG.md index da5875f..813245b 100644 --- a/docs/MIGRATION_LOG.md +++ b/docs/MIGRATION_LOG.md @@ -16,6 +16,11 @@ Dieses Dokument ist nur noch ein historischer Verlauf. Der aktuelle operative Ab ## Historische Meilensteine +### 2026-05-25 - Unraid Flash-Backup in Borg-Scope aufgenommen + +- `pre-backup-dumps.sh` erzeugt zusaetzlich zu den DB-Dumps ein sensibles `unraid-flash-config.tar.gz` aus `/boot/config` inklusive SHA256 und Manifest unter `/mnt/user/backups/borg/dumps/latest`. +- Da `/local/borg-dumps` bereits Teil des Borg-Scopes ist, wird das Flash-Konfigurationsartefakt mit dem bestehenden Hetzner/Borg-Backup historisiert. Downloadbare Plugin-Paketarchive unter `/boot/config/plugins/*/` werden aus dem Artefakt ausgeschlossen; Restore-relevante Konfiguration bleibt enthalten. + ### 2026-05-25 - Monitoring-Zielstack finalisiert und Uptime Kuma entfernt - `monitoring` und `glance` wurden auf Commit `b6bbca4` deployed; Komodo zeigt fuer beide `latest_hash` = `deployed_hash` = `b6bbca4` ohne `remote_errors`. Die zehn `monitoring-*` Container laufen, `monitoring.kaleschke.info` und `glance.kaleschke.info` leiten anonym zu Authelia, Prometheus ist ready und Loki `/ready` liefert `ready`. diff --git a/docs/RESTORE_MATRIX.md b/docs/RESTORE_MATRIX.md index 27f3a2f..bb8a142 100644 --- a/docs/RESTORE_MATRIX.md +++ b/docs/RESTORE_MATRIX.md @@ -26,6 +26,7 @@ Sie ist die fachliche Ergaenzung zu `docs/DISASTER_RECOVERY.md`. | Dienst | Fuehrende Quelle | Datei-Restore | Dump / DB | Secrets / ENV | Abhaengigkeiten | Smoke-Test | |---|---|---|---|---|---|---| +| Unraid OS Flash | Borg-Artefakt + optional Unraid Connect | `/boot/config` aus `unraid-flash-config.tar.gz` | `unraid-flash-config.tar.gz`, `.sha256`, Manifest | enthaelt sensible Host-Konfiguration, wie Secret-Material behandeln | Unraid USB Flash Creator / neuer Boot-Stick | Unraid bootet, Array-Zuordnung und Shares sind sichtbar | | Traefik | Share / Borg | `/mnt/user/appdata/traefik`, besonders `dynamic/`, `letsencrypt`, `secrets` | keine eigene DB | `cloudflare_dns_api_token` | `frontend_net`, `backend_net` | `https://traefik.kaleschke.info` erreichbar, Dashboard ueber Authelia | | AdGuard Home | Share / Borg | `/mnt/user/appdata/adguard/conf` | keine | keine zusaetzlichen Repo-Secrets dokumentiert | `dns_net`, `frontend_net` | DNS-Aufloesung funktioniert | | Tailscale | Share / Borg | `/mnt/user/appdata/tailscale` | keine | Tailscale-State im Pfad | Host-Netz | Tailscale verbunden | @@ -90,6 +91,7 @@ Aktuell relevante Dump-Artefakte unter `/mnt/user/backups/borg/dumps/latest`: - `filebrowser.bolt.dump` - `borg-ui.sqlite` - `grafana.sqlite` +- `unraid-flash-config.tar.gz` plus `unraid-flash-config.tar.gz.sha256` und Manifest - Monitoring-Stack: keine verpflichtenden Dump-Artefakte; Prometheus/Loki/Grafana named volumes sind Diagnose-/Dashboard-Zustand, keine primaere Restore-Quelle. - `komodo-mongo.archive.gz` (noch gesondert verifizieren) diff --git a/docs/SECRETS_MAP.md b/docs/SECRETS_MAP.md index f82cb10..ed72bf4 100644 --- a/docs/SECRETS_MAP.md +++ b/docs/SECRETS_MAP.md @@ -46,6 +46,7 @@ Dieses Dokument listet sensible Daten, deren Ablageorte und die vorgesehene Einb | nextcloud-postgres | DB Password | `/mnt/user/appdata/secrets/nextcloud_postgres_password.txt` -> `POSTGRES_PASSWORD_FILE` | neu | | Borg UI / Borg | Admin-Login, `SECRET_KEY`, SSH-Keys, Repo-Credentials | persistent unter `/mnt/user/appdata/borg-ui/data/` | aktiv | | Borg Repo | Borg-Passphrase fuer Restore-Tests und Notfallzugriff | `/mnt/user/appdata/secrets/borg_repo_passphrase.txt` -> Host-Secret-Datei, nicht im Repo | aktiv | +| Unraid Flash Backup | Boot-/Array-/Share-/Plugin-Konfiguration, ggf. Hashes/Keys/Templates | `/mnt/user/backups/borg/dumps/latest/unraid-flash-config.tar.gz`, via Borg/Hetzner gesichert | aktiv; wie Secret-Material behandeln | | Hermes Agent | Provider-Keys, Bot-Tokens, API-Server-Key | `/mnt/user/appdata/hermes-agent/data/.env` | VM-seitig offen | | Hermes Agent | SSH-Runner Private Key | `/mnt/user/appdata/secrets/hermes_runner_id_ed25519` -> `/root/.ssh/id_ed25519` | VM-seitig offen | | Grafana | Admin Password | `/mnt/user/appdata/secrets/grafana_admin_password.txt` -> `GF_SECURITY_ADMIN_PASSWORD__FILE` | aktiv | diff --git a/ops/borg-ui/BACKUP_SCOPE.md b/ops/borg-ui/BACKUP_SCOPE.md index 2bee77b..15b77a2 100644 --- a/ops/borg-ui/BACKUP_SCOPE.md +++ b/ops/borg-ui/BACKUP_SCOPE.md @@ -11,16 +11,18 @@ Use Borg as the single backup system for: - critical file-backed application data - secrets, keys, and reverse-proxy state - database dumps generated before each Borg backup +- Unraid flash configuration artifacts generated before each Borg backup Do not back up raw live database storage directories as the primary recovery artifact. ## Strategy -1. A pre-backup dump script runs on the host and writes fresh dumps to `/mnt/user/backups/borg/dumps/latest`. +1. A pre-backup dump script runs on the host and writes fresh dumps plus `unraid-flash-config.tar.gz` to `/mnt/user/backups/borg/dumps/latest`. 2. Borg backs up `/local/borg-dumps` plus the critical mounted paths below. 3. Borg retention handles history; the dump directory itself keeps only the latest artifacts. The inclusion of `/local/secrets` is intentional: Borg is expected to cover disaster recovery for selected secret material as part of the current homelab restore strategy. +The Unraid flash configuration archive is intentional as well and must be treated as secret backup material. ## Service Inventory @@ -41,6 +43,7 @@ The inclusion of `/local/secrets` is intentional: Borg is expected to cover disa | Borg UI | SQLite dump + self-backup | `/local/borg-dumps`, `/local/appdata/borg-ui/data` | | Komodo | config + Mongo dump | `/local/borg-dumps`, `/local/appdata/komodo/periphery`, `/local/appdata/komodo/core` | | GitOps host automation | repo clone + Komodo workspaces + host-check state | `/local/services/homelab-infra`, `/local/services/stacks`, `/local/services/posture-check` | +| Unraid OS flash | generated config archive | `/local/borg-dumps/unraid-flash-config.tar.gz` plus checksum and manifest | | Nextcloud | DB dump + file data | `/local/borg-dumps`, `/local/appdata/nextcloud/html`, `/local/nextcloud/data` | | Grafana | SQLite dump + file data | `/local/borg-dumps`, `/local/appdata/grafana` | | Filebrowser | file-backed state dump + file data | `/local/borg-dumps`, `/local/appdata/filebrowser` | @@ -83,6 +86,7 @@ The live Unraid User Scripts execute repo scripts from `/mnt/user/services/homel - Komodo MongoDB - SQLite: `gitea`, `vaultwarden`, `speedtest-tracker`, `borg-ui`, `grafana` - File-backed state: `filebrowser.bolt.dump` +- Unraid flash config: `unraid-flash-config.tar.gz` plus `unraid-flash-config.tar.gz.sha256` ## Explicitly Not Backed Up as Raw Live DB Files diff --git a/ops/borg-ui/scripts/README.md b/ops/borg-ui/scripts/README.md index 7cd2bc0..506d44b 100644 --- a/ops/borg-ui/scripts/README.md +++ b/ops/borg-ui/scripts/README.md @@ -14,6 +14,10 @@ Fresh dump artifacts are written to: Borg UI should include `/local/borg-dumps` as a backup source. +The dump set also includes `unraid-flash-config.tar.gz`, a host-generated +archive of `/boot/config` plus checksum and manifest. Treat this archive as +secret backup material. + ## Notes - The script is written for host execution where `docker` is available. diff --git a/ops/borg-ui/scripts/USER_SCRIPTS_SETUP.md b/ops/borg-ui/scripts/USER_SCRIPTS_SETUP.md index 7d57469..0151bf1 100644 --- a/ops/borg-ui/scripts/USER_SCRIPTS_SETUP.md +++ b/ops/borg-ui/scripts/USER_SCRIPTS_SETUP.md @@ -17,7 +17,7 @@ It should **not** be implemented as a Borg UI inline hook in the current design. `pre-borg.sh` currently chains the host-side checks: - `services/posture-check/posture-check.sh` -- `ops/borg-ui/scripts/pre-backup-dumps.sh` +- `ops/borg-ui/scripts/pre-backup-dumps.sh` including the Unraid flash config archive - `ops/restore-tests/check-restore-freshness.sh` The dump step assumes: @@ -56,9 +56,10 @@ The intended sequence is: 1. Host wrapper checks posture. 2. Host script refreshes `latest` dump artifacts. -3. Freshness check verifies expected dumps. -4. Borg UI backs up `/local/borg-dumps` together with the rest of `critical_infra`. -5. Borg history preserves dump history, so the host only needs to keep the most recent dump set. +3. Host script writes `unraid-flash-config.tar.gz` plus checksum and manifest into the same dump set. +4. Freshness check verifies expected dumps and the flash config archive. +5. Borg UI backs up `/local/borg-dumps` together with the rest of `critical_infra`. +6. Borg history preserves dump history, so the host only needs to keep the most recent dump set. ## Current dump target diff --git a/ops/borg-ui/scripts/pre-backup-dumps.sh b/ops/borg-ui/scripts/pre-backup-dumps.sh index bd70453..ffb0543 100755 --- a/ops/borg-ui/scripts/pre-backup-dumps.sh +++ b/ops/borg-ui/scripts/pre-backup-dumps.sh @@ -155,6 +155,56 @@ dump_file_copy() { atomic_write "$output" "$tmp" } +backup_unraid_flash_config() { + output="$LATEST_DIR/unraid-flash-config.tar.gz" + checksum="$LATEST_DIR/unraid-flash-config.tar.gz.sha256" + manifest="$LATEST_DIR/unraid-flash-config.manifest.txt" + tmp="$TMP_DIR/unraid-flash-config.tar.gz.tmp" + tmp_checksum="$TMP_DIR/unraid-flash-config.tar.gz.sha256.tmp" + tmp_manifest="$TMP_DIR/unraid-flash-config.manifest.txt.tmp" + + if [ ! -d /boot/config ]; then + warn "Skipping Unraid flash config backup because /boot/config is missing" + return 1 + fi + + log "Backing up Unraid flash configuration from /boot/config" + rm -f "$tmp" "$tmp_checksum" "$tmp_manifest" + + tar -C /boot \ + --exclude='config/plugins/*/*.txz' \ + --exclude='config/plugins/*/*.tgz' \ + --exclude='config/plugins/*/*.tar' \ + --exclude='config/plugins/*/*.tar.*' \ + --exclude='config/plugins/*/*.zip' \ + --exclude='config/plugins/*/*.md5' \ + -czf "$tmp" config + chmod 600 "$tmp" + atomic_write "$output" "$tmp" + + ( + cd "$LATEST_DIR" + sha256sum "$(basename "$output")" + ) > "$tmp_checksum" + chmod 600 "$tmp_checksum" + atomic_write "$checksum" "$tmp_checksum" + + { + printf 'created_utc=%s\n' "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + printf 'host=%s\n' "$(hostname)" + if [ -f /etc/unraid-version ]; then + sed 's/^/unraid_/' /etc/unraid-version + fi + printf 'source=/boot/config\n' + printf 'archive=%s\n' "$(basename "$output")" + printf 'checksum=%s\n' "$(basename "$checksum")" + printf 'note=%s\n' 'Contains Unraid configuration and must be treated as secret backup material.' + printf 'excluded=%s\n' 'downloadable plugin package archives under /boot/config/plugins/*/' + } > "$tmp_manifest" + chmod 600 "$tmp_manifest" + atomic_write "$manifest" "$tmp_manifest" +} + dump_optional_pg_db() { container="$1" password="$2" @@ -219,6 +269,8 @@ dump_mongo_container() { main() { need_cmd docker need_cmd sqlite3 + need_cmd tar + need_cmd sha256sum ensure_dirs # Shared PostgreSQL 17 @@ -272,6 +324,10 @@ main() { # MongoDB dump_mongo_container "komodo-mongo" "$LATEST_DIR/komodo-mongo.archive.gz" + # Unraid USB flash configuration. This is generated into the existing dump + # set so Borg carries it off-site together with the database artifacts. + backup_unraid_flash_config + log "Finished refreshing dump set in $LATEST_DIR" } diff --git a/ops/restore-tests/check-restore-freshness.ps1 b/ops/restore-tests/check-restore-freshness.ps1 index 790db4e..dbfcd75 100644 --- a/ops/restore-tests/check-restore-freshness.ps1 +++ b/ops/restore-tests/check-restore-freshness.ps1 @@ -14,7 +14,8 @@ $checks = @( @{ Name = "gitea.sqlite.dump"; Path = Join-Path $DumpRoot "gitea.sqlite.dump" }, @{ Name = "vaultwarden.sqlite.dump"; Path = Join-Path $DumpRoot "vaultwarden.sqlite.dump" }, @{ Name = "speedtest-tracker.sqlite.dump"; Path = Join-Path $DumpRoot "speedtest-tracker.sqlite.dump" }, - @{ Name = "filebrowser.bolt.dump"; Path = Join-Path $DumpRoot "filebrowser.bolt.dump" } + @{ Name = "filebrowser.bolt.dump"; Path = Join-Path $DumpRoot "filebrowser.bolt.dump" }, + @{ Name = "unraid-flash-config.tar.gz"; Path = Join-Path $DumpRoot "unraid-flash-config.tar.gz" } ) $reportChecks = @( diff --git a/ops/restore-tests/check-restore-freshness.sh b/ops/restore-tests/check-restore-freshness.sh index af3c90b..0e42fcb 100755 --- a/ops/restore-tests/check-restore-freshness.sh +++ b/ops/restore-tests/check-restore-freshness.sh @@ -34,7 +34,8 @@ for dump in \ gitea.sqlite.dump \ vaultwarden.sqlite.dump \ speedtest-tracker.sqlite.dump \ - filebrowser.bolt.dump; do + filebrowser.bolt.dump \ + unraid-flash-config.tar.gz; do path="$DUMP_ROOT/$dump" if [ ! -f "$path" ]; then critical+=("DUMP_MISSING $dump")