From 479eb291c4999ce42014e3909b345e17987340e3 Mon Sep 17 00:00:00 2001 From: Micha Date: Mon, 1 Jun 2026 12:19:17 +0200 Subject: [PATCH] Prepare final homelab cleanup gates --- .gitattributes | 6 + docs/AI_CONTEXT.md | 2 + docs/AUDIT_2026-05-25_TODO.md | 5 +- docs/SERVICE_CATALOG.md | 4 +- monitoring/README.md | 4 +- .../grafana/dashboards/family-status.json | 295 ++++++++++++++++++ ops/maintenance/release-alt-volumes.sh | 104 ++++++ 7 files changed, 414 insertions(+), 6 deletions(-) create mode 100644 .gitattributes create mode 100644 monitoring/grafana/dashboards/family-status.json create mode 100755 ops/maintenance/release-alt-volumes.sh diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3ee5675 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +*.sh text eol=lf +*.ps1 text eol=crlf +*.md text +*.json text +*.yml text +*.yaml text diff --git a/docs/AI_CONTEXT.md b/docs/AI_CONTEXT.md index c64f478..5e085ed 100644 --- a/docs/AI_CONTEXT.md +++ b/docs/AI_CONTEXT.md @@ -56,3 +56,5 @@ Letzte Bestaetigung: - Borg-Nachlauf 2026-06-01 erfolgreich: Archiv `Taegliche-Sicherung-2026-06-01T04:30:26.913`, Freshness Critical 0 / Warnings 0. - H:/ Nearline-Pull 2026-06-01 repariert: Borg-Dumps werden kuratiert kopiert, Gitea-Bundles aktuell. +- Family-Status-Dashboard liegt als `monitoring/grafana/dashboards/family-status.json` im Repo. +- Alt-Volume-Freigabe ist per `ops/maintenance/release-alt-volumes.sh` vorbereitet; `--execute` nicht vor 2026-06-02. diff --git a/docs/AUDIT_2026-05-25_TODO.md b/docs/AUDIT_2026-05-25_TODO.md index 654ab42..f84bcfa 100644 --- a/docs/AUDIT_2026-05-25_TODO.md +++ b/docs/AUDIT_2026-05-25_TODO.md @@ -7,11 +7,10 @@ Audit-Snapshots wurden aus der Arbeitskopie entfernt; Detailhistorie liegt in Gi | Prioritaet | Punkt | Naechster Schritt | |---|---|---| -| P0 | Alt-Volumes nach Burn-in freigeben | Nicht vor 2026-06-02 loeschen; vorher aktiven Pfad und letzten Borg-Lauf pruefen | +| P0 | Alt-Volumes nach Burn-in freigeben | Ab 2026-06-02 `ops/maintenance/release-alt-volumes.sh --dry-run` pruefen, danach nur bei sauberem Ergebnis mit `--execute` freigeben | | P1 | Hetzner-Account-Hygiene | Starkes einzigartiges Passwort, Backup-Zahlungsweg und Login-Benachrichtigungen extern bestaetigen | | P1 | Borg `--append-only` fuer Hetzner pruefen | Rollback-faehigen Test fuer `borg serve --append-only` in Hetzner `authorized_keys` planen | | P1 | FRITZ!Box-Servicefenster | FRITZ!OS-Update, FRITZ!Box-Konfig-Backup und IPv6-Exposure-Pruefung gemeinsam erledigen | -| P2 | Family-View Dashboard als JSON bauen | Erst nach manuellem Grafana-Layout-Check; Metriken fuer Borg, Certs und Critical-Container sind vorhanden | | P2 | Family-Onboarding praktisch starten | Familienkonten, Vaultwarden-Organisation und Immich-Mobile-Backup gemeinsam einrichten | ## Bewusst geparkt @@ -28,6 +27,8 @@ Audit-Snapshots wurden aus der Arbeitskopie entfernt; Detailhistorie liegt in Gi ## Zuletzt geschlossen +- Family-View Dashboard ist repo-seitig gebaut: `monitoring/grafana/dashboards/family-status.json` zeigt Family-App-Uptime, Backup-Alter, TLS-Restlaufzeit, Critical-Container und Image-Drift. +- Alt-Volume-Freigabe ist vorbereitet: `ops/maintenance/release-alt-volumes.sh --dry-run` validiert aktive Pfade, Container-Health, Restore-Freshness und gemountete Altpfade; Test am 2026-06-01 fand vier Kandidaten und keine Blocker, Ausfuehrung bleibt wegen Cutoff bis 2026-06-02 gesperrt. - Borg-Nachlauf nach dem 2026-05-31-Sprint ist belegt: Archiv `Taegliche-Sicherung-2026-06-01T04:30:26.913`, 101669 Dateien, `rc=0`; Freshness-Check am 2026-06-01: Critical 0, Warnings 0. - H:/ Nearline-Pull am 2026-06-01 repariert und manuell validiert: kuratierte Borg-Dumps Exit 0, Gitea-Bundles Exit 1 (Robocopy-Erfolg mit Kopien), Report `nearline-pull-2026-06-01-082553.md`. - Immich-, Paperless-, Gitea- und Vaultwarden-Restore-Pfade sind belegt. diff --git a/docs/SERVICE_CATALOG.md b/docs/SERVICE_CATALOG.md index f537d50..8bba0e7 100644 --- a/docs/SERVICE_CATALOG.md +++ b/docs/SERVICE_CATALOG.md @@ -1,6 +1,6 @@ # Service Catalog -Stand: 2026-05-31 +Stand: 2026-06-01 Dieser Katalog beschreibt produktive und repo-vorbereitete Dienste aus Sicht von Betrieb, Restore und KI-Kontext. Er basiert auf dem Repo-Sollzustand. Vor produktiven Eingriffen immer den Live-Zustand in Komodo/Docker pruefen. @@ -65,7 +65,7 @@ Secret-Werte sind nicht enthalten. Es werden nur Secret-Namen, Env-Key-Namen und | `speedtest-tracker` | Speedtest-Monitoring | `ops/speedtest/docker-compose.yml` | `https://speedtest.kaleschke.info` | Traefik + Authelia | `/mnt/user/appdata/speedtest-tracker/config` | Tier 3, `speedtest-tracker.sqlite.dump` | ja + Authelia | `APP_KEY`, `ADMIN_PASSWORD` Stack ENV | | `filebrowser` | Datei-Browser fuer Documents/Photos/Projekte | `ops/filebrowser/docker-compose.yml` | `https://files.kaleschke.info` | Traefik + Authelia | `/mnt/user/appdata/filebrowser/*`, `/mnt/user/documents`, `/mnt/user/photos`, `/mnt/user/projekte` | Tier 3, `filebrowser.bolt.dump` + Share | ja + Authelia | Breiter Appdata-Mount entfernt; Secrets und Traefik-Dynamic-Config sind nicht mehr ueber Filebrowser gemountet | | `code-server` | Web-Editor / Operations Workspace | `ops/code-server/docker-compose.yml` | `https://code.kaleschke.info` | Traefik + Authelia | `/mnt/user/appdata/code-server`, `/mnt/user/services/dev` | Tier 3 | ja + Authelia | Passwort ueber LSIO `FILE__PASSWORD`; Workspaces beachten | -| `monitoring-grafana` | zentrale Observability-UI fuer Metriken, Logs und InfluxDB | `monitoring/docker-compose.yml` | `https://monitoring.kaleschke.info` | Traefik + Authelia, Prometheus, Loki, InfluxDB 3 Core | named volume `grafana_data`, Provisioning unter `monitoring/grafana/provisioning`, Dashboards unter `monitoring/grafana/dashboards` | Tier 3, named volume | ja + Authelia | Admin-Passwort ueber `monitoring_grafana_admin_password.txt`; Zielbestand: `Homelab / Availability`, `Homelab / Host Overview`, `Homelab / Containers + Logs`, `Traefik Official Standalone Dashboard`; Dashboard-Importer ist optionales `bootstrap`-Profil fuer Traefik | +| `monitoring-grafana` | zentrale Observability-UI fuer Metriken, Logs und InfluxDB | `monitoring/docker-compose.yml` | `https://monitoring.kaleschke.info` | Traefik + Authelia, Prometheus, Loki, InfluxDB 3 Core | named volume `grafana_data`, Provisioning unter `monitoring/grafana/provisioning`, Dashboards unter `monitoring/grafana/dashboards` | Tier 3, named volume | ja + Authelia | Admin-Passwort ueber `monitoring_grafana_admin_password.txt`; Zielbestand: `Homelab / Availability`, `Homelab / Host Overview`, `Homelab / Containers + Logs`, `Homelab / Family Status`, `Traefik Official Standalone Dashboard`; Dashboard-Importer ist optionales `bootstrap`-Profil fuer Traefik | | `monitoring-prometheus` | Metrik-Speicher fuer Homelab-Monitoring | `monitoring/docker-compose.yml`, `monitoring/prometheus/prometheus.yml`, `monitoring/prometheus/alerts.yml` | intern `http://prometheus:9090` | `monitoring_net`, node-exporter, cAdvisor, Traefik-Metrics, Blackbox Exporter, Alertmanager | named volume `prometheus_data` | Tier 3, transiente Metriken mit 30 Tagen Retention | nein | Scrapes: Prometheus, node-exporter, cAdvisor, Traefik `:8082`, `blackbox-http`; Prometheus-Regeln senden an Alertmanager und von dort nach ntfy | | `monitoring-alertmanager` | Alert-Routing fuer Prometheus-Regeln | `monitoring/docker-compose.yml`, `monitoring/alertmanager/alertmanager.yml` | intern `:9093` | Prometheus, ntfy Bridge | named volume `alertmanager_data` | Tier 3 | nein | sendet firing und resolved Alerts an `monitoring-alertmanager-ntfy-bridge` | | `monitoring-alertmanager-ntfy-bridge` | Alertmanager-Webhook nach ntfy Push | `monitoring/docker-compose.yml`, `monitoring/alertmanager-ntfy-bridge/bridge.py` | intern `:8080` | Alertmanager, `https://ntfy.kaleschke.info/homelab-alerts` | kein kritischer Zustand | rebuildbar | nein | formatiert Alertmanager JSON als ntfy Titel, Nachricht, Priority und Tags; keine Secrets | diff --git a/monitoring/README.md b/monitoring/README.md index e11a309..530f2c9 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -17,7 +17,7 @@ Zielzustand: ein zentraler Observability-Stack fuer KalliLab CORE. Die alten Pfade `ops/loki` und `ops/grafana-influxdb` wurden am 2026-05-26 aus dem aktiven Repo entfernt. Rollback erfolgt bei Bedarf ueber Git-Historie, nicht ueber parallel gepflegte Compose-Verzeichnisse. -Live-Stand 2026-05-25: die zehn `monitoring-*` Container laufen produktiv, die alten Container `grafana`, `influxdb3-core`, `loki` und `alloy` sind in Docker nicht mehr vorhanden. Uptime Kuma ist durch Blackbox Exporter, Prometheus-Alerts und das Dashboard `Homelab / Availability` abgeloest. +Live-Stand 2026-06-01: die zehn `monitoring-*` Container laufen produktiv, die alten Container `grafana`, `influxdb3-core`, `loki` und `alloy` sind in Docker nicht mehr vorhanden. Uptime Kuma ist durch Blackbox Exporter, Prometheus-Alerts und das Dashboard `Homelab / Availability` abgeloest. ## Secrets @@ -72,7 +72,7 @@ INFLUXDB_BIND_IP=192.168.178.58 - Dozzle bleibt abgeloest: `Homelab / Containers + Logs` ersetzt Live-Logs und Error-Rate. - Glances erst stoppen, wenn `Homelab / Host Overview` und `Homelab / Containers + Logs` fuer CPU, RAM, Disk, Network, Container-CPU und Container-RAM passen. - Uptime Kuma ist entfernt; `Homelab / Availability`, Blackbox Exporter und Prometheus-Alerts sind der Zielzustand fuer HTTP-Verfuegbarkeit. -- Dashboard-Zielbestand: `Homelab / Availability`, `Homelab / Containers + Logs`, `Homelab / Host Overview`, `Traefik Official Standalone Dashboard`. +- Dashboard-Zielbestand: `Homelab / Availability`, `Homelab / Containers + Logs`, `Homelab / Host Overview`, `Homelab / Family Status`, `Traefik Official Standalone Dashboard`. ## Alerting diff --git a/monitoring/grafana/dashboards/family-status.json b/monitoring/grafana/dashboards/family-status.json new file mode 100644 index 0000000..63f51cf --- /dev/null +++ b/monitoring/grafana/dashboards/family-status.json @@ -0,0 +1,295 @@ +{ + "uid": "homelab-family-status", + "title": "Homelab / Family Status", + "tags": ["homelab", "family", "status"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-24h", + "to": "now" + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Family Apps Up", + "datasource": "Prometheus", + "gridPos": {"h": 5, "w": 6, "x": 0, "y": 0}, + "targets": [ + { + "refId": "A", + "expr": "sum(probe_success{job=\"blackbox-http\", instance=~\"https://(vault|immich|cloud|paperless|mealie|ntfy|glance)\\\\.kaleschke\\\\.info\"})" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "yellow", "value": 5}, + {"color": "green", "value": 7} + ] + } + }, + "overrides": [] + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 2, + "type": "stat", + "title": "Family Apps Down", + "datasource": "Prometheus", + "gridPos": {"h": 5, "w": 6, "x": 6, "y": 0}, + "targets": [ + { + "refId": "A", + "expr": "count(probe_success{job=\"blackbox-http\", instance=~\"https://(vault|immich|cloud|paperless|mealie|ntfy|glance)\\\\.kaleschke\\\\.info\"}) - sum(probe_success{job=\"blackbox-http\", instance=~\"https://(vault|immich|cloud|paperless|mealie|ntfy|glance)\\\\.kaleschke\\\\.info\"})" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + }, + "overrides": [] + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 3, + "type": "stat", + "title": "Backup Age", + "datasource": "Prometheus", + "gridPos": {"h": 5, "w": 6, "x": 12, "y": 0}, + "targets": [ + { + "refId": "A", + "expr": "(time() - homelab_borg_last_completed_timestamp_seconds) / 3600" + } + ], + "fieldConfig": { + "defaults": { + "unit": "h", + "decimals": 1, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 24}, + {"color": "red", "value": 30} + ] + } + }, + "overrides": [] + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 4, + "type": "stat", + "title": "TLS Days Left", + "datasource": "Prometheus", + "gridPos": {"h": 5, "w": 6, "x": 18, "y": 0}, + "targets": [ + { + "refId": "A", + "expr": "min((probe_ssl_earliest_cert_expiry{job=\"blackbox-http\"} - time()) / 86400)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "d", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "yellow", "value": 7}, + {"color": "green", "value": 21} + ] + } + }, + "overrides": [] + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 5, + "type": "stat", + "title": "Critical Containers Down", + "datasource": "Prometheus", + "gridPos": {"h": 5, "w": 6, "x": 0, "y": 5}, + "targets": [ + { + "refId": "A", + "expr": "count(homelab_critical_container_running) - sum(homelab_critical_container_running)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "red", "value": 1} + ] + } + }, + "overrides": [] + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 6, + "type": "stat", + "title": "Image Drift", + "datasource": "Prometheus", + "gridPos": {"h": 5, "w": 6, "x": 6, "y": 5}, + "targets": [ + { + "refId": "A", + "expr": "count(homelab_gitops_runtime_image_match) - sum(homelab_gitops_runtime_image_match)" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 1}, + {"color": "red", "value": 3} + ] + } + }, + "overrides": [] + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 7, + "type": "stat", + "title": "Last Backup OK", + "datasource": "Prometheus", + "gridPos": {"h": 5, "w": 6, "x": 12, "y": 5}, + "targets": [ + { + "refId": "A", + "expr": "homelab_borg_last_success" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bool", + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "red", "value": null}, + {"color": "green", "value": 1} + ] + } + }, + "overrides": [] + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 8, + "type": "stat", + "title": "Metrics Age", + "datasource": "Prometheus", + "gridPos": {"h": 5, "w": 6, "x": 18, "y": 5}, + "targets": [ + { + "refId": "A", + "expr": "(time() - homelab_textfile_exporter_last_run_timestamp_seconds) / 60" + } + ], + "fieldConfig": { + "defaults": { + "unit": "m", + "decimals": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + {"color": "green", "value": null}, + {"color": "yellow", "value": 60}, + {"color": "red", "value": 120} + ] + } + }, + "overrides": [] + }, + "options": {"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": false}} + }, + { + "id": 9, + "type": "timeseries", + "title": "Family App Availability", + "datasource": "Prometheus", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 10}, + "targets": [ + { + "refId": "A", + "expr": "probe_success{job=\"blackbox-http\", instance=~\"https://(vault|immich|cloud|paperless|mealie|ntfy|glance)\\\\.kaleschke\\\\.info\"}", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": {"defaults": {"unit": "bool", "min": 0, "max": 1}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}} + }, + { + "id": 10, + "type": "timeseries", + "title": "Family App Response Time", + "datasource": "Prometheus", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 10}, + "targets": [ + { + "refId": "A", + "expr": "probe_duration_seconds{job=\"blackbox-http\", instance=~\"https://(vault|immich|cloud|paperless|mealie|ntfy|glance)\\\\.kaleschke\\\\.info\"}", + "legendFormat": "{{instance}}" + } + ], + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "options": {"legend": {"displayMode": "list", "placement": "bottom"}} + }, + { + "id": 11, + "type": "table", + "title": "Public Endpoint Status", + "datasource": "Prometheus", + "gridPos": {"h": 9, "w": 24, "x": 0, "y": 18}, + "targets": [ + { + "refId": "A", + "expr": "probe_http_status_code{job=\"blackbox-http\"}", + "format": "table", + "instant": true + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": {"Time": true, "__name__": true, "job": true}, + "renameByName": {"Value": "status_code", "instance": "target"} + } + } + ] + } + ] +} diff --git a/ops/maintenance/release-alt-volumes.sh b/ops/maintenance/release-alt-volumes.sh new file mode 100755 index 0000000..aef8006 --- /dev/null +++ b/ops/maintenance/release-alt-volumes.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -euo pipefail + +MODE="dry-run" +CUTOFF_DATE="2026-06-02" + +if [[ "${1:-}" == "--execute" ]]; then + MODE="execute" +elif [[ "${1:-}" != "" && "${1:-}" != "--dry-run" ]]; then + echo "Usage: $0 [--dry-run|--execute]" >&2 + exit 2 +fi + +if [[ "$(id -u)" -ne 0 ]]; then + echo "Must run as root on the Unraid host." >&2 + exit 1 +fi + +today="$(date +%F)" +if [[ "$MODE" == "execute" && "$today" < "$CUTOFF_DATE" ]]; then + echo "Refusing: cutoff is $CUTOFF_DATE, today is $today." >&2 + exit 1 +fi + +declare -a CANDIDATES=( + "/mnt/user/appdata/postgresql17|/mnt/user/appdata/postgresql18|shared PostgreSQL 17 rollback" + "/mnt/user/appdata/mealie/postgres|/mnt/user/appdata/mealie/postgres18|Mealie PostgreSQL 17 rollback" + "/mnt/user/appdata/nextcloud/postgres|/mnt/user/appdata/nextcloud/postgres18|Nextcloud PostgreSQL 17 rollback" + "/mnt/user/appdata/immich_postgres|/mnt/user/appdata/immich_postgres_vectorchord|Immich pgvecto.rs rollback" +) + +require_container_healthy() { + local name="$1" + local state + local health + + state="$(docker inspect "$name" --format '{{.State.Status}}' 2>/dev/null || true)" + health="$(docker inspect "$name" --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' 2>/dev/null || true)" + + if [[ "$state" != "running" ]]; then + echo "Container $name is not running (state=$state)." >&2 + exit 1 + fi + + if [[ "$health" != "healthy" && "$health" != "none" ]]; then + echo "Container $name is not healthy (health=$health)." >&2 + exit 1 + fi +} + +echo "Alt-volume release check" +echo "Mode: $MODE" +echo "Date: $today" +echo + +require_container_healthy postgresql17 +require_container_healthy mealie-postgres +require_container_healthy nextcloud-postgres +require_container_healthy immich_postgres + +if docker ps --filter health=unhealthy --format '{{.Names}}' | grep -q .; then + echo "Refusing: unhealthy containers exist." >&2 + docker ps --filter health=unhealthy --format ' {{.Names}} {{.Status}}' >&2 + exit 1 +fi + +if [[ -x /mnt/user/services/homelab-infra/ops/restore-tests/run-restore-checks.sh ]]; then + /mnt/user/services/homelab-infra/ops/restore-tests/run-restore-checks.sh freshness +fi + +mapfile -t active_mounts < <(docker inspect $(docker ps -q) --format '{{range .Mounts}}{{println .Source}}{{end}}' 2>/dev/null || true) + +for entry in "${CANDIDATES[@]}"; do + IFS='|' read -r old_path active_path label <<< "$entry" + + if [[ ! -d "$active_path" ]]; then + echo "Missing active path for $label: $active_path" >&2 + exit 1 + fi + + if [[ ! -d "$old_path" ]]; then + echo "Already absent: $old_path ($label)" + continue + fi + + if printf '%s\n' "${active_mounts[@]}" | grep -Fxq "$old_path"; then + echo "Refusing: old path is still mounted by a running container: $old_path" >&2 + exit 1 + fi + + size="$(du -sh "$old_path" 2>/dev/null | awk '{print $1}')" + echo "Candidate: $old_path ($label, $size)" + echo "Active: $active_path" + + if [[ "$MODE" == "execute" ]]; then + rm -rf --one-file-system "$old_path" + echo "Removed: $old_path" + else + echo "Dry-run: would remove $old_path" + fi + echo +done + +echo "Alt-volume release check completed."