Harden backup and posture checks

2026-05-16 13:04:22 +02:00
parent 12a87ad342
commit 878ad2d5f1
25 changed files with 716 additions and 54 deletions
@@ -94,6 +94,39 @@ dump_sqlite_file() {
  atomic_write "$output" "$tmp"
 }

+dump_sqlite_container() {
+  container="$1"
+  db_path="$2"
+  output="$3"
+
+  if ! need_container "$container"; then
+    warn "Skipping missing container: $container"
+    return 0
+  fi
+
+  container_tmp="/tmp/$(basename "$output").bak"
+  tmp="$TMP_DIR/$(basename "$output").tmp"
+
+  log "Dumping SQLite database '$db_path' from $container"
+  rm -f "$tmp"
+  docker exec "$container" rm -f "$container_tmp" >/dev/null 2>&1 || true
+  if ! docker exec "$container" sqlite3 "$db_path" ".backup $container_tmp"; then
+    warn "SQLite backup failed for $container:$db_path"
+    docker exec "$container" rm -f "$container_tmp" >/dev/null 2>&1 || true
+    rm -f "$tmp"
+    return 1
+  fi
+  docker cp "$container:$container_tmp" "$tmp"
+  docker exec "$container" rm -f "$container_tmp" >/dev/null 2>&1 || true
+
+  if [ "$(sqlite3 "$tmp" 'PRAGMA quick_check;')" != "ok" ]; then
+    warn "SQLite quick_check failed for $container:$db_path"
+    rm -f "$tmp"
+    return 1
+  fi
+  atomic_write "$output" "$tmp"
+}
+
 dump_optional_pg_db() {
  container="$1"
  password="$2"
@@ -196,12 +229,14 @@ main() {
    warn "Skipping missing container: nextcloud-postgres"
  fi

-  # SQLite databases. Use host-side sqlite3 so the dump does not depend on
-  # utility packages inside application images.
-  dump_sqlite_file "/mnt/user/services/gitea/data/gitea/gitea.db" "$LATEST_DIR/gitea.sqlite" "gitea"
-  dump_sqlite_file "/mnt/user/appdata/vaultwarden/db.sqlite3" "$LATEST_DIR/vaultwarden.sqlite" "vaultwarden"
-  dump_sqlite_file "/mnt/user/appdata/uptime-kuma/kuma.db" "$LATEST_DIR/uptime-kuma.sqlite" "uptime-kuma"
-  dump_sqlite_file "/mnt/user/appdata/speedtest-tracker/config/database.sqlite" "$LATEST_DIR/speedtest-tracker.sqlite" "speedtest-tracker"
+  # SQLite databases
+  dump_sqlite_container "gitea" "/data/gitea/gitea.db" "$LATEST_DIR/gitea.sqlite.dump"
+  dump_sqlite_container "vaultwarden" "/data/db.sqlite3" "$LATEST_DIR/vaultwarden.sqlite.dump"
+  dump_sqlite_container "uptime-kuma" "/app/data/kuma.db" "$LATEST_DIR/uptime-kuma.sqlite.dump"
+  dump_sqlite_container "speedtest-tracker" "/config/database.sqlite" "$LATEST_DIR/speedtest-tracker.sqlite.dump"
+  dump_sqlite_container "filebrowser" "/database/filebrowser.db" "$LATEST_DIR/filebrowser.sqlite.dump"
+
+  # Additional host-side SQLite dumps for admin tooling with appdata files.
  dump_sqlite_file "/mnt/user/appdata/borg-ui/data/borg.db" "$LATEST_DIR/borg-ui.sqlite" "borg-ui"
  dump_sqlite_file "/mnt/user/appdata/grafana/grafana.db" "$LATEST_DIR/grafana.sqlite" "grafana"

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="${REPO_ROOT:-$(cd "$SCRIPT_DIR/../../.." && pwd)}"
+POSTURE_CHECK="${POSTURE_CHECK:-$REPO_ROOT/services/posture-check/posture-check.sh}"
+FRESHNESS_CHECK="${FRESHNESS_CHECK:-$REPO_ROOT/ops/restore-tests/check-restore-freshness.sh}"
+PRE_BACKUP_DUMPS="${PRE_BACKUP_DUMPS:-$SCRIPT_DIR/pre-backup-dumps.sh}"
+NTFY_SCRIPT="${NTFY_SCRIPT:-$REPO_ROOT/ops/restore-tests/send-ntfy.sh}"
+NTFY_TOPIC="${NTFY_TOPIC:-kallilab-critical}"
+
+notify_failure() {
+  local step="$1"
+  local message="$2"
+  if [ -x "$NTFY_SCRIPT" ]; then
+    "$NTFY_SCRIPT" "$NTFY_TOPIC" "Borg pre-hook failed: $step" "$message" high || true
+  fi
+}
+
+run_step() {
+  local step="$1"
+  shift
+
+  echo "[pre-borg] Running $step"
+  if "$@"; then
+    echo "[pre-borg] OK: $step"
+  else
+    rc=$?
+    notify_failure "$step" "Command failed with exit code $rc: $*"
+    exit "$rc"
+  fi
+}
+
+run_step "posture-check" "$POSTURE_CHECK"
+run_step "pre-backup-dumps" "$PRE_BACKUP_DUMPS"
+run_step "restore-freshness" "$FRESHNESS_CHECK"
+
+echo "[pre-borg] All pre-flight checks passed"