Harden backup and posture checks

ops/borg-ui/BACKUP_SCOPE.md

@@ -43,6 +43,7 @@ The inclusion of `/local/secrets` is intentional: Borg is expected to cover disa
 | Komodo | config + Mongo dump | `/local/borg-dumps`, `/local/appdata/komodo/periphery`, `/local/appdata/komodo/core` |
 | Nextcloud | DB dump + file data | `/local/borg-dumps`, `/local/appdata/nextcloud/html`, `/local/nextcloud/data` |
 | Grafana | SQLite dump + file data | `/local/borg-dumps`, `/local/appdata/grafana` |
+| Filebrowser | SQLite dump + file data | `/local/borg-dumps`, `/local/appdata/filebrowser` |
 | InfluxDB 3 Core | file data | `/local/appdata/influxdb3/data`, `/local/appdata/influxdb3/plugins` |
 | Hermes Agent | file data + SSH key | `/local/appdata/hermes-agent/data`, `/local/secrets/hermes_runner_id_ed25519` |
 | BentoPDF | rebuildable | no critical persistence in compose |
@@ -53,7 +54,7 @@ These are deviations from the standard "DB dump first, file path second" strateg
 
 ### Nextcloud
 
-`pre-backup-dumps.sh` writes `nextcloud.dump` from `nextcloud-postgres`. Borg UI also mounts `/mnt/user/documents/nextcloud-data` read-only as `/local/nextcloud/data`, so database and user files are both inside scope after the Borg UI stack is recreated.
+Option A umgesetzt: `pre-backup-dumps.sh` writes `nextcloud.dump` from `nextcloud-postgres`. Borg UI also mounts `/mnt/user/documents/nextcloud-data` read-only as `/local/nextcloud/data`, so database and user files are both inside scope after the Borg UI stack is recreated.
 
 ### Komodo Mongo dump
 
@@ -76,7 +77,7 @@ These are deviations from the standard "DB dump first, file path second" strateg
 ### Other Databases
 
 - Komodo MongoDB
-- SQLite: `gitea`, `vaultwarden`, `uptime-kuma`, `speedtest-tracker`, `borg-ui`, `grafana`
+- SQLite: `gitea`, `vaultwarden`, `uptime-kuma`, `speedtest-tracker`, `filebrowser`, `borg-ui`, `grafana`
 
 ## Explicitly Not Backed Up as Raw Live DB Files
 
@@ -98,7 +99,6 @@ These are not part of the first-class Borg scope:
 - uptime-kuma
 - scrutiny metrics history
 - dozzle, glances, speedtest
-- filebrowser app state
 
 ## Suggested Retention
 

ops/borg-ui/scripts/pre-backup-dumps.sh

@@ -94,6 +94,39 @@ dump_sqlite_file() {
   atomic_write "$output" "$tmp"
 }
 
+dump_sqlite_container() {
+  container="$1"
+  db_path="$2"
+  output="$3"
+
+  if ! need_container "$container"; then
+    warn "Skipping missing container: $container"
+    return 0
+  fi
+
+  container_tmp="/tmp/$(basename "$output").bak"
+  tmp="$TMP_DIR/$(basename "$output").tmp"
+
+  log "Dumping SQLite database '$db_path' from $container"
+  rm -f "$tmp"
+  docker exec "$container" rm -f "$container_tmp" >/dev/null 2>&1 || true
+  if ! docker exec "$container" sqlite3 "$db_path" ".backup $container_tmp"; then
+    warn "SQLite backup failed for $container:$db_path"
+    docker exec "$container" rm -f "$container_tmp" >/dev/null 2>&1 || true
+    rm -f "$tmp"
+    return 1
+  fi
+  docker cp "$container:$container_tmp" "$tmp"
+  docker exec "$container" rm -f "$container_tmp" >/dev/null 2>&1 || true
+
+  if [ "$(sqlite3 "$tmp" 'PRAGMA quick_check;')" != "ok" ]; then
+    warn "SQLite quick_check failed for $container:$db_path"
+    rm -f "$tmp"
+    return 1
+  fi
+  atomic_write "$output" "$tmp"
+}
+
 dump_optional_pg_db() {
   container="$1"
   password="$2"
@@ -196,12 +229,14 @@ main() {
     warn "Skipping missing container: nextcloud-postgres"
   fi
 
-  # SQLite databases. Use host-side sqlite3 so the dump does not depend on
-  # utility packages inside application images.
-  dump_sqlite_file "/mnt/user/services/gitea/data/gitea/gitea.db" "$LATEST_DIR/gitea.sqlite" "gitea"
-  dump_sqlite_file "/mnt/user/appdata/vaultwarden/db.sqlite3" "$LATEST_DIR/vaultwarden.sqlite" "vaultwarden"
-  dump_sqlite_file "/mnt/user/appdata/uptime-kuma/kuma.db" "$LATEST_DIR/uptime-kuma.sqlite" "uptime-kuma"
-  dump_sqlite_file "/mnt/user/appdata/speedtest-tracker/config/database.sqlite" "$LATEST_DIR/speedtest-tracker.sqlite" "speedtest-tracker"
+  # SQLite databases
+  dump_sqlite_container "gitea" "/data/gitea/gitea.db" "$LATEST_DIR/gitea.sqlite.dump"
+  dump_sqlite_container "vaultwarden" "/data/db.sqlite3" "$LATEST_DIR/vaultwarden.sqlite.dump"
+  dump_sqlite_container "uptime-kuma" "/app/data/kuma.db" "$LATEST_DIR/uptime-kuma.sqlite.dump"
+  dump_sqlite_container "speedtest-tracker" "/config/database.sqlite" "$LATEST_DIR/speedtest-tracker.sqlite.dump"
+  dump_sqlite_container "filebrowser" "/database/filebrowser.db" "$LATEST_DIR/filebrowser.sqlite.dump"
+
+  # Additional host-side SQLite dumps for admin tooling with appdata files.
   dump_sqlite_file "/mnt/user/appdata/borg-ui/data/borg.db" "$LATEST_DIR/borg-ui.sqlite" "borg-ui"
   dump_sqlite_file "/mnt/user/appdata/grafana/grafana.db" "$LATEST_DIR/grafana.sqlite" "grafana"
 

ops/borg-ui/scripts/pre-borg.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="${REPO_ROOT:-$(cd "$SCRIPT_DIR/../../.." && pwd)}"
+POSTURE_CHECK="${POSTURE_CHECK:-$REPO_ROOT/services/posture-check/posture-check.sh}"
+FRESHNESS_CHECK="${FRESHNESS_CHECK:-$REPO_ROOT/ops/restore-tests/check-restore-freshness.sh}"
+PRE_BACKUP_DUMPS="${PRE_BACKUP_DUMPS:-$SCRIPT_DIR/pre-backup-dumps.sh}"
+NTFY_SCRIPT="${NTFY_SCRIPT:-$REPO_ROOT/ops/restore-tests/send-ntfy.sh}"
+NTFY_TOPIC="${NTFY_TOPIC:-kallilab-critical}"
+
+notify_failure() {
+  local step="$1"
+  local message="$2"
+  if [ -x "$NTFY_SCRIPT" ]; then
+    "$NTFY_SCRIPT" "$NTFY_TOPIC" "Borg pre-hook failed: $step" "$message" high || true
+  fi
+}
+
+run_step() {
+  local step="$1"
+  shift
+
+  echo "[pre-borg] Running $step"
+  if "$@"; then
+    echo "[pre-borg] OK: $step"
+  else
+    rc=$?
+    notify_failure "$step" "Command failed with exit code $rc: $*"
+    exit "$rc"
+  fi
+}
+
+run_step "posture-check" "$POSTURE_CHECK"
+run_step "pre-backup-dumps" "$PRE_BACKUP_DUMPS"
+run_step "restore-freshness" "$FRESHNESS_CHECK"
+
+echo "[pre-borg] All pre-flight checks passed"