Harden backup and posture checks

ops/restore-tests/check-restore-freshness.ps1

@@ -1,7 +1,7 @@
 param(
   [string]$DumpRoot = "/mnt/user/backups/borg/dumps/latest",
   [string]$ReportRoot = "/mnt/user/backups/restore-reports",
-  [int]$MaxDumpAgeHours = 36,
+  [int]$MaxDumpAgeHours = 26,
   [int]$MaxReportAgeDays = 45
 )
 
@@ -9,7 +9,13 @@ $checks = @(
   @{ Name = "postgresql17-paperless.dump"; Path = Join-Path $DumpRoot "postgresql17-paperless.dump" },
   @{ Name = "postgresql17-mailarchiver.dump"; Path = Join-Path $DumpRoot "postgresql17-mailarchiver.dump" },
   @{ Name = "mealie.dump"; Path = Join-Path $DumpRoot "mealie.dump" },
-  @{ Name = "immich.dump"; Path = Join-Path $DumpRoot "immich.dump" }
+  @{ Name = "immich.dump"; Path = Join-Path $DumpRoot "immich.dump" },
+  @{ Name = "nextcloud.dump"; Path = Join-Path $DumpRoot "nextcloud.dump" },
+  @{ Name = "gitea.sqlite.dump"; Path = Join-Path $DumpRoot "gitea.sqlite.dump" },
+  @{ Name = "vaultwarden.sqlite.dump"; Path = Join-Path $DumpRoot "vaultwarden.sqlite.dump" },
+  @{ Name = "uptime-kuma.sqlite.dump"; Path = Join-Path $DumpRoot "uptime-kuma.sqlite.dump" },
+  @{ Name = "speedtest-tracker.sqlite.dump"; Path = Join-Path $DumpRoot "speedtest-tracker.sqlite.dump" },
+  @{ Name = "filebrowser.sqlite.dump"; Path = Join-Path $DumpRoot "filebrowser.sqlite.dump" }
 )
 
 $reportChecks = @(
@@ -30,15 +36,25 @@ foreach ($check in $checks) {
   }
 
   $item = Get-Item $check.Path
+  if ($item.Length -le 0) {
+    $critical.Add("DUMP_EMPTY $($check.Name)")
+    continue
+  }
+
   $ageHours = ($now - $item.LastWriteTime).TotalHours
   if ($ageHours -gt $MaxDumpAgeHours) {
-    $warnings.Add(("DUMP_STALE {0} age={1:N1}h" -f $check.Name, $ageHours))
+    $critical.Add(("DUMP_STALE {0} age={1:N1}h" -f $check.Name, $ageHours))
   } else {
     $info.Add(("DUMP_OK {0} age={1:N1}h" -f $check.Name, $ageHours))
   }
 }
 
 foreach ($check in $reportChecks) {
+  if (-not (Test-Path $ReportRoot)) {
+    $warnings.Add("REPORT_ROOT_MISSING $ReportRoot")
+    break
+  }
+
   $latest = Get-ChildItem -Path $ReportRoot -Filter ([System.IO.Path]::GetFileName($check.Path)) -ErrorAction SilentlyContinue |
     Sort-Object LastWriteTime -Descending |
     Select-Object -First 1

ops/restore-tests/check-restore-freshness.sh

@@ -3,7 +3,7 @@ set -euo pipefail
 
 DUMP_ROOT="${DUMP_ROOT:-/mnt/user/backups/borg/dumps/latest}"
 REPORT_ROOT="${REPORT_ROOT:-/mnt/user/backups/restore-reports}"
-MAX_DUMP_AGE_HOURS="${MAX_DUMP_AGE_HOURS:-36}"
+MAX_DUMP_AGE_HOURS="${MAX_DUMP_AGE_HOURS:-26}"
 MAX_REPORT_AGE_DAYS="${MAX_REPORT_AGE_DAYS:-45}"
 
 now_epoch="$(date +%s)"
@@ -25,21 +25,40 @@ check_file_age_days() {
   echo $(( (now_epoch - mtime) / 86400 ))
 }
 
-for dump in postgresql17-paperless.dump postgresql17-mailarchiver.dump mealie.dump immich.dump; do
+for dump in \
+  postgresql17-paperless.dump \
+  postgresql17-mailarchiver.dump \
+  mealie.dump \
+  immich.dump \
+  nextcloud.dump \
+  gitea.sqlite.dump \
+  vaultwarden.sqlite.dump \
+  uptime-kuma.sqlite.dump \
+  speedtest-tracker.sqlite.dump \
+  filebrowser.sqlite.dump; do
   path="$DUMP_ROOT/$dump"
   if [ ! -f "$path" ]; then
     critical+=("DUMP_MISSING $dump")
     continue
   fi
+  if [ ! -s "$path" ]; then
+    critical+=("DUMP_EMPTY $dump")
+    continue
+  fi
   age="$(check_file_age_hours "$path")"
   if [ "$age" -gt "$MAX_DUMP_AGE_HOURS" ]; then
-    warnings+=("DUMP_STALE $dump age=${age}h")
+    critical+=("DUMP_STALE $dump age=${age}h")
   else
     info+=("DUMP_OK $dump age=${age}h")
   fi
 done
 
 for service in vaultwarden gitea paperless; do
+  if [ ! -d "$REPORT_ROOT" ]; then
+    warnings+=("REPORT_ROOT_MISSING $REPORT_ROOT")
+    break
+  fi
+
   latest="$(find "$REPORT_ROOT" -maxdepth 1 -type f -name "$service-*.md" | sort | tail -n 1 || true)"
   if [ -z "$latest" ]; then
     warnings+=("REPORT_MISSING $service")

ops/restore-tests/monthly-random-restore.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TOPIC="${TOPIC:-kallilab-info}"
+TESTS="${TESTS:-vaultwarden gitea paperless}"
+
+pick_random() {
+  printf '%s\n' $TESTS | awk 'BEGIN { srand() } { items[++count] = $0 } END { print items[int(rand() * count) + 1] }'
+}
+
+selected="$(pick_random)"
+echo "Selected monthly restore test: $selected"
+
+exec "$SCRIPT_DIR/run-restore-job-with-ntfy.sh" "$selected" "$TOPIC"