ops: add restore freshness negative alert test

2026-06-06 13:01:54 +02:00
parent 2628a0c795
commit c33e29016b
4 changed files with 81 additions and 2 deletions
@@ -0,0 +1,74 @@
+#!/bin/bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+LAB_ROOT="${LAB_ROOT:-/mnt/user/backups/restore-lab/freshness-negative}"
+REPORT_ROOT="${REPORT_ROOT:-/mnt/user/backups/restore-reports}"
+ALERT_TOPIC="${ALERT_TOPIC:-homelab-alerts}"
+INFO_TOPIC="${INFO_TOPIC:-homelab-info}"
+SEND_NTFY="${SEND_NTFY:-1}"
+
+stamp="$(date +%F-%H%M%S)"
+test_root="$LAB_ROOT/$stamp"
+dump_root="$test_root/dumps"
+test_report_root="$test_root/reports"
+report_file="$REPORT_ROOT/freshness-negative-$stamp.md"
+raw_log="$test_root/check-output.md"
+
+mkdir -p "$dump_root" "$test_report_root" "$REPORT_ROOT"
+
+cleanup() {
+  rm -rf "$test_root"
+}
+trap cleanup EXIT
+
+set +e
+DUMP_ROOT="$dump_root" \
+REPORT_ROOT="$test_report_root" \
+MAX_DUMP_AGE_HOURS=26 \
+MAX_REPORT_AGE_DAYS=45 \
+  "$SCRIPT_DIR/check-restore-freshness.sh" >"$raw_log" 2>&1
+rc=$?
+set -e
+
+critical_count="$(awk -F': ' '/^Critical:/ {print $2; exit}' "$raw_log" | tr -d '[:space:]')"
+critical_count="${critical_count:-0}"
+
+{
+  echo "# Restore Freshness Negative Alert Test"
+  echo
+  echo "Timestamp: $(date '+%F %T')"
+  echo "Result: $([ "$rc" -ne 0 ] && [ "$critical_count" -gt 0 ] && echo "ok" || echo "failed")"
+  echo "Check exit code: $rc"
+  echo "Critical count: $critical_count"
+  echo "Synthetic dump root: $dump_root"
+  echo "Synthetic report root: $test_report_root"
+  echo "Production dump root touched: no"
+  echo
+  echo "## Check Output"
+  echo
+  cat "$raw_log"
+} >"$report_file"
+
+if [ "$rc" -ne 0 ] && [ "$critical_count" -gt 0 ]; then
+  if [ "$SEND_NTFY" = "1" ]; then
+    "$SCRIPT_DIR/send-ntfy.sh" \
+      "$ALERT_TOPIC" \
+      "TEST: Restore freshness alert path ok" \
+      "Negativtest erfolgreich: check-restore-freshness.sh meldete ${critical_count} Criticals gegen synthetischen leeren Testpfad. Produktive Dumps wurden nicht veraendert. Report: $report_file" \
+      high
+  fi
+  echo "Negative freshness alert test ok. Report: $report_file"
+  exit 0
+fi
+
+if [ "$SEND_NTFY" = "1" ]; then
+  "$SCRIPT_DIR/send-ntfy.sh" \
+    "$ALERT_TOPIC" \
+    "TEST FAILED: Restore freshness alert path" \
+    "Negativtest fehlgeschlagen: erwarteter Critical-Zustand wurde nicht erkannt. Report: $report_file" \
+    high || true
+fi
+
+echo "Negative freshness alert test failed. Report: $report_file" >&2
+exit 1