Close Gitea signup, dedup posture-check alerts, extend Borg scope
Operational hardening across several services after live incident analysis between 2026-05-18 and 2026-05-20: - Gitea: disable public registration and OpenID signup/signin to stop the external POST / 5xx bursts that triggered availability alerts. New repo-wide policy requires every productive Micha/homelab-infra Komodo stack to ship with an active Gitea->Komodo webhook on the current stack ID (documented in CLAUDE.md, AI_CONTEXT.md, WORKFLOW.md). - posture-check: extract the Disk1 fstype check into its own function so the documented Disk1 NTFS exception no longer raises ntfy warnings, skip POSIX inode checks on NTFS, and dedup ntfy alerts via a fingerprint state file with ALERT_REPEAT_SECONDS (default 24h). Repeat-spam on the same cause now suppressed. - docker-critical-events: parse the event JSON for container name, action, exit code and signal; drop `die exit=0` events (clean stops); ship a structured ntfy message instead of the raw event line. - Borg UI: mount /mnt/user/services into the backup container as /local/services:ro and include homelab-infra, stacks and posture-check in all-important-sources.txt. RESTORE_MATRIX and DISASTER_RECOVERY updated accordingly. - Unraid user scripts: document the new homelab-operations-report-daily cron job and the SMTP password file it expects on the host. - MIGRATION_LOG: capture the four live events from this window - Gitea 5xx burst + signup closure, Komodo webhook reconciliation, posture-check host-version verification, Borg scope extension, and Traefik 5xx alert detuning. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -9,15 +9,79 @@ EVENT_FILTERS="${EVENT_FILTERS:---filter event=die --filter event=oom --filter e
|
||||
|
||||
mkdir -p "$(dirname "$OUTPUT_PATH")"
|
||||
|
||||
json_value() {
|
||||
local key="$1"
|
||||
local json="$2"
|
||||
|
||||
printf '%s' "$json" | sed -n "s/.*\"$key\":\"\\([^\"]*\\)\".*/\\1/p" | head -n 1
|
||||
}
|
||||
|
||||
event_summary() {
|
||||
local event="$1"
|
||||
local action name image exit_code signal
|
||||
|
||||
action="$(json_value "Action" "$event")"
|
||||
name="$(json_value "name" "$event")"
|
||||
image="$(json_value "image" "$event")"
|
||||
exit_code="$(json_value "exitCode" "$event")"
|
||||
signal="$(json_value "signal" "$event")"
|
||||
|
||||
printf 'Container: %s\nAction: %s\nImage: %s\nExit-Code: %s\nSignal: %s\n\nFull event logged in: %s\n' \
|
||||
"${name:-unknown}" \
|
||||
"${action:-unknown}" \
|
||||
"${image:-unknown}" \
|
||||
"${exit_code:-n/a}" \
|
||||
"${signal:-n/a}" \
|
||||
"$OUTPUT_PATH"
|
||||
}
|
||||
|
||||
event_title() {
|
||||
local event="$1"
|
||||
local action name exit_code
|
||||
|
||||
action="$(json_value "Action" "$event")"
|
||||
name="$(json_value "name" "$event")"
|
||||
exit_code="$(json_value "exitCode" "$event")"
|
||||
|
||||
if [ -n "$exit_code" ]; then
|
||||
printf 'Docker critical: %s %s exit=%s' "${name:-unknown}" "${action:-event}" "$exit_code"
|
||||
else
|
||||
printf 'Docker critical: %s %s' "${name:-unknown}" "${action:-event}"
|
||||
fi
|
||||
}
|
||||
|
||||
should_send_event() {
|
||||
local event="$1"
|
||||
local action exit_code
|
||||
|
||||
action="$(json_value "Action" "$event")"
|
||||
exit_code="$(json_value "exitCode" "$event")"
|
||||
|
||||
case "$action" in
|
||||
die)
|
||||
[ "${exit_code:-}" != "0" ]
|
||||
;;
|
||||
oom|kill)
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
send_event() {
|
||||
local line="$1"
|
||||
local title message
|
||||
local timestamp
|
||||
timestamp="$(date -Iseconds)"
|
||||
title="$(event_title "$line")"
|
||||
message="$(event_summary "$line")"
|
||||
|
||||
printf '%s %s\n' "$timestamp" "$line" | tee -a "$OUTPUT_PATH" >/dev/null
|
||||
|
||||
if [ "$SEND_NTFY" = "1" ] && [ -f "$NTFY_SCRIPT" ]; then
|
||||
bash "$NTFY_SCRIPT" "$NTFY_TOPIC" "Docker critical event" "$line" high || true
|
||||
bash "$NTFY_SCRIPT" "$NTFY_TOPIC" "$title" "$message" high || true
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -29,5 +93,6 @@ fi
|
||||
# shellcheck disable=SC2086
|
||||
docker events $EVENT_FILTERS --format '{{json .}}' | while IFS= read -r event; do
|
||||
[ -n "$event" ] || continue
|
||||
should_send_event "$event" || continue
|
||||
send_event "$event"
|
||||
done
|
||||
|
||||
@@ -8,6 +8,8 @@ CRITICAL_TOPIC="${CRITICAL_TOPIC:-homelab-alerts}"
|
||||
SEND_NTFY="${SEND_NTFY:-1}"
|
||||
TMP_DIR="${TMP_DIR:-/tmp/kallilab-posture-check}"
|
||||
ALLOW_DISK1_NTFS="${ALLOW_DISK1_NTFS:-1}"
|
||||
ALERT_STATE_PATH="${ALERT_STATE_PATH:-/mnt/user/services/posture-check/last-alert.state}"
|
||||
ALERT_REPEAT_SECONDS="${ALERT_REPEAT_SECONDS:-86400}"
|
||||
|
||||
mkdir -p "$TMP_DIR"
|
||||
RESULTS_FILE="$TMP_DIR/results.$$"
|
||||
@@ -63,6 +65,34 @@ check_fstype() {
|
||||
fi
|
||||
}
|
||||
|
||||
check_disk1_fstype() {
|
||||
local actual
|
||||
|
||||
if ! command -v findmnt >/dev/null 2>&1; then
|
||||
add_result "warning" "disk1_fstype" "Cannot check /mnt/disk1 filesystem because findmnt is missing"
|
||||
return
|
||||
fi
|
||||
|
||||
if ! actual="$(findmnt -no FSTYPE "/mnt/disk1" 2>/dev/null)"; then
|
||||
add_result "warning" "disk1_fstype" "Mount not found: /mnt/disk1"
|
||||
return
|
||||
fi
|
||||
|
||||
if [ "$ALLOW_DISK1_NTFS" = "1" ]; then
|
||||
if [ "$actual" = "ntfs3" ] || [ "$actual" = "fuseblk" ]; then
|
||||
add_result "ok" "disk1_fstype" "/mnt/disk1 filesystem is $actual; temporarily allowed until Disk1 phase 2 migration"
|
||||
else
|
||||
add_result "warning" "disk1_fstype" "/mnt/disk1 filesystem is $actual, expected ntfs3/fuseblk during temporary Disk1 migration exception"
|
||||
fi
|
||||
else
|
||||
if [ "$actual" = "xfs" ]; then
|
||||
add_result "ok" "disk1_fstype" "/mnt/disk1 filesystem is $actual"
|
||||
else
|
||||
add_result "critical" "disk1_fstype" "/mnt/disk1 filesystem is $actual, expected xfs"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
check_no_ntfs_on_core_mounts() {
|
||||
local hits
|
||||
local pattern="^/mnt/(cache|disk1)(/|$)"
|
||||
@@ -80,7 +110,7 @@ check_no_ntfs_on_core_mounts() {
|
||||
if [ -n "$hits" ]; then
|
||||
add_result "critical" "no_ntfs_core_mounts" "NTFS-like filesystem on core mount: $hits"
|
||||
elif [ "$ALLOW_DISK1_NTFS" = "1" ]; then
|
||||
add_result "warning" "no_ntfs_core_mounts" "No NTFS on /mnt/cache; /mnt/disk1 NTFS is temporarily allowed until Disk1 phase 2 migration"
|
||||
add_result "ok" "no_ntfs_core_mounts" "No NTFS on /mnt/cache; /mnt/disk1 NTFS is temporarily allowed until Disk1 phase 2 migration"
|
||||
else
|
||||
add_result "ok" "no_ntfs_core_mounts" "No ntfs3/fuseblk mounts below /mnt/cache or /mnt/disk1"
|
||||
fi
|
||||
@@ -122,6 +152,15 @@ check_inode_usage() {
|
||||
fi
|
||||
}
|
||||
|
||||
check_disk1_inode_usage() {
|
||||
if [ "$ALLOW_DISK1_NTFS" = "1" ]; then
|
||||
add_result "ok" "disk1_inode_usage" "/mnt/disk1 inode usage skipped; NTFS transition filesystem does not expose POSIX inode usage"
|
||||
return
|
||||
fi
|
||||
|
||||
check_inode_usage "/mnt/disk1" 80 "disk1_inode_usage"
|
||||
}
|
||||
|
||||
check_filesystem_usage() {
|
||||
local path="$1"
|
||||
local max_percent="$2"
|
||||
@@ -198,6 +237,79 @@ send_ntfy() {
|
||||
fi
|
||||
}
|
||||
|
||||
alert_fingerprint() {
|
||||
awk -F '\t' '$1 != "ok" { printf "%s|%s|%s\n", $1, $2, $3 }' "$RESULTS_FILE" | cksum | awk '{ print $1 ":" $2 }'
|
||||
}
|
||||
|
||||
alert_summary() {
|
||||
awk -F '\t' '$1 != "ok" { printf "%s:%s; ", $1, $2 }' "$RESULTS_FILE" | sed 's/; $//'
|
||||
}
|
||||
|
||||
should_send_alert() {
|
||||
local fingerprint="$1"
|
||||
local now
|
||||
local last_fingerprint=""
|
||||
local last_sent="0"
|
||||
|
||||
now="$(date +%s)"
|
||||
|
||||
if ! printf '%s' "$ALERT_REPEAT_SECONDS" | grep -Eq '^[0-9]+$'; then
|
||||
ALERT_REPEAT_SECONDS=86400
|
||||
fi
|
||||
|
||||
if [ -f "$ALERT_STATE_PATH" ]; then
|
||||
IFS="$(printf '\t')" read -r last_fingerprint last_sent < "$ALERT_STATE_PATH" || true
|
||||
fi
|
||||
|
||||
if [ "$fingerprint" != "$last_fingerprint" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if ! printf '%s' "$last_sent" | grep -Eq '^[0-9]+$'; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ $((now - last_sent)) -ge "$ALERT_REPEAT_SECONDS" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
remember_alert() {
|
||||
local fingerprint="$1"
|
||||
local now
|
||||
|
||||
now="$(date +%s)"
|
||||
mkdir -p "$(dirname "$ALERT_STATE_PATH")"
|
||||
printf '%s\t%s\n' "$fingerprint" "$now" > "$ALERT_STATE_PATH.tmp"
|
||||
mv "$ALERT_STATE_PATH.tmp" "$ALERT_STATE_PATH"
|
||||
}
|
||||
|
||||
clear_alert_state() {
|
||||
rm -f "$ALERT_STATE_PATH" "$ALERT_STATE_PATH.tmp"
|
||||
}
|
||||
|
||||
send_alert_once() {
|
||||
local severity="$1"
|
||||
local topic="$2"
|
||||
local body="$3"
|
||||
local fingerprint
|
||||
local summary
|
||||
|
||||
fingerprint="$(alert_fingerprint)"
|
||||
summary="$(alert_summary)"
|
||||
|
||||
if [ -n "$summary" ]; then
|
||||
body="$body Checks: $summary"
|
||||
fi
|
||||
|
||||
if should_send_alert "$fingerprint"; then
|
||||
send_ntfy "$severity" "$topic" "$body"
|
||||
remember_alert "$fingerprint"
|
||||
fi
|
||||
}
|
||||
|
||||
write_json() {
|
||||
local timestamp
|
||||
local critical_count
|
||||
@@ -243,13 +355,15 @@ write_json() {
|
||||
cat "$OUTPUT_PATH"
|
||||
|
||||
if [ "$status" = "critical" ]; then
|
||||
send_ntfy "critical" "$CRITICAL_TOPIC" "Posture-check critical: $critical_count critical, $warning_count warning. See $OUTPUT_PATH"
|
||||
send_alert_once "critical" "$CRITICAL_TOPIC" "Posture-check critical: $critical_count critical, $warning_count warning. See $OUTPUT_PATH"
|
||||
return 2
|
||||
fi
|
||||
if [ "$status" = "warning" ]; then
|
||||
send_ntfy "warning" "$WARNING_TOPIC" "Posture-check warning: $warning_count warning. See $OUTPUT_PATH"
|
||||
send_alert_once "warning" "$WARNING_TOPIC" "Posture-check warning: $warning_count warning. See $OUTPUT_PATH"
|
||||
return 1
|
||||
fi
|
||||
|
||||
clear_alert_state
|
||||
}
|
||||
|
||||
main() {
|
||||
@@ -258,15 +372,11 @@ main() {
|
||||
need_cmd awk || true
|
||||
|
||||
check_fstype "/mnt/cache" "xfs" "critical" "cache_fstype"
|
||||
if [ "$ALLOW_DISK1_NTFS" = "1" ]; then
|
||||
check_fstype "/mnt/disk1" "ntfs3" "warning" "disk1_fstype"
|
||||
else
|
||||
check_fstype "/mnt/disk1" "xfs" "critical" "disk1_fstype"
|
||||
fi
|
||||
check_disk1_fstype
|
||||
check_no_ntfs_on_core_mounts
|
||||
check_mover_drift
|
||||
check_inode_usage "/mnt/cache" 80 "cache_inode_usage"
|
||||
check_inode_usage "/mnt/disk1" 80 "disk1_inode_usage"
|
||||
check_disk1_inode_usage
|
||||
check_filesystem_usage "/mnt/cache" 70 "cache_fill_level" "warning"
|
||||
|
||||
for share in appdata system domains; do
|
||||
|
||||
@@ -42,6 +42,34 @@ Zeit: taeglich 06:20, Cron `20 6 * * *`.
|
||||
bash /mnt/user/services/homelab-infra/services/posture-check/compose-runtime-drift.sh
|
||||
```
|
||||
|
||||
## `homelab-operations-report-daily`
|
||||
|
||||
Zeit: taeglich nach Borg und den Morgenchecks, z. B. 07:30, Cron `30 7 * * *`.
|
||||
|
||||
Voraussetzung: SMTP-Passwort liegt **nicht im Repo**, sondern auf dem Host:
|
||||
|
||||
```bash
|
||||
mkdir -p /mnt/user/appdata/secrets
|
||||
chmod 700 /mnt/user/appdata/secrets
|
||||
printf '%s' 'SMTP_PASSWORT_HIER_EINTRAGEN' > /mnt/user/appdata/secrets/homelab_smtp_password.txt
|
||||
chmod 600 /mnt/user/appdata/secrets/homelab_smtp_password.txt
|
||||
```
|
||||
|
||||
User Script:
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
SEND_MAIL=1 \
|
||||
MAIL_MODE=always \
|
||||
MAIL_FROM="michideheld@gmx.de" \
|
||||
MAIL_TO="Mi.Kaleschke@gmx.de" \
|
||||
SMTP_HOST="smtp.gmx.net" \
|
||||
SMTP_PORT="587" \
|
||||
SMTP_USER="michideheld@gmx.de" \
|
||||
SMTP_PASS_FILE="/mnt/user/appdata/secrets/homelab_smtp_password.txt" \
|
||||
bash /mnt/user/services/homelab-infra/services/posture-check/daily-status-report.sh
|
||||
```
|
||||
|
||||
## `docker-critical-events-at-start`
|
||||
|
||||
Zeit: Array Start. Dieser Job startet einen Hintergrund-Watcher und beendet sich sofort.
|
||||
|
||||
Reference in New Issue
Block a user