diff --git a/ops/borg-ui/scripts/gitea-bundle-mirror.sh b/ops/borg-ui/scripts/gitea-bundle-mirror.sh index 769c373..b461e9a 100644 --- a/ops/borg-ui/scripts/gitea-bundle-mirror.sh +++ b/ops/borg-ui/scripts/gitea-bundle-mirror.sh @@ -87,13 +87,20 @@ main() { continue fi + rel="${repo#$SOURCE_ROOT/}" + if ! git --git-dir="$repo" show-ref --quiet; then + skipped=$((skipped + 1)) + printf 'SKIP\t%s\tempty repository without refs\n' "$rel" >> "$details" + printf '%s\t%s\t%s\t%s\n' "$total" "$bundled" "$skipped" "$failed" > "$run_tmp/counts" + continue + fi + target="$(bundle_target_for_repo "$repo")" target_dir="$(dirname "$target")" tmp="$run_tmp/$(basename "$target").tmp" target_tmp="$target_dir/.$(basename "$target").tmp" mkdir -p "$target_dir" - rel="${repo#$SOURCE_ROOT/}" log "Bundling $rel" if git --git-dir="$repo" bundle create "$tmp" --all >/dev/null 2>&1 && diff --git a/services/posture-check/cert-token-check.sh b/services/posture-check/cert-token-check.sh index 2af5a07..fc823f8 100755 --- a/services/posture-check/cert-token-check.sh +++ b/services/posture-check/cert-token-check.sh @@ -9,7 +9,7 @@ SEND_NTFY="${SEND_NTFY:-1}" CLOUDFLARE_TOKEN_FILE="${CLOUDFLARE_TOKEN_FILE:-/mnt/user/appdata/traefik/secrets/cloudflare_dns_api_token}" WARN_DAYS="${WARN_DAYS:-14}" CRITICAL_DAYS="${CRITICAL_DAYS:-7}" -DOMAINS="${DOMAINS:-traefik.kaleschke.info auth.kaleschke.info vault.kaleschke.info git.kaleschke.info cloud.kaleschke.info glance.kaleschke.info borg.kaleschke.info monitoring.kaleschke.info ntfy.kaleschke.info}" +DOMAINS="${DOMAINS:-traefik.kaleschke.info auth.kaleschke.info vault.kaleschke.info git.kaleschke.info cloud.kaleschke.info glance.kaleschke.info borg.kaleschke.info monitoring.kaleschke.info ntfy.kaleschke.info hc.kaleschke.info komodo.kaleschke.info files.kaleschke.info code.kaleschke.info glances.kaleschke.info scrutiny.kaleschke.info speedtest.kaleschke.info home.kaleschke.info plex.kaleschke.info pdf.kaleschke.info immich.kaleschke.info mealie.kaleschke.info n8n.kaleschke.info mail.kaleschke.info sp.kaleschke.info paperless.kaleschke.info paperless-gpt.kaleschke.info}" TMP_DIR="${TMP_DIR:-/tmp/kallilab-cert-token-check}" mkdir -p "$TMP_DIR" diff --git a/services/posture-check/compose-runtime-drift.sh b/services/posture-check/compose-runtime-drift.sh index c156f7d..f6cb2d4 100755 --- a/services/posture-check/compose-runtime-drift.sh +++ b/services/posture-check/compose-runtime-drift.sh @@ -26,7 +26,73 @@ add_result() { printf '%s\t%s\t%s\n' "$1" "$2" "$3" >> "$RESULTS_FILE" } -parse_compose() { +parse_normalized_compose() { + awk ' + function clean(value) { + gsub(/\r/, "", value) + gsub(/["'\''"]/, "", value) + return value + } + function emit() { + if (in_services && service && image) { + print clean(container) "\t" clean(image) + } + } + /^services:/ { + emit() + in_services=1 + service="" + image="" + container="" + next + } + /^[A-Za-z0-9_.-]+:/ && $0 !~ /^services:/ { + if (in_services) { + emit() + in_services=0 + service="" + image="" + container="" + } + } + in_services && /^ [A-Za-z0-9_.-]+:/ { + emit() + service=$1 + sub(/:$/, "", service) + image="" + container=service + next + } + in_services && service && /^ image:/ { + image=$0 + sub(/^[[:space:]]*image:[[:space:]]*/, "", image) + next + } + in_services && service && /^ container_name:/ { + container=$0 + sub(/^[[:space:]]*container_name:[[:space:]]*/, "", container) + next + } + END { emit() } + ' +} + +parse_compose_with_docker() { + local compose="$1" + local dir + local file + + command -v docker >/dev/null 2>&1 || return 1 + + dir="$(dirname "$compose")" + file="$(basename "$compose")" + ( + cd "$dir" + docker compose -f "$file" config 2>/dev/null + ) | parse_normalized_compose +} + +parse_compose_raw() { local compose="$1" awk ' function clean(value) { @@ -36,9 +102,25 @@ parse_compose() { } function emit() { if (service && image && !has_profile) { + if (image ~ /^\*/) { + alias=image + sub(/^\*/, "", alias) + if (alias in anchors) { + image=anchors[alias] + } + } print clean(container) "\t" clean(image) } } + /^x-[A-Za-z0-9_.-]+:[[:space:]]*&[A-Za-z0-9_.-]+[[:space:]]+/ { + alias=$0 + sub(/^.*&/, "", alias) + sub(/[[:space:]].*$/, "", alias) + value=$0 + sub(/^.*&[A-Za-z0-9_.-]+[[:space:]]+/, "", value) + anchors[alias]=value + next + } /^ [A-Za-z0-9_.-]+:/ { emit() service=$1 @@ -66,6 +148,16 @@ parse_compose() { ' "$compose" } +parse_compose() { + local compose="$1" + + if parse_compose_with_docker "$compose"; then + return 0 + fi + + parse_compose_raw "$compose" +} + while IFS= read -r -d '' compose; do while IFS="$(printf '\t')" read -r container expected_image; do [ -n "$container" ] || continue diff --git a/services/posture-check/komodo-stack-hygiene.sh b/services/posture-check/komodo-stack-hygiene.sh index 03c448a..f5a6906 100644 --- a/services/posture-check/komodo-stack-hygiene.sh +++ b/services/posture-check/komodo-stack-hygiene.sh @@ -15,6 +15,8 @@ NTFY_TOPIC="${NTFY_TOPIC:-homelab-alerts}" SEND_NTFY="${SEND_NTFY:-1}" KOMODO_ENV_FILE="${KOMODO_ENV_FILE:-/mnt/user/appdata/secrets/codex_komodo_api.env}" KOMODO_CONTAINER="${KOMODO_CONTAINER:-komodo-core}" +KOMODO_CLI_HOST_FOR_CONTAINER="${KOMODO_CLI_HOST_FOR_CONTAINER:-http://localhost:9120}" +FETCH_BEFORE_DIFF="${FETCH_BEFORE_DIFF:-1}" # Komma-separierte Allowlist fuer bewusst inline-managed Stacks. # Quelle: memory/komodo-stack-inline-managed.md, CLAUDE.md. @@ -32,12 +34,15 @@ TMP_DIR="${TMP_DIR:-/tmp/kallilab-komodo-stack-hygiene}" mkdir -p "$TMP_DIR" RESULTS_FILE="$TMP_DIR/results.$$" STACKS_FILE="$TMP_DIR/stacks.$$.json" +API_ERROR_FILE="$TMP_DIR/komodo-api.$$.err" +API_UNREACHABLE=0 : > "$RESULTS_FILE" +: > "$API_ERROR_FILE" # Healthchecks Heartbeat (endpoint-agnostisch; Capability-URL ist ein Secret, nie ins Repo) HC_URL_FILE="${HC_URL_FILE:-/mnt/user/appdata/secrets/healthchecks_komodo_hygiene_url}" hc_url=""; [ -r "$HC_URL_FILE" ] && hc_url="$(tr -d '[:space:]' < "$HC_URL_FILE")" hc_ping() { [ -n "$hc_url" ] || return 0; curl -fsS -m 10 --retry 3 "${hc_url}${1:-}" >/dev/null 2>&1 || true; } -trap 'hc_rc=$?; rm -f "$RESULTS_FILE" "$STACKS_FILE"; [ "$hc_rc" -le 2 ] && hc_ping "" || hc_ping "/fail"' EXIT +trap 'hc_rc=$?; rm -f "$RESULTS_FILE" "$STACKS_FILE" "$API_ERROR_FILE"; [ "$hc_rc" -le 2 ] && hc_ping "" || hc_ping "/fail"' EXIT hc_ping "/start" json_escape() { @@ -73,11 +78,21 @@ is_expected_not_in_komodo() { stack_files_changed() { local name="$1" deployed="$2" latest="$3" local dir + HASH_COMPARE_REASON="" # Locate the stack's compose dir (case-insensitive, same as Mode 3). dir="$(find "$REPO_ROOT" -type d -iname "$name" -not -path "*/.git/*" 2>/dev/null | head -1)" - [ -n "$dir" ] || return 0 # No dir -> can't tell, treat as drift to be safe - ( cd "$REPO_ROOT" && git rev-parse --verify --quiet "$deployed" >/dev/null ) || return 0 - ( cd "$REPO_ROOT" && git rev-parse --verify --quiet "$latest" >/dev/null ) || return 0 + if [ -z "$dir" ]; then + HASH_COMPARE_REASON="no compose directory found for stack" + return 1 + fi + if ! ( cd "$REPO_ROOT" && git rev-parse --verify --quiet "$deployed" >/dev/null ); then + HASH_COMPARE_REASON="deployed_hash $deployed is not available in local repo" + return 1 + fi + if ! ( cd "$REPO_ROOT" && git rev-parse --verify --quiet "$latest" >/dev/null ); then + HASH_COMPARE_REASON="latest_hash $latest is not available in local repo" + return 1 + fi local rel="${dir#$REPO_ROOT/}" if ( cd "$REPO_ROOT" && git diff --quiet "$deployed".."$latest" -- "$rel" ); then return 1 # no change @@ -85,20 +100,29 @@ stack_files_changed() { return 0 # real change } +if [ "$FETCH_BEFORE_DIFF" = "1" ]; then + if ! ( cd "$REPO_ROOT" && git fetch --quiet origin >/dev/null 2>&1 ); then + add_result "warning" "repo-fetch" "Could not fetch origin before hash drift comparisons" + fi +fi + # Komodo-API-Credentials laden und Stack-Liste holen. if [ ! -r "$KOMODO_ENV_FILE" ]; then - add_result "warning" "komodo-api" "Komodo env file not readable: $KOMODO_ENV_FILE" + API_UNREACHABLE=1 + add_result "critical" "komodo-api" "Komodo env file not readable: $KOMODO_ENV_FILE" else set -a # shellcheck disable=SC1090 . "$KOMODO_ENV_FILE" set +a if ! docker exec \ - -e KOMODO_CLI_HOST \ + -e "KOMODO_CLI_HOST=$KOMODO_CLI_HOST_FOR_CONTAINER" \ -e KOMODO_CLI_KEY \ -e KOMODO_CLI_SECRET \ - "$KOMODO_CONTAINER" km list -a stacks -f json > "$STACKS_FILE" 2>/dev/null; then - add_result "warning" "komodo-api" "km list stacks failed (container=$KOMODO_CONTAINER)" + "$KOMODO_CONTAINER" km list -a stacks -f json > "$STACKS_FILE" 2>"$API_ERROR_FILE"; then + API_UNREACHABLE=1 + api_error="$(tr '\n' ' ' < "$API_ERROR_FILE" | sed -E 's/[[:space:]]+/ /g' | cut -c 1-180)" + add_result "critical" "komodo-api" "km list stacks failed (container=$KOMODO_CONTAINER): ${api_error:-unknown error}" : > "$STACKS_FILE" fi fi @@ -150,9 +174,12 @@ if [ -s "$STACKS_FILE" ]; then # ohne Stack-Inhalt aendert nichts und ist kein echter Drift. # "-" = unbekannt (z.B. gitea self-host edge case), nicht als Drift werten. if [ "$deployed_hash" != "-" ] && [ "$latest_hash" != "-" ] \ - && [ "$deployed_hash" != "$latest_hash" ] \ - && stack_files_changed "$name" "$deployed_hash" "$latest_hash"; then - add_result "warning" "$name" "deployed_hash $deployed_hash != latest_hash $latest_hash (stack files changed)" + && [ "$deployed_hash" != "$latest_hash" ]; then + if stack_files_changed "$name" "$deployed_hash" "$latest_hash"; then + add_result "warning" "$name" "deployed_hash $deployed_hash != latest_hash $latest_hash (stack files changed)" + elif [ -n "${HASH_COMPARE_REASON:-}" ]; then + add_result "warning" "$name" "deployed_hash $deployed_hash != latest_hash $latest_hash (cannot compare: $HASH_COMPARE_REASON)" + fi fi # Failure-Mode 5: Stack ist down. @@ -237,6 +264,9 @@ if [ "$critical_count" -gt 0 ] || [ "$warning_count" -gt 0 ]; then "Komodo stack hygiene: $critical_count critical, $warning_count warning" \ "See $OUTPUT_PATH" "$priority" || true fi + # If Komodo could not be queried at all, the hygiene monitor itself is broken. + # Use rc=3 so the Healthchecks EXIT trap sends /fail instead of a green ping. + [ "$API_UNREACHABLE" -eq 1 ] && exit 3 [ "$critical_count" -gt 0 ] && exit 2 exit 1 fi diff --git a/services/posture-check/unraid-user-scripts.md b/services/posture-check/unraid-user-scripts.md index aa5fadb..3939c40 100644 --- a/services/posture-check/unraid-user-scripts.md +++ b/services/posture-check/unraid-user-scripts.md @@ -93,6 +93,8 @@ User Script: ```bash #!/bin/bash SEND_MAIL=1 \ +SEND_NTFY=1 \ +NTFY_TOPIC="homelab-info" \ MAIL_MODE=always \ INCLUDE_WEATHER_REPORT=1 \ MAIL_FROM="michideheld@gmx.de" \