dbd517b279
- Update dbis_core, cross-chain-pmm-lps, explorer-monorepo, metamask-integration, pr-workspace/chains - Omit embedded publish git dirs and empty placeholders from index Made-with: Cursor
74 lines
2.6 KiB
Bash
Executable File
74 lines
2.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Simple Alert Script
|
|
# Sends alerts when Besu services are down
|
|
# Can be extended to send email, Slack, etc.
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
# Configuration
|
|
ALERT_EMAIL="${ALERT_EMAIL:-}"
|
|
ALERT_LOG="$PROJECT_ROOT/logs/alerts.log"
|
|
ALERT_SENT_LOG="$PROJECT_ROOT/logs/alerts-sent.log"
|
|
|
|
# Ensure log directory exists
|
|
mkdir -p "$(dirname "$ALERT_LOG")"
|
|
|
|
log_alert() {
|
|
local message="$1"
|
|
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
|
echo "[$timestamp] ALERT: $message" >> "$ALERT_LOG"
|
|
|
|
# Check if we've already sent this alert (avoid spam)
|
|
local alert_key=$(echo "$message" | md5sum | cut -d' ' -f1)
|
|
if ! grep -q "$alert_key" "$ALERT_SENT_LOG" 2>/dev/null; then
|
|
echo "[$timestamp] $alert_key" >> "$ALERT_SENT_LOG"
|
|
|
|
# Send email if configured
|
|
if [[ -n "$ALERT_EMAIL" ]] && command -v mail >/dev/null 2>&1; then
|
|
echo "$message" | mail -s "Besu Alert: Container Issue" "$ALERT_EMAIL" 2>/dev/null || true
|
|
fi
|
|
|
|
# Log to console
|
|
echo "ALERT: $message"
|
|
fi
|
|
}
|
|
|
|
# Check all containers
|
|
for vmid in 1000 1001 1002 1003 1004 1500 1501 1502 1503 2101 2102 2103 2201 2301 2303 2304 2305 2306 2307 2308 2400 2401 2402 2403 2420 2430 2440 2460 2470 2480; do
|
|
# Check if container is running
|
|
if ! pct status "$vmid" 2>/dev/null | grep -q running; then
|
|
log_alert "Container $vmid is not running"
|
|
continue
|
|
fi
|
|
|
|
# Determine service name
|
|
service_name=""
|
|
if [[ $vmid -ge 1000 ]] && [[ $vmid -le 1004 ]]; then
|
|
service_name="besu-validator"
|
|
elif [[ $vmid -ge 1500 ]] && [[ $vmid -le 1503 ]]; then
|
|
service_name="besu-sentry"
|
|
elif [[ $vmid -ge 2101 ]] && [[ $vmid -le 2480 ]]; then
|
|
service_name="besu-rpc"
|
|
fi
|
|
|
|
# Check service status
|
|
if [[ -n "$service_name" ]]; then
|
|
if ! pct exec "$vmid" -- systemctl is-active --quiet "$service_name" 2>/dev/null; then
|
|
log_alert "Service $service_name on container $vmid is not running"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
# Check disk space (alert if < 10% free)
|
|
for vmid in 1000 1001 1002 1003 1004 1500 1501 1502 1503 2101 2102 2103 2201 2301 2303 2304 2305 2306 2307 2308 2400 2401 2402 2403 2420 2430 2440 2460 2470 2480; do
|
|
if pct status "$vmid" 2>/dev/null | grep -q running; then
|
|
disk_usage=$(pct exec "$vmid" -- df -h / | awk 'NR==2 {print $5}' | sed 's/%//' 2>/dev/null || echo "0")
|
|
if [[ $disk_usage -gt 90 ]]; then
|
|
log_alert "Container $vmid disk usage is at ${disk_usage}%"
|
|
fi
|
|
fi
|
|
done
|