proxmox/scripts/archive/consolidated/deploy/setup-alerting.sh

#!/usr/bin/env bash
# Setup Alerting System
# Configures email and webhook alerts for monitoring

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
CONFIG_FILE="${PROJECT_ROOT}/smom-dbis-138/.env.alerts"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }

create_alert_config() {
    log_info "Creating alerting configuration..."
    
    if [ -f "$CONFIG_FILE" ]; then
        log_warn "Alert config already exists: $CONFIG_FILE"
        read -p "Overwrite? (y/N): " -n 1 -r
        echo
        if [[ ! $REPLY =~ ^[Yy]$ ]]; then
            log_info "Skipping config creation"
            return 0
        fi
    fi
    
    log_info "Creating alert configuration file..."
    
    cat > "$CONFIG_FILE" <<'EOF'
# Alerting Configuration
# Configure alert channels for blockchain monitoring

# Email Alerts
ALERT_EMAIL_ENABLED=false
ALERT_EMAIL_TO="admin@example.com"
ALERT_EMAIL_FROM="alerts@blockchain.local"
ALERT_EMAIL_SMTP_HOST="smtp.example.com"
ALERT_EMAIL_SMTP_PORT=587
ALERT_EMAIL_SMTP_USER=""
ALERT_EMAIL_SMTP_PASS=""

# Webhook Alerts
ALERT_WEBHOOK_ENABLED=false
ALERT_WEBHOOK_URL=""
ALERT_WEBHOOK_METHOD="POST"
ALERT_WEBHOOK_HEADERS="Content-Type: application/json"

# Slack Webhook (example)
ALERT_SLACK_ENABLED=false
ALERT_SLACK_WEBHOOK_URL=""

# Discord Webhook (example)
ALERT_DISCORD_ENABLED=false
ALERT_DISCORD_WEBHOOK_URL=""

# Alert Thresholds
ALERT_BLOCK_STALL_SECONDS=60
ALERT_VALIDATOR_DOWN_MINUTES=5
ALERT_TRANSACTION_STUCK_MINUTES=10
ALERT_QUORUM_LOST=true
EOF
    
    log_success "Alert configuration created: $CONFIG_FILE"
    log_warn "Please edit $CONFIG_FILE to configure your alert channels"
}

setup_email_alerting() {
    log_info "Setting up email alerting..."
    
    # Check if mail command is available
    if ! command -v mail &> /dev/null; then
        log_warn "mail command not found, installing mailutils..."
        if command -v apt-get &> /dev/null; then
            log_info "To enable email alerts, install mailutils:"
            log_info "  sudo apt-get install mailutils"
        fi
    else
        log_success "Email alerting ready (mail command available)"
    fi
}

setup_webhook_alerting() {
    log_info "Setting up webhook alerting..."
    
    # Check if curl is available
    if ! command -v curl &> /dev/null; then
        log_error "curl command not found - required for webhook alerts"
        return 1
    fi
    
    log_success "Webhook alerting ready (curl available)"
}

update_alert_scripts() {
    log_info "Updating alert scripts to use configuration..."
    
    # Update alert-block-stall.sh to source config
    if [ -f "$SCRIPT_DIR/alert-block-stall.sh" ]; then
        if ! grep -q "\.env.alerts" "$SCRIPT_DIR/alert-block-stall.sh"; then
            log_info "Updating alert-block-stall.sh to use config..."
            # Add config loading at the top
            sed -i '2a\
# Load alert configuration\
if [ -f "$SCRIPT_DIR/../smom-dbis-138/.env.alerts" ]; then\
    source "$SCRIPT_DIR/../smom-dbis-138/.env.alerts"\
fi
' "$SCRIPT_DIR/alert-block-stall.sh" 2>/dev/null || true
        fi
    fi
    
    log_success "Alert scripts updated"
}

create_send_alert_function() {
    log_info "Creating send-alert.sh utility..."
    
    cat > "$SCRIPT_DIR/send-alert.sh" <<'ALERTSCRIPT'
#!/usr/bin/env bash
# Universal Alert Sender
# Sends alerts via configured channels

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
CONFIG_FILE="${PROJECT_ROOT}/smom-dbis-138/.env.alerts"

# Load configuration
if [ -f "$CONFIG_FILE" ]; then
    source "$CONFIG_FILE"
fi

SEVERITY="${1:-WARNING}"
TITLE="${2:-Alert}"
MESSAGE="${3:-}"

send_email_alert() {
    if [ "${ALERT_EMAIL_ENABLED:-false}" != "true" ]; then
        return 0
    fi
    
    local to="${ALERT_EMAIL_TO:-}"
    if [ -z "$to" ]; then
        return 0
    fi
    
    echo "$MESSAGE" | mail -s "[$SEVERITY] $TITLE" "$to" 2>/dev/null || true
}

send_webhook_alert() {
    local url="${1:-}"
    if [ -z "$url" ] || [ "${ALERT_WEBHOOK_ENABLED:-false}" != "true" ]; then
        return 0
    fi
    
    local payload=$(cat <<EOF
{
  "severity": "$SEVERITY",
  "title": "$TITLE",
  "message": "$MESSAGE",
  "timestamp": "$(date -Iseconds)"
}
EOF
)
    
    curl -X POST "$url" \
        -H "Content-Type: application/json" \
        -d "$payload" \
        2>/dev/null || true
}

send_slack_alert() {
    if [ "${ALERT_SLACK_ENABLED:-false}" != "true" ]; then
        return 0
    fi
    
    local webhook="${ALERT_SLACK_WEBHOOK_URL:-}"
    if [ -z "$webhook" ]; then
        return 0
    fi
    
    local color="warning"
    case "$SEVERITY" in
        CRITICAL|ERROR) color="danger" ;;
        WARNING) color="warning" ;;
        INFO) color="good" ;;
    esac
    
    local payload=$(cat <<EOF
{
  "attachments": [{
    "color": "$color",
    "title": "$TITLE",
    "text": "$MESSAGE",
    "footer": "Blockchain Monitoring",
    "ts": $(date +%s)
  }]
}
EOF
)
    
    curl -X POST "$webhook" \
        -H "Content-Type: application/json" \
        -d "$payload" \
        2>/dev/null || true
}

send_discord_alert() {
    if [ "${ALERT_DISCORD_ENABLED:-false}" != "true" ]; then
        return 0
    fi
    
    local webhook="${ALERT_DISCORD_WEBHOOK_URL:-}"
    if [ -z "$webhook" ]; then
        return 0
    fi
    
    local payload=$(cat <<EOF
{
  "embeds": [{
    "title": "$TITLE",
    "description": "$MESSAGE",
    "color": $([ "$SEVERITY" = "CRITICAL" ] && echo "16711680" || echo "16776960"),
    "timestamp": "$(date -Iseconds)"
  }]
}
EOF
)
    
    curl -X POST "$webhook" \
        -H "Content-Type: application/json" \
        -d "$payload" \
        2>/dev/null || true
}

# Send alerts via all enabled channels
send_email_alert
send_webhook_alert "${ALERT_WEBHOOK_URL:-}"
send_slack_alert
send_discord_alert

ALERTSCRIPT
    
    chmod +x "$SCRIPT_DIR/send-alert.sh"
    log_success "send-alert.sh utility created"
}

main() {
    log_info "Setting up alerting system..."
    echo ""
    
    create_alert_config
    echo ""
    
    setup_email_alerting
    echo ""
    
    setup_webhook_alerting
    echo ""
    
    update_alert_scripts
    echo ""
    
    create_send_alert_function
    echo ""
    
    log_success "Alerting system setup complete!"
    log_warn "Next steps:"
    log_warn "  1. Edit $CONFIG_FILE to configure your alert channels"
    log_warn "  2. Test alerts with: ./scripts/monitoring/send-alert.sh CRITICAL 'Test' 'This is a test alert'"
}

main "$@"