#!/bin/bash # Test HA failover scenarios set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" if [ -f "$PROJECT_ROOT/.env" ]; then set +euo pipefail source "$PROJECT_ROOT/.env" 2>/dev/null || true set -euo pipefail fi PRIMARY_HOST="${PRIMARY_HOST:-192.168.11.11}" SECONDARY_HOST="${SECONDARY_HOST:-192.168.11.12}" VIP="${VIP:-192.168.11.166}" # Colors GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' RED='\033[0;31m' NC='\033[0m' log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[✓]${NC} $1"; } log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; } log_error() { echo -e "${RED}[✗]${NC} $1"; } echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "🧪 HA Failover Test" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" # Check initial state log_info "Checking initial VIP ownership..." if ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" "ip addr show vmbr0 2>/dev/null | grep -q $VIP"; then log_success "VIP $VIP is on primary host (expected)" INITIAL_OWNER="$PRIMARY_HOST" elif ssh -o StrictHostKeyChecking=no root@"$SECONDARY_HOST" "ip addr show vmbr0 2>/dev/null | grep -q $VIP"; then log_warn "VIP $VIP is on secondary host (unexpected)" INITIAL_OWNER="$SECONDARY_HOST" else log_error "VIP $VIP not found on either host" exit 1 fi # Test 1: Stop Keepalived on primary log_info "" log_info "Test 1: Stopping Keepalived on primary..." ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" "systemctl stop keepalived" || true sleep 10 # Check if VIP moved if ssh -o StrictHostKeyChecking=no root@"$SECONDARY_HOST" "ip addr show vmbr0 2>/dev/null | grep -q $VIP"; then log_success "VIP moved to secondary host (failover successful)" else log_error "VIP did not move to secondary" fi # Test connectivity log_info "Testing connectivity to VIP..." if curl -k -s -o /dev/null -w "%{http_code}" --max-time 5 "https://$VIP:81" | grep -qE "200|301|302"; then log_success "NPMplus accessible via VIP" else log_warn "NPMplus not accessible via VIP" fi # Restore primary log_info "" log_info "Restoring primary..." ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" "systemctl start keepalived" || true sleep 10 # Check if VIP moved back if ssh -o StrictHostKeyChecking=no root@"$PRIMARY_HOST" "ip addr show vmbr0 2>/dev/null | grep -q $VIP"; then log_success "VIP moved back to primary (failback successful)" else log_warn "VIP did not move back to primary" fi log_success "Failover test complete"