#!/bin/bash
# Comprehensive verification script for all Proxmox nodes
# Checks cluster membership, storage, and service status
# Usage: ./scripts/verify-all-nodes-complete.sh

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }

# Host configuration
declare -A HOSTS
HOSTS[ml110]="192.168.11.10:L@kers2010"
HOSTS[r630-01]="192.168.11.11:password"
HOSTS[r630-02]="192.168.11.12:password"
HOSTS[r630-03]="192.168.11.13:L@kers2010"
HOSTS[r630-04]="192.168.11.14:L@kers2010"

log_info "========================================="
log_info "Complete Proxmox Cluster Verification"
log_info "========================================="
echo ""

verify_node() {
    local hostname="$1"
    local ip="${HOSTS[$hostname]%%:*}"
    local password="${HOSTS[$hostname]#*:}"
    
    log_info "=== Verifying ${hostname} (${ip}) ==="
    echo ""
    
    # Test connectivity
    if ! ping -c 2 -W 2 "$ip" >/dev/null 2>&1; then
        log_error "${hostname} is NOT reachable"
        echo ""
        return 1
    fi
    
    # Test SSH
    if ! sshpass -p "$password" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@"$ip" "echo 'SSH OK'" >/dev/null 2>&1; then
        log_warn "${hostname} SSH access failed (password may be incorrect)"
        echo ""
        return 1
    fi
    
    # Run verification
    sshpass -p "$password" ssh -o StrictHostKeyChecking=no root@"$ip" bash <<'ENDSSH'
        echo "--- System Information ---"
        echo "Hostname: $(hostname)"
        echo "IP: $(hostname -I | awk '{print $1}')"
        echo "Proxmox Version: $(pveversion 2>/dev/null || echo 'Not installed')"
        echo ""
        
        echo "--- Cluster Membership ---"
        if command -v pvecm >/dev/null 2>&1; then
            pvecm status 2>&1 | head -15 || echo "Not in cluster"
            echo ""
            echo "Cluster Nodes:"
            pvecm nodes 2>&1 || echo "Cannot list nodes"
        else
            echo "pvecm not available"
        fi
        echo ""
        
        echo "--- Proxmox Services ---"
        for service in pve-cluster pvestatd pvedaemon pveproxy; do
            if systemctl is-active --quiet $service 2>/dev/null; then
                echo "✓ $service: Active"
            elif systemctl is-enabled --quiet $service 2>/dev/null; then
                echo "✗ $service: Inactive"
            else
                echo "? $service: Not found"
            fi
        done
        echo ""
        
        echo "--- Storage Status ---"
        pvesm status 2>&1 | head -20 || echo "Cannot list storage"
        echo ""
        
        echo "--- Web Interface ---"
        if ss -tlnp | grep -q ":8006"; then
            echo "✓ Port 8006: Listening"
            HTTP_CODE=$(curl -k -s -o /dev/null -w "%{http_code}" https://localhost:8006/ 2>/dev/null || echo "000")
            if [[ "$HTTP_CODE" =~ ^(200|401|302)$ ]]; then
                echo "✓ Web Interface: Accessible (HTTP $HTTP_CODE)"
            else
                echo "✗ Web Interface: Not accessible (HTTP $HTTP_CODE)"
            fi
        else
            echo "✗ Port 8006: Not listening"
        fi
        echo ""
ENDSSH
    
    echo "----------------------------------------"
    echo ""
}

# Verify all nodes
for hostname in ml110 r630-01 r630-02 r630-03 r630-04; do
    verify_node "$hostname" || true
done

# Cluster summary
log_info "=== Cluster Summary ==="
echo ""

sshpass -p "L@kers2010" ssh -o StrictHostKeyChecking=no root@192.168.11.10 bash <<'ENDSSH'
    echo "Cluster Name: h"
    echo ""
    echo "Cluster Status:"
    pvecm status 2>&1 | head -10
    echo ""
    echo "Cluster Nodes:"
    pvecm nodes 2>&1
    echo ""
ENDSSH

log_success "Verification complete!"
echo ""
log_info "Review the output above for:"
log_info "  - Cluster membership status"
log_info "  - Storage configuration"
log_info "  - Service status"
log_info "  - Web interface accessibility"
echo ""