proxmox/scripts/archive/test/test-npmplus-full-connectivity.sh

#!/usr/bin/env bash
# Comprehensive network connectivity test for NPMplus
# Tests connectivity from all servers: Proxmox hosts, container, and backend services

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }

PROXMOX_HOST="${1:-192.168.11.11}"
CONTAINER_ID="${2:-10233}"
UDM_PRO_IP="${3:-192.168.11.1}"

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "🔍 Comprehensive NPMplus Connectivity Test"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Backend services to test
declare -A BACKEND_SERVICES=(
    ["${IP_BLOCKSCOUT:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-192.168.11.14}}}0}:80"]="VMID 5000 (blockscout-1) on ${PROXMOX_HOST_R630_02:-192.168.11.12}"
    ["${IP_DBIS_FRONTEND:-${IP_SERVICE_13:-${IP_SERVICE_13:-${IP_SERVICE_13:-192.168.11.13}}}0}:80"]="VMID 10130 (dbis-frontend) on ${PROXMOX_HOST_R630_01:-192.168.11.11}"
    ["${IP_DBIS_API:-192.168.11.155}:3000"]="VMID 10150 (dbis-api-primary) on ${PROXMOX_HOST_R630_01:-192.168.11.11}"
    ["${IP_DBIS_API_2:-192.168.11.156}:3000"]="VMID 10151 (dbis-api-secondary) on ${PROXMOX_HOST_R630_01:-192.168.11.11}"
    ["${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-192.168.11.36}}}}:80"]="VMID 7811 (mim-api-1) on ${PROXMOX_HOST_R630_02:-192.168.11.12}"
    ["${RPC_CORE_1:-${IP_SERVICE_21:-${IP_SERVICE_21:-${IP_SERVICE_21:-192.168.11.21}}}1}:443"]="VMID 2101 (besu-rpc-core-1) on ${PROXMOX_HOST_ML110:-192.168.11.10}"
    ["${RPC_PUBLIC_1:-192.168.11.221}:443"]="VMID 2201 (besu-rpc-public-1) on ${PROXMOX_HOST_ML110:-192.168.11.10}"
    ["${RPC_PRIVATE_1:-192.168.11.232}:443"]="VMID 2301 (besu-rpc-private-1) on ${PROXMOX_HOST_ML110:-192.168.11.10}"
    # Note: VMID 2302 (besu-rpc-private-2) - not in latest mapping, may need different IP or is new service
)

# Proxmox hosts to test from
PROXMOX_HOSTS=("${PROXMOX_HOST_R630_01:-192.168.11.11}" "${PROXMOX_HOST_ML110:-192.168.11.10}" "${PROXMOX_HOST_R630_02:-192.168.11.12}")

# Test 1: From Proxmox Host (${PROXMOX_HOST_R630_01:-192.168.11.11})
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Test 1: Connectivity from Proxmox Host ($PROXMOX_HOST)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

host_success=0
host_failed=0

for backend in "${!BACKEND_SERVICES[@]}"; do
    service_info="${BACKEND_SERVICES[$backend]}"
    ip="${backend%%:*}"
    port="${backend##*:}"
    
    log_info "Testing: $service_info"
    
    # Test ping
    ping_result=$(ssh root@"$PROXMOX_HOST" "ping -c 2 -W 2 $ip 2>&1" || echo "failed")
    
    if echo "$ping_result" | grep -q "2 received"; then
        log_success "  ✓ Ping successful"
        host_success=$((host_success + 1))
    else
        log_error "  ✗ Ping failed"
        host_failed=$((host_failed + 1))
    fi
done

echo ""
log_info "Proxmox Host Results: $host_success successful, $host_failed failed"
echo ""

# Test 2: From NPMplus Container
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Test 2: Connectivity from NPMplus Container ($CONTAINER_ID)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

CONTAINER_IP=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- hostname -I | awk '{print \$1}'" 2>/dev/null || echo "unknown")
log_info "Container IP: $CONTAINER_IP"
echo ""

container_success=0
container_failed=0

for backend in "${!BACKEND_SERVICES[@]}"; do
    service_info="${BACKEND_SERVICES[$backend]}"
    ip="${backend%%:*}"
    port="${backend##*:}"
    
    log_info "Testing: $service_info"
    
    # Test ping from container
    ping_result=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- ping -c 2 -W 2 $ip 2>&1" || echo "failed")
    
    if echo "$ping_result" | grep -q "2 received"; then
        log_success "  ✓ Ping successful"
        
        # Test port connectivity
        if [ "$port" = "443" ]; then
            port_test=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- timeout 3 bash -c '</dev/tcp/$ip/$port' 2>&1" || echo "failed")
        else
            port_test=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- timeout 3 bash -c '</dev/tcp/$ip/$port' 2>&1" || echo "failed")
        fi
        
        if [ "$port_test" = "" ]; then
            log_success "  ✓ Port $port accessible"
            container_success=$((container_success + 1))
        else
            log_warn "  ⚠️  Port $port not accessible"
        fi
    else
        log_error "  ✗ Ping failed: $(echo "$ping_result" | grep -E 'Unreachable|100%' | head -1)"
        container_failed=$((container_failed + 1))
    fi
    echo ""
done

echo ""
log_info "Container Results: $container_success successful, $container_failed failed"
echo ""

# Test 3: Container to Gateway
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Test 3: Container to Gateway Connectivity"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

log_info "Testing container to gateway ($UDM_PRO_IP)..."
GATEWAY_PING=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- ping -c 2 -W 2 $UDM_PRO_IP 2>&1" || echo "failed")

if echo "$GATEWAY_PING" | grep -q "2 received"; then
    log_success "Gateway is reachable from container"
else
    log_error "Gateway is NOT reachable from container"
    log_info "Output: $(echo "$GATEWAY_PING" | grep -E 'Unreachable|100%' | head -1)"
fi
echo ""

# Test 4: Check Container Network Configuration
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Test 4: Container Network Configuration"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

log_info "Container IP addresses:"
CONTAINER_IPS=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- hostname -I" 2>/dev/null || echo "unknown")
log_info "  $CONTAINER_IPS"
echo ""

log_info "Container routing table:"
CONTAINER_ROUTES=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- ip route show 2>/dev/null" || echo "failed")
echo "$CONTAINER_ROUTES" | while IFS= read -r line; do
    log_info "  $line"
done
echo ""

log_info "Container eth0 interface:"
ETH0_STATUS=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- ip addr show eth0 2>/dev/null | grep -E 'inet |state'" || echo "failed")
echo "$ETH0_STATUS" | while IFS= read -r line; do
    log_info "  $line"
done
echo ""

# Test 5: Check Proxmox Bridge VLAN Configuration
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Test 5: Proxmox Bridge VLAN Configuration"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

VETH_NAME=$(ssh root@"$PROXMOX_HOST" "bridge vlan show vmbr0 | grep veth | grep $CONTAINER_ID | awk '{print \$1}' | head -1" || echo "veth${CONTAINER_ID}i0")

log_info "Container veth interface: $VETH_NAME"
VETH_VLAN=$(ssh root@"$PROXMOX_HOST" "bridge vlan show $VETH_NAME 2>/dev/null | grep $VETH_NAME || bridge vlan show vmbr0 | grep $VETH_NAME" || echo "not found")
if echo "$VETH_VLAN" | grep -q "11.*PVID"; then
    log_success "Veth interface is on VLAN 11"
else
    log_warn "Veth interface may not be on VLAN 11"
    log_info "VLAN config: $(echo "$VETH_VLAN" | grep "$VETH_NAME" | head -1)"
fi
echo ""

log_info "Container network config:"
CONTAINER_NET=$(ssh root@"$PROXMOX_HOST" "pct config $CONTAINER_ID | grep net0" || echo "not found")
log_info "  $CONTAINER_NET"
echo ""

# Test 6: Test from Other Proxmox Hosts
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Test 6: Connectivity from Other Proxmox Hosts"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

for host in "${PROXMOX_HOSTS[@]}"; do
    if [ "$host" = "$PROXMOX_HOST" ]; then
        continue
    fi
    
    log_info "Testing from $host..."
    TEST_IP="${IP_BLOCKSCOUT}"  # Test with one backend service
    
    ping_result=$(ssh root@"$host" "ping -c 2 -W 2 $TEST_IP 2>&1" || echo "failed")
    
    if echo "$ping_result" | grep -q "2 received"; then
        log_success "  ✓ $host can reach backend services"
    else
        log_warn "  ⚠️  $host cannot reach backend services"
    fi
done
echo ""

# Test 7: ARP Table Check
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Test 7: ARP Table Check (Container)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

log_info "ARP entries in container:"
CONTAINER_ARP=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- arp -a 2>/dev/null | head -5" || echo "no entries")
echo "$CONTAINER_ARP" | while IFS= read -r line; do
    log_info "  $line"
done
echo ""

# Summary
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "📊 Test Summary"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""
log_info "Proxmox Host ($PROXMOX_HOST):"
log_info "  Successful: $host_success/${#BACKEND_SERVICES[@]}"
log_info "  Failed: $host_failed/${#BACKEND_SERVICES[@]}"
echo ""
log_info "NPMplus Container ($CONTAINER_ID):"
log_info "  Successful: $container_success/${#BACKEND_SERVICES[@]}"
log_info "  Failed: $container_failed/${#BACKEND_SERVICES[@]}"
echo ""

if [ $container_failed -eq 0 ]; then
    log_success "✅ All tests passed! Container can reach all backend services."
elif [ $host_success -gt 0 ] && [ $container_failed -gt 0 ]; then
    log_warn "⚠️  Container-specific networking issue detected"
    log_info "  Proxmox host can reach backends, but container cannot"
    log_info "  This suggests:"
    log_info "    1. UDM Pro firewall blocking container traffic"
    log_info "    2. Proxmox bridge VLAN tagging issue"
    log_info "    3. Container veth interface not properly configured"
else
    log_error "❌ Network connectivity issues detected"
fi
echo ""