#!/usr/bin/env bash # Comprehensive network connectivity test for NPMplus # Tests connectivity from all servers: Proxmox hosts, container, and backend services set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[✓]${NC} $1"; } log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; } log_error() { echo -e "${RED}[✗]${NC} $1"; } PROXMOX_HOST="${1:-192.168.11.11}" CONTAINER_ID="${2:-10233}" UDM_PRO_IP="${3:-192.168.11.1}" echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "🔍 Comprehensive NPMplus Connectivity Test" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" # Backend services to test declare -A BACKEND_SERVICES=( ["${IP_BLOCKSCOUT:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-192.168.11.14}}}0}:80"]="VMID 5000 (blockscout-1) on ${PROXMOX_HOST_R630_02:-192.168.11.12}" ["${IP_DBIS_FRONTEND:-${IP_SERVICE_13:-${IP_SERVICE_13:-${IP_SERVICE_13:-192.168.11.13}}}0}:80"]="VMID 10130 (dbis-frontend) on ${PROXMOX_HOST_R630_01:-192.168.11.11}" ["${IP_DBIS_API:-192.168.11.155}:3000"]="VMID 10150 (dbis-api-primary) on ${PROXMOX_HOST_R630_01:-192.168.11.11}" ["${IP_DBIS_API_2:-192.168.11.156}:3000"]="VMID 10151 (dbis-api-secondary) on ${PROXMOX_HOST_R630_01:-192.168.11.11}" ["${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-192.168.11.36}}}}:80"]="VMID 7811 (mim-api-1) on ${PROXMOX_HOST_R630_02:-192.168.11.12}" ["${RPC_CORE_1:-${IP_SERVICE_21:-${IP_SERVICE_21:-${IP_SERVICE_21:-192.168.11.21}}}1}:443"]="VMID 2101 (besu-rpc-core-1) on ${PROXMOX_HOST_ML110:-192.168.11.10}" ["${RPC_PUBLIC_1:-192.168.11.221}:443"]="VMID 2201 (besu-rpc-public-1) on ${PROXMOX_HOST_ML110:-192.168.11.10}" ["${RPC_PRIVATE_1:-192.168.11.232}:443"]="VMID 2301 (besu-rpc-private-1) on ${PROXMOX_HOST_ML110:-192.168.11.10}" # Note: VMID 2302 (besu-rpc-private-2) - not in latest mapping, may need different IP or is new service ) # Proxmox hosts to test from PROXMOX_HOSTS=("${PROXMOX_HOST_R630_01:-192.168.11.11}" "${PROXMOX_HOST_ML110:-192.168.11.10}" "${PROXMOX_HOST_R630_02:-192.168.11.12}") # Test 1: From Proxmox Host (${PROXMOX_HOST_R630_01:-192.168.11.11}) echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Test 1: Connectivity from Proxmox Host ($PROXMOX_HOST)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" host_success=0 host_failed=0 for backend in "${!BACKEND_SERVICES[@]}"; do service_info="${BACKEND_SERVICES[$backend]}" ip="${backend%%:*}" port="${backend##*:}" log_info "Testing: $service_info" # Test ping ping_result=$(ssh root@"$PROXMOX_HOST" "ping -c 2 -W 2 $ip 2>&1" || echo "failed") if echo "$ping_result" | grep -q "2 received"; then log_success " ✓ Ping successful" host_success=$((host_success + 1)) else log_error " ✗ Ping failed" host_failed=$((host_failed + 1)) fi done echo "" log_info "Proxmox Host Results: $host_success successful, $host_failed failed" echo "" # Test 2: From NPMplus Container echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Test 2: Connectivity from NPMplus Container ($CONTAINER_ID)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" CONTAINER_IP=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- hostname -I | awk '{print \$1}'" 2>/dev/null || echo "unknown") log_info "Container IP: $CONTAINER_IP" echo "" container_success=0 container_failed=0 for backend in "${!BACKEND_SERVICES[@]}"; do service_info="${BACKEND_SERVICES[$backend]}" ip="${backend%%:*}" port="${backend##*:}" log_info "Testing: $service_info" # Test ping from container ping_result=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- ping -c 2 -W 2 $ip 2>&1" || echo "failed") if echo "$ping_result" | grep -q "2 received"; then log_success " ✓ Ping successful" # Test port connectivity if [ "$port" = "443" ]; then port_test=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- timeout 3 bash -c '&1" || echo "failed") else port_test=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- timeout 3 bash -c '&1" || echo "failed") fi if [ "$port_test" = "" ]; then log_success " ✓ Port $port accessible" container_success=$((container_success + 1)) else log_warn " ⚠️ Port $port not accessible" fi else log_error " ✗ Ping failed: $(echo "$ping_result" | grep -E 'Unreachable|100%' | head -1)" container_failed=$((container_failed + 1)) fi echo "" done echo "" log_info "Container Results: $container_success successful, $container_failed failed" echo "" # Test 3: Container to Gateway echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Test 3: Container to Gateway Connectivity" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" log_info "Testing container to gateway ($UDM_PRO_IP)..." GATEWAY_PING=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- ping -c 2 -W 2 $UDM_PRO_IP 2>&1" || echo "failed") if echo "$GATEWAY_PING" | grep -q "2 received"; then log_success "Gateway is reachable from container" else log_error "Gateway is NOT reachable from container" log_info "Output: $(echo "$GATEWAY_PING" | grep -E 'Unreachable|100%' | head -1)" fi echo "" # Test 4: Check Container Network Configuration echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Test 4: Container Network Configuration" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" log_info "Container IP addresses:" CONTAINER_IPS=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- hostname -I" 2>/dev/null || echo "unknown") log_info " $CONTAINER_IPS" echo "" log_info "Container routing table:" CONTAINER_ROUTES=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- ip route show 2>/dev/null" || echo "failed") echo "$CONTAINER_ROUTES" | while IFS= read -r line; do log_info " $line" done echo "" log_info "Container eth0 interface:" ETH0_STATUS=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- ip addr show eth0 2>/dev/null | grep -E 'inet |state'" || echo "failed") echo "$ETH0_STATUS" | while IFS= read -r line; do log_info " $line" done echo "" # Test 5: Check Proxmox Bridge VLAN Configuration echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Test 5: Proxmox Bridge VLAN Configuration" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" VETH_NAME=$(ssh root@"$PROXMOX_HOST" "bridge vlan show vmbr0 | grep veth | grep $CONTAINER_ID | awk '{print \$1}' | head -1" || echo "veth${CONTAINER_ID}i0") log_info "Container veth interface: $VETH_NAME" VETH_VLAN=$(ssh root@"$PROXMOX_HOST" "bridge vlan show $VETH_NAME 2>/dev/null | grep $VETH_NAME || bridge vlan show vmbr0 | grep $VETH_NAME" || echo "not found") if echo "$VETH_VLAN" | grep -q "11.*PVID"; then log_success "Veth interface is on VLAN 11" else log_warn "Veth interface may not be on VLAN 11" log_info "VLAN config: $(echo "$VETH_VLAN" | grep "$VETH_NAME" | head -1)" fi echo "" log_info "Container network config:" CONTAINER_NET=$(ssh root@"$PROXMOX_HOST" "pct config $CONTAINER_ID | grep net0" || echo "not found") log_info " $CONTAINER_NET" echo "" # Test 6: Test from Other Proxmox Hosts echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Test 6: Connectivity from Other Proxmox Hosts" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" for host in "${PROXMOX_HOSTS[@]}"; do if [ "$host" = "$PROXMOX_HOST" ]; then continue fi log_info "Testing from $host..." TEST_IP="${IP_BLOCKSCOUT}" # Test with one backend service ping_result=$(ssh root@"$host" "ping -c 2 -W 2 $TEST_IP 2>&1" || echo "failed") if echo "$ping_result" | grep -q "2 received"; then log_success " ✓ $host can reach backend services" else log_warn " ⚠️ $host cannot reach backend services" fi done echo "" # Test 7: ARP Table Check echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Test 7: ARP Table Check (Container)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" log_info "ARP entries in container:" CONTAINER_ARP=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- arp -a 2>/dev/null | head -5" || echo "no entries") echo "$CONTAINER_ARP" | while IFS= read -r line; do log_info " $line" done echo "" # Summary echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "📊 Test Summary" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" log_info "Proxmox Host ($PROXMOX_HOST):" log_info " Successful: $host_success/${#BACKEND_SERVICES[@]}" log_info " Failed: $host_failed/${#BACKEND_SERVICES[@]}" echo "" log_info "NPMplus Container ($CONTAINER_ID):" log_info " Successful: $container_success/${#BACKEND_SERVICES[@]}" log_info " Failed: $container_failed/${#BACKEND_SERVICES[@]}" echo "" if [ $container_failed -eq 0 ]; then log_success "✅ All tests passed! Container can reach all backend services." elif [ $host_success -gt 0 ] && [ $container_failed -gt 0 ]; then log_warn "⚠️ Container-specific networking issue detected" log_info " Proxmox host can reach backends, but container cannot" log_info " This suggests:" log_info " 1. UDM Pro firewall blocking container traffic" log_info " 2. Proxmox bridge VLAN tagging issue" log_info " 3. Container veth interface not properly configured" else log_error "❌ Network connectivity issues detected" fi echo ""