#!/usr/bin/env bash set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true # Comprehensive RPC node peer count analysis # Usage: ./analyze-all-rpc-peers.sh [proxmox-host] set -e PROXMOX_HOST="${1:-pve2}" echo "==========================================" echo "RPC Node Peer Count Analysis" echo "==========================================" echo "" # Function to execute command on Proxmox host exec_proxmox() { if command -v pct &>/dev/null; then eval "$@" else ssh root@$PROXMOX_HOST "$@" fi } declare -A NODES=( ["2101"]="${RPC_CORE_1}" ["2201"]="${RPC_PUBLIC_1}" ["2303"]="${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-${RPC_NODE_233:-192.168.11.233}}}}}}}" ["2304"]="${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-${RPC_NODE_234:-192.168.11.234}}}}}}}" ["2305"]="${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-${RPC_NODE_235:-192.168.11.235}}}}}}}" ["2306"]="${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-${RPC_NODE_236:-192.168.11.236}}}}}}}" ["2307"]="${IP_RPC_237:-${IP_RPC_237:-${IP_RPC_237:-192.168.11.237}}}" ["2308"]="${IP_RPC_238:-${IP_RPC_238:-${IP_RPC_238:-192.168.11.238}}}" ["2400"]="${RPC_THIRDWEB_PRIMARY}" ["2401"]="${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-${RPC_THIRDWEB_1:-192.168.11.241}}}}}}}" ["2402"]="${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-${RPC_THIRDWEB_2:-192.168.11.242}}}}}}}" ["2403"]="${RPC_THIRDWEB_3:-${RPC_THIRDWEB_3:-${RPC_THIRDWEB_3:-192.168.11.243}}}" ) declare -A PEER_COUNTS declare -A BLOCK_NUMBERS declare -A STATUS echo "Gathering data from all RPC nodes..." echo "" for VMID in "${!NODES[@]}"; do IP="${NODES[$VMID]}" # Get block number BLOCK_RESPONSE=$(exec_proxmox "curl -s -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":1}' http://$IP:8545 2>/dev/null" || echo "") if [ -n "$BLOCK_RESPONSE" ]; then BLOCK_HEX=$(echo "$BLOCK_RESPONSE" | grep -o '"result":"[^"]*"' | cut -d'"' -f4) if [ -n "$BLOCK_HEX" ]; then BLOCK_NUM=$(printf "%d" $BLOCK_HEX 2>/dev/null || echo "0") BLOCK_NUMBERS[$VMID]=$BLOCK_NUM else BLOCK_NUMBERS[$VMID]="N/A" fi else BLOCK_NUMBERS[$VMID]="N/A" STATUS[$VMID]="❌ Not responding" continue fi # Get peer count PEER_RESPONSE=$(exec_proxmox "curl -s -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"net_peerCount\",\"params\":[],\"id\":1}' http://$IP:8545 2>/dev/null" || echo "") if [ -n "$PEER_RESPONSE" ]; then PEER_HEX=$(echo "$PEER_RESPONSE" | grep -o '"result":"[^"]*"' | cut -d'"' -f4) if [ -n "$PEER_HEX" ]; then PEER_NUM=$(printf "%d" $PEER_HEX 2>/dev/null || echo "0") PEER_COUNTS[$VMID]=$PEER_NUM else PEER_COUNTS[$VMID]="0" fi else PEER_COUNTS[$VMID]="N/A" fi # Determine status if [ "${PEER_COUNTS[$VMID]}" = "N/A" ] || [ "${BLOCK_NUMBERS[$VMID]}" = "N/A" ]; then STATUS[$VMID]="❌ Not responding" elif [ "${PEER_COUNTS[$VMID]}" -ge 7 ]; then STATUS[$VMID]="✅ Excellent" elif [ "${PEER_COUNTS[$VMID]}" -ge 5 ]; then STATUS[$VMID]="✅ Good" elif [ "${PEER_COUNTS[$VMID]}" -ge 3 ]; then STATUS[$VMID]="⚠️ Acceptable" elif [ "${PEER_COUNTS[$VMID]}" -ge 1 ]; then STATUS[$VMID]="⚠️ Warning" else STATUS[$VMID]="❌ Critical" fi done # Display results echo "==========================================" echo "Peer Count Analysis Results" echo "==========================================" printf "%-6s | %-15s | %-12s | %-6s | %-15s\n" "VMID" "IP Address" "Block Height" "Peers" "Status" echo "----------------------------------------------------------------------------" for VMID in "${!NODES[@]}"; do IP="${NODES[$VMID]}" BLOCK="${BLOCK_NUMBERS[$VMID]}" PEERS="${PEER_COUNTS[$VMID]}" STAT="${STATUS[$VMID]}" printf "%-6s | %-15s | %-12s | %-6s | %-15s\n" "$VMID" "$IP" "$BLOCK" "$PEERS" "$STAT" done echo "" echo "==========================================" echo "Summary" echo "==========================================" echo "" # Group by peer count EXCELLENT=0 GOOD=0 ACCEPTABLE=0 WARNING=0 CRITICAL=0 NON_RESPONDING=0 for VMID in "${!STATUS[@]}"; do case "${STATUS[$VMID]}" in *Excellent*) ((EXCELLENT++)) ;; *Good*) ((GOOD++)) ;; *Acceptable*) ((ACCEPTABLE++)) ;; *Warning*) ((WARNING++)) ;; *Critical*) ((CRITICAL++)) ;; *Not responding*) ((NON_RESPONDING++)) ;; esac done echo "✅ Excellent (7+ peers): $EXCELLENT nodes" echo "✅ Good (5-6 peers): $GOOD nodes" echo "⚠️ Acceptable (3-4 peers): $ACCEPTABLE nodes" echo "⚠️ Warning (1-2 peers): $WARNING nodes" echo "❌ Critical (0 peers): $CRITICAL nodes" echo "❌ Not responding: $NON_RESPONDING nodes" echo "" # Expected peer count echo "==========================================" echo "Expected Peer Count" echo "==========================================" echo "" echo "Network Size: ~19-20 active nodes" echo "" echo "Recommended Peer Counts:" echo " - Minimum healthy: 2-3 peers" echo " - Recommended: 5-7 peers ✅" echo " - Maximum: 20-25 peers (max-peers setting)" echo "" # Analysis echo "==========================================" echo "Analysis" echo "==========================================" echo "" # Check for block height mismatches MAIN_BLOCK="" MAIN_COUNT=0 for VMID in "${!BLOCK_NUMBERS[@]}"; do BLOCK="${BLOCK_NUMBERS[$VMID]}" if [ "$BLOCK" != "N/A" ] && [[ "$BLOCK" =~ ^[0-9]+$ ]]; then # Find most common block height COUNT=0 for VMID2 in "${!BLOCK_NUMBERS[@]}"; do if [ "${BLOCK_NUMBERS[$VMID2]}" = "$BLOCK" ]; then ((COUNT++)) fi done if [ $COUNT -gt $MAIN_COUNT ]; then MAIN_COUNT=$COUNT MAIN_BLOCK=$BLOCK fi fi done echo "Main network block height: $MAIN_BLOCK (${MAIN_COUNT} nodes)" echo "" # Check for nodes ahead or behind AHEAD=() BEHIND=() for VMID in "${!BLOCK_NUMBERS[@]}"; do BLOCK="${BLOCK_NUMBERS[$VMID]}" if [ "$BLOCK" != "N/A" ] && [[ "$BLOCK" =~ ^[0-9]+$ ]] && [ "$MAIN_BLOCK" != "" ]; then DIFF=$((BLOCK - MAIN_BLOCK)) if [ $DIFF -gt 1000 ]; then AHEAD+=("$VMID (block $BLOCK, +$DIFF)") elif [ $DIFF -lt -1000 ]; then BEHIND+=("$VMID (block $BLOCK, $DIFF)") fi fi done if [ ${#AHEAD[@]} -gt 0 ]; then echo "⚠️ Nodes ahead of main network:" for node in "${AHEAD[@]}"; do echo " - VMID $node" done echo "" fi if [ ${#BEHIND[@]} -gt 0 ]; then echo "⏳ Nodes behind main network (syncing):" for node in "${BEHIND[@]}"; do echo " - VMID $node" done echo "" fi # Recommendations echo "==========================================" echo "Recommendations" echo "==========================================" echo "" # Nodes with 2 peers (ThirdWeb) LOW_PEER_NODES=() for VMID in "${!PEER_COUNTS[@]}"; do PEERS="${PEER_COUNTS[$VMID]}" if [ "$PEERS" != "N/A" ] && [ "$PEERS" -lt 3 ] && [ "$PEERS" -gt 0 ]; then LOW_PEER_NODES+=("$VMID") fi done if [ ${#LOW_PEER_NODES[@]} -gt 0 ]; then echo "⚠️ Nodes with low peer count (${#LOW_PEER_NODES[@]} nodes):" for vmid in "${LOW_PEER_NODES[@]}"; do echo " - VMID $vmid: ${PEER_COUNTS[$vmid]} peers (should have 5-7)" done echo "" echo "Actions needed:" echo " 1. Verify static-nodes.json contains all 15 nodes" echo " 2. Check discovery-enabled=true" echo " 3. Verify permissions-nodes.toml is correct" echo " 4. Restart Besu services" echo "" echo "Run: ./scripts/fix-thirdweb-peer-connectivity.sh" echo "" fi # Nodes with 0 peers ZERO_PEER_NODES=() for VMID in "${!PEER_COUNTS[@]}"; do PEERS="${PEER_COUNTS[$VMID]}" if [ "$PEERS" = "0" ]; then ZERO_PEER_NODES+=("$VMID") fi done if [ ${#ZERO_PEER_NODES[@]} -gt 0 ]; then echo "⏳ Nodes with 0 peers (${#ZERO_PEER_NODES[@]} nodes):" for vmid in "${ZERO_PEER_NODES[@]}"; do BLOCK="${BLOCK_NUMBERS[$vmid]}" echo " - VMID $vmid: Block $BLOCK" if [ "$BLOCK" != "N/A" ] && [ "$BLOCK" -lt 1000000 ]; then echo " Status: Syncing (expected during initial sync)" else echo " Status: May be isolated - investigate" fi done echo "" fi echo "==========================================" echo "Complete" echo "=========================================="