proxmox/scripts/verify/verify-rpc-2101-approve-and-sync.sh

#!/usr/bin/env bash
# Verify RPC on VMID 2101 is full approve (permissioned with all 5 validators) and in sync.
# Run after flush/cancel/deploy steps. Usage: ./scripts/verify/verify-rpc-2101-approve-and-sync.sh
# Optional: set RPC_URL_138 if different. SSH to r630-01/ml110 used only for validator service check.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true

RPC_CORE_1="${RPC_CORE_1:-192.168.11.211}"
RPC_URL="${RPC_URL_138:-http://${RPC_CORE_1}:8545}"
PROXMOX_USER="${PROXMOX_USER:-root}"
PROXMOX_R630="${PROXMOX_R630_01:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
PROXMOX_ML110="${PROXMOX_ML110:-${PROXMOX_HOST_ML110:-192.168.11.10}}"

# Five validator IPs (1000-1004)
VALIDATOR_IPS=(192.168.11.100 192.168.11.101 192.168.11.102 192.168.11.103 192.168.11.104)

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'
log_ok()   { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_err()  { echo -e "${RED}[✗]${NC} $1"; }
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }

FAIL=0

echo ""
echo "=== Verify RPC 2101: full approve and in sync with 5 validators ==="
echo "RPC: $RPC_URL"
echo ""

# 1. RPC reachable and Chain 138
log_info "1. RPC reachable and Chain 138..."
CHAIN_ID=$(curl -s -m 10 -X POST -H "Content-Type: application/json" \
  --data '{"jsonrpc":"2.0","method":"eth_chainId","params":[],"id":1}' "$RPC_URL" 2>/dev/null | jq -r '.result // empty' | tr -d '"')
if [[ "$CHAIN_ID" != "0x8a" ]]; then
  log_err "RPC eth_chainId: got '$CHAIN_ID', expected 0x8a (Chain 138)"
  ((FAIL++)) || true
else
  log_ok "RPC Chain ID: 0x8a (138)"
fi

# 2. Block number
BLOCK_HEX=$(curl -s -m 10 -X POST -H "Content-Type: application/json" \
  --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' "$RPC_URL" 2>/dev/null | jq -r '.result // empty' | tr -d '"')
BLOCK_DEC=$((BLOCK_HEX))
if [[ -z "$BLOCK_HEX" || "$BLOCK_DEC" -eq 0 ]]; then
  log_err "RPC eth_blockNumber: no valid response"
  ((FAIL++)) || true
else
  log_ok "RPC block number: $BLOCK_DEC ($BLOCK_HEX)"
fi

# 3. admin_peers (ADMIN API) — full approve = RPC has peers and ideally all 5 validators
log_info "2. Peer connections (admin_peers — RPC must have ADMIN API)..."
PEERS_JSON=$(curl -s -m 10 -X POST -H "Content-Type: application/json" \
  --data '{"jsonrpc":"2.0","method":"admin_peers","params":[],"id":1}' "$RPC_URL" 2>/dev/null || echo "{}")
if ! echo "$PEERS_JSON" | jq -e '.result' >/dev/null 2>&1; then
  log_err "admin_peers failed or not available (RPC 2101 must have ADMIN in rpc-http-api)"
  ((FAIL++)) || true
else
  PEER_COUNT=$(echo "$PEERS_JSON" | jq '.result | length' 2>/dev/null || echo "0")
  if [[ "$PEER_COUNT" -lt 5 ]]; then
    log_err "RPC has $PEER_COUNT peers (expected >= 5 for full approve with 5 validators)"
    ((FAIL++)) || true
  else
    log_ok "RPC peer count: $PEER_COUNT"
  fi
  # Check that all 5 validator IPs appear in peer list (any key: remoteAddress, url, etc.)
  PEER_STR=$(echo "$PEERS_JSON" | jq -r '.result | tostring' 2>/dev/null || echo "$PEERS_JSON")
  VALIDATORS_CONNECTED=0
  for ip in "${VALIDATOR_IPS[@]}"; do
    if echo "$PEER_STR" | grep -q "$ip"; then
      ((VALIDATORS_CONNECTED++)) || true
    fi
  done
  if [[ "$VALIDATORS_CONNECTED" -lt 5 ]]; then
    log_warn "Only $VALIDATORS_CONNECTED/5 validator IPs (192.168.11.100–104) seen in peers; ensure permissions-nodes.toml on 2101 includes all 5 and run scripts/deploy-besu-node-lists-to-all.sh"
  else
    log_ok "All 5 validator IPs (192.168.11.100–104) present in peer list (full approve)"
  fi
fi

# 4. Block production (in sync = blocks advancing)
log_info "3. Block production (RPC in sync)..."
sleep 5
BLOCK_HEX2=$(curl -s -m 10 -X POST -H "Content-Type: application/json" \
  --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' "$RPC_URL" 2>/dev/null | jq -r '.result // empty' | tr -d '"')
BLOCK_DEC2=$((BLOCK_HEX2))
BLOCK_DIFF=$((BLOCK_DEC2 - BLOCK_DEC))
if [[ "$BLOCK_DIFF" -gt 0 ]]; then
  log_ok "Blocks advancing ($BLOCK_DIFF in 5s) — RPC in sync"
else
  log_warn "No new blocks in 5s (RPC block $BLOCK_DEC); check validators and block production"
  ((FAIL++)) || true
fi

# 5. Optional: validator service status (requires SSH to r630-01 and ml110)
log_info "4. Validator status (5/5 active) — requires SSH to r630-01 and ml110..."
ACTIVE=0
SSH_OK=false
for entry in "1000:$PROXMOX_R630" "1001:$PROXMOX_R630" "1002:$PROXMOX_R630" "1003:$PROXMOX_ML110" "1004:$PROXMOX_ML110"; do
  IFS=':' read -r VMID HOST <<< "$entry"
  STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "${PROXMOX_USER}@${HOST}" \
    "pct exec $VMID -- systemctl is-active besu-validator 2>/dev/null" 2>/dev/null || echo "unknown")
  if [[ "$STATUS" = "active" ]]; then
    ((ACTIVE++)) || true
    SSH_OK=true
  elif [[ "$STATUS" != "unknown" ]]; then
    SSH_OK=true
  fi
done
if [[ "$ACTIVE" -eq 5 ]]; then
  log_ok "All 5 validators (1000–1004) active"
elif [[ "$SSH_OK" = true ]]; then
  log_warn "Only $ACTIVE/5 validators active"
  ((FAIL++)) || true
else
  log_warn "Could not SSH to Proxmox hosts (run from LAN for validator check); RPC peer count and block sync still verified"
fi

echo ""
if [[ $FAIL -eq 0 ]]; then
  log_ok "RPC 2101: full approve and in sync with 5 validators."
  echo ""
  exit 0
else
  log_err "RPC 2101 verification had $FAIL issue(s). Fix: ensure node lists deployed (scripts/deploy-besu-node-lists-to-all.sh), validators running, then scripts/maintenance/health-check-rpc-2101.sh and docs/09-troubleshooting/RPC_NODES_BLOCK_PRODUCTION_FIX.md"
  echo ""
  exit 1
fi