proxmox/scripts/review-sentry-and-rpc-nodes.sh

#!/usr/bin/env bash
# Review live Besu VMs: validators, sentries, and RPC nodes (status, service, block height).
# Optional: apply tx-pool eviction with --apply-txpool.
# Run from project root.
# Usage:
#   bash scripts/review-sentry-and-rpc-nodes.sh
#   bash scripts/review-sentry-and-rpc-nodes.sh --vmid 2301
#   bash scripts/review-sentry-and-rpc-nodes.sh --vmid 2301 --apply-txpool
# Historical 25xx residue is excluded by default; set INCLUDE_HISTORICAL_25XX=1 to include it explicitly.
# Note: RPC 2307/2308 (Putu) may show block behind if not synced; check logs if needed.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
cd "$PROJECT_ROOT"
source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"

PROXMOX_SSH_USER="${PROXMOX_SSH_USER:-${PROXMOX_USER:-root}}"
[[ "$PROXMOX_SSH_USER" == *"@"* ]] && PROXMOX_SSH_USER="root"

# VMID:host:service:config_name (config path under /etc/besu/).
# Host placement is derived from get_host_for_vmid() so this script follows the live inventory in ALL_VMIDS_ENDPOINTS.md.
VALIDATORS=(
  "1000:$(get_host_for_vmid 1000):besu-validator:config-validator.toml"
  "1001:$(get_host_for_vmid 1001):besu-validator:config-validator.toml"
  "1002:$(get_host_for_vmid 1002):besu-validator:config-validator.toml"
  "1003:$(get_host_for_vmid 1003):besu-validator:config-validator.toml"
  "1004:$(get_host_for_vmid 1004):besu-validator:config-validator.toml"
)
SENTRIES=(
  "1500:$(get_host_for_vmid 1500):besu-sentry:config-sentry.toml"
  "1501:$(get_host_for_vmid 1501):besu-sentry:config-sentry.toml"
  "1502:$(get_host_for_vmid 1502):besu-sentry:config-sentry.toml"
  "1503:$(get_host_for_vmid 1503):besu-sentry:config-sentry.toml"
  "1504:$(get_host_for_vmid 1504):besu-sentry:config-sentry.toml"
  "1505:$(get_host_for_vmid 1505):besu-sentry:config-sentry.toml"
  "1506:$(get_host_for_vmid 1506):besu-sentry:config-sentry.toml"
)
RPC_NODES=(
  "2101:$(get_host_for_vmid 2101):besu-rpc:config-rpc-core.toml"
  "2201:$(get_host_for_vmid 2201):besu-rpc:config-rpc-public.toml"
  "2301:$(get_host_for_vmid 2301):besu-rpc:config-rpc-private.toml"
  "2303:$(get_host_for_vmid 2303):besu-rpc:config-rpc.toml"
  "2304:$(get_host_for_vmid 2304):besu-rpc:config-rpc.toml"
  "2305:$(get_host_for_vmid 2305):besu-rpc:config-rpc.toml"
  "2306:$(get_host_for_vmid 2306):besu-rpc:config-rpc.toml"
  "2307:$(get_host_for_vmid 2307):besu-rpc:config-rpc.toml"
  "2308:$(get_host_for_vmid 2308):besu-rpc:config-rpc.toml"
  "2400:$(get_host_for_vmid 2400):besu-rpc:config-rpc.toml"
  "2401:$(get_host_for_vmid 2401):besu-rpc:config-rpc.toml"
  "2402:$(get_host_for_vmid 2402):besu-rpc:config-rpc.toml"
  "2403:$(get_host_for_vmid 2403):besu-rpc:config-rpc.toml"
)

if [[ "${INCLUDE_HISTORICAL_25XX:-0}" == "1" ]]; then
  RPC_NODES+=(
    "2503:${PROXMOX_HOST_R630_01}:besu-rpc:config-rpc.toml"
    "2504:${PROXMOX_HOST_R630_01}:besu-rpc:config-rpc.toml"
    "2505:${PROXMOX_HOST_R630_01}:besu-rpc:config-rpc.toml"
  )
fi

RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }

APPLY_TXPOOL=false
TARGET_VMIDS=()

usage() {
  cat <<'EOF'
Usage: bash scripts/review-sentry-and-rpc-nodes.sh [--apply-txpool] [--vmid <N>]

Options:
  --apply-txpool  Mutate config and restart service on selected nodes
  --vmid <N>      Limit review/apply to one VMID; repeatable
EOF
}

while [[ $# -gt 0 ]]; do
  case "$1" in
    --apply-txpool)
      APPLY_TXPOOL=true
      shift
      ;;
    --vmid)
      [[ $# -ge 2 ]] || { usage >&2; exit 2; }
      TARGET_VMIDS+=("$2")
      shift 2
      ;;
    -h|--help)
      usage
      exit 0
      ;;
    *)
      echo "Unknown argument: $1" >&2
      usage >&2
      exit 2
      ;;
  esac
done

selected_vmid() {
  local vmid="$1"
  [[ ${#TARGET_VMIDS[@]} -eq 0 ]] && return 0
  local wanted
  for wanted in "${TARGET_VMIDS[@]}"; do
    [[ "$vmid" == "$wanted" ]] && return 0
  done
  return 1
}

review_one() {
  local vmid="$1" host="$2" service="$3" config_name="$4" is_rpc="${5:-false}"
  local ssh_target="${PROXMOX_SSH_USER}@${host}"
  local ct_status service_status block_info=""

  ct_status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$ssh_target" "pct status $vmid 2>/dev/null" | awk '{print $2}' || echo "unknown")
  if [[ "$ct_status" != "running" ]]; then
    echo -e "  VMID $vmid: container ${RED}$ct_status${NC} (host $host)"
    return 0
  fi

  # HYBX 2503-2505 use besu.service; check it if besu-rpc not active
  service_status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$ssh_target" "pct exec $vmid -- systemctl is-active $service 2>/dev/null" || echo "unknown")
  if [[ "$service_status" != "active" && "$vmid" =~ ^250[345]$ ]]; then
    service_status=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$ssh_target" "pct exec $vmid -- systemctl is-active besu.service 2>/dev/null" || echo "unknown")
  fi
  if [[ "$service_status" = "active" ]]; then
    echo -e "  VMID $vmid: container running, ${GREEN}$service active${NC}"
  elif [[ "$service_status" = "inactive" && "$vmid" =~ ^250[345]$ ]]; then
    echo -e "  VMID $vmid: container running, ${YELLOW}besu.service inactive (HYBX: Besu not installed or disabled)${NC}"
  else
    echo -e "  VMID $vmid: container running, ${YELLOW}$service $service_status${NC}"
  fi

  if [[ "$is_rpc" = "true" ]]; then
    local ip
    ip=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$ssh_target" "pct exec $vmid -- hostname -I 2>/dev/null | awk '{print \$1}'" 2>/dev/null || echo "")
    if [[ -n "$ip" ]]; then
      local resp
      resp=$(curl -s -m 3 -X POST -H "Content-Type: application/json" -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' "http://$ip:8545" 2>/dev/null || echo "")
      if echo "$resp" | grep -q '"result"'; then
        local block_hex block_dec
        block_hex=$(echo "$resp" | jq -r '.result' 2>/dev/null)
        block_dec=$((block_hex))
        # Chain 138 is past 2M; flag if node is >50k blocks behind
        if [[ "$block_dec" -lt 2050000 ]]; then
          echo -e "    RPC $ip:8545 → block ${YELLOW}$block_dec (behind)${NC}"
        else
          echo -e "    RPC $ip:8545 → block ${GREEN}$block_dec${NC}"
        fi
      else
        echo -e "    RPC $ip:8545 → ${YELLOW}no response${NC}"
      fi
    fi
  fi

  if "$APPLY_TXPOOL"; then
    # Apply layered tx-pool + min-score=0; try /etc/besu/ then /config/
    local cfg_etc="/etc/besu/$config_name" cfg_alt="/config/$config_name"
    ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$ssh_target" "pct exec $vmid -- bash -c '
      CFG=\"$cfg_etc\"
      [ -f \"\$CFG\" ] || CFG=\"$cfg_alt\"
      [ ! -f \"\$CFG\" ] && exit 0
      sed -i \"/^tx-pool-max-size=/d\" \"\$CFG\" 2>/dev/null || true
      sed -i \"/^tx-pool-limit-by-account-percentage=/d\" \"\$CFG\" 2>/dev/null || true
      sed -i \"/^tx-pool-retention-hours=/d\" \"\$CFG\" 2>/dev/null || true
      if ! grep -q \"tx-pool-max-future-by-sender\" \"\$CFG\"; then
        echo \"\" >> \"\$CFG\"
        echo \"# Layered Tx-Pool (tx-pool-min-score not used: unsupported in some Besu builds)\" >> \"\$CFG\"
        echo \"tx-pool-max-future-by-sender=200\" >> \"\$CFG\"
        echo \"tx-pool-layer-max-capacity=12500000\" >> \"\$CFG\"
        echo \"tx-pool-max-prioritized=2000\" >> \"\$CFG\"
        echo \"tx-pool-price-bump=10\" >> \"\$CFG\"
      fi
      sed -i \"/^tx-pool-min-score=/d\" \"\$CFG\" 2>/dev/null || true
    '" 2>/dev/null && {
      ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no "$ssh_target" "pct exec $vmid -- systemctl restart $service" 2>/dev/null && \
        echo -e "    ${GREEN}Tx-pool eviction applied, $service restarted${NC}" || echo -e "    ${YELLOW}Config updated, restart failed${NC}"
    } || true
  fi
}

echo ""
echo -e "${CYAN}=== Validator Nodes ===${NC}"
for entry in "${VALIDATORS[@]}"; do
  IFS=: read -r vmid host service config_name <<< "$entry"
  selected_vmid "$vmid" || continue
  review_one "$vmid" "$host" "$service" "$config_name" "false"
done

echo ""
echo -e "${CYAN}=== Sentry Nodes ===${NC}"
for entry in "${SENTRIES[@]}"; do
  IFS=: read -r vmid host service config_name <<< "$entry"
  selected_vmid "$vmid" || continue
  review_one "$vmid" "$host" "$service" "$config_name" "false"
done

echo ""
echo -e "${CYAN}=== RPC Nodes ===${NC}"
for entry in "${RPC_NODES[@]}"; do
  IFS=: read -r vmid host service config_name <<< "$entry"
  selected_vmid "$vmid" || continue
  review_one "$vmid" "$host" "$service" "$config_name" "true"
done

echo ""
if "$APPLY_TXPOOL"; then
  log_success "Review complete; tx-pool eviction applied and services restarted where config was updated."
else
  log_info "Review complete. To apply tx-pool eviction (layered + min-score=0) and restart: $0 --apply-txpool"
fi