#!/usr/bin/env bash # Thorough Transaction Pool Database Clear on All Besu Nodes # Stops all nodes simultaneously, clears databases, prevents replay set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}" # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[✓]${NC} $1"; } log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; } log_error() { echo -e "${RED}[✗]${NC} $1"; } log_section() { echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN}$1${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"; } # All nodes RPC_NODES=( "2101:besu-rpc.service" "2201:besu-rpc.service" ) VALIDATORS=( "1000:besu-validator.service" "1001:besu-validator.service" "1002:besu-validator.service" "1003:besu-validator.service" "1004:besu-validator.service" ) log_section "Thorough Transaction Pool Database Clear" log_warn "⚠️ This will stop ALL Besu nodes and clear transaction pools" log_warn "⚠️ All pending transactions will be permanently lost" log_info "Proxmox Host: $PROXMOX_HOST" echo "" # Step 1: Stop all nodes simultaneously log_section "Step 1: Stop All Nodes Simultaneously" log_info "Stopping all RPC nodes..." RPC_STOPPED=0 for node in "${RPC_NODES[@]}"; do IFS=':' read -r vmid service <<< "$node" if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then log_info "Stopping RPC-$vmid..." if ssh root@$PROXMOX_HOST "pct exec $vmid -- systemctl stop $service 2>&1"; then ((RPC_STOPPED++)) log_success "RPC-$vmid stopped" fi sleep 1 fi done log_info "Stopping all validators..." VALIDATOR_STOPPED=0 for validator in "${VALIDATORS[@]}"; do IFS=':' read -r vmid service <<< "$validator" if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then log_info "Stopping Validator-$vmid..." if ssh root@$PROXMOX_HOST "pct exec $vmid -- systemctl stop $service 2>&1"; then ((VALIDATOR_STOPPED++)) log_success "Validator-$vmid stopped" fi sleep 1 fi done log_info "Waiting 5 seconds for services to fully stop..." sleep 5 log_info "Stopped: $RPC_STOPPED RPC nodes, $VALIDATOR_STOPPED validators" # Step 2: Clear transaction pool databases thoroughly log_section "Step 2: Clear Transaction Pool Databases Thoroughly" clear_node_database() { local vmid=$1 local name=$2 log_info "Clearing $name (VMID $vmid)..." CLEARED=0 # Clear transaction pool directories if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type d -name '*pool*' -exec rm -rf {} \; 2>/dev/null || true"; then ((CLEARED++)) fi # Clear transaction pool files if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type f \\( -name '*transaction*' -o -name '*pool*' -o -name '*mempool*' \\) -delete 2>/dev/null || true"; then ((CLEARED++)) fi # Clear caches if ssh root@$PROXMOX_HOST "pct exec $vmid -- rm -rf /data/besu/caches/* 2>/dev/null || true"; then ((CLEARED++)) fi # Clear any RocksDB transaction pool databases if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type d -name '*rocksdb*' -exec find {} -name '*transaction*' -delete \\; 2>/dev/null || true"; then ((CLEARED++)) fi # Clear pending transactions from any database if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type f -name '*.ldb' -exec rm -f {} \\; 2>/dev/null || true"; then ((CLEARED++)) fi if [ $CLEARED -gt 0 ]; then log_success "$name: Database cleared ($CLEARED operations)" else log_warn "$name: No database files found (may already be clear)" fi } # Clear RPC nodes RPC_CLEARED=0 for node in "${RPC_NODES[@]}"; do IFS=':' read -r vmid service <<< "$node" if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then clear_node_database "$vmid" "RPC-$vmid" ((RPC_CLEARED++)) fi done # Clear validators VALIDATOR_CLEARED=0 for validator in "${VALIDATORS[@]}"; do IFS=':' read -r vmid service <<< "$validator" if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then clear_node_database "$vmid" "Validator-$vmid" ((VALIDATOR_CLEARED++)) fi done log_info "Cleared: $RPC_CLEARED RPC nodes, $VALIDATOR_CLEARED validators" # Step 3: Start validators first (they need to be ready) log_section "Step 3: Start Validators First" log_info "Starting validators (network consensus nodes)..." VALIDATOR_STARTED=0 for validator in "${VALIDATORS[@]}"; do IFS=':' read -r vmid service <<< "$validator" if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then log_info "Starting Validator-$vmid..." if ssh root@$PROXMOX_HOST "pct exec $vmid -- systemctl start $service 2>&1"; then ((VALIDATOR_STARTED++)) log_success "Validator-$vmid started" fi sleep 2 fi done log_info "Waiting 10 seconds for validators to initialize..." sleep 10 # Step 4: Start RPC nodes log_section "Step 4: Start RPC Nodes" log_info "Starting RPC nodes..." RPC_STARTED=0 for node in "${RPC_NODES[@]}"; do IFS=':' read -r vmid service <<< "$node" if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then log_info "Starting RPC-$vmid..." if ssh root@$PROXMOX_HOST "pct exec $vmid -- systemctl start $service 2>&1"; then ((RPC_STARTED++)) log_success "RPC-$vmid started" fi sleep 2 fi done # Step 5: Wait for network to stabilize log_section "Step 5: Network Stabilization" log_info "Waiting 20 seconds for network to stabilize and sync..." sleep 20 # Step 6: Verify nodes are online log_section "Step 6: Verify Nodes Online" RPC_IPS=( "2101:${RPC_CORE_1}" "2201:${RPC_PUBLIC_1}" ) RPC_ONLINE=0 for rpc in "${RPC_IPS[@]}"; do IFS=':' read -r vmid ip <<< "$rpc" log_info "Checking RPC-$vmid ($ip)..." if cast block-number --rpc-url "http://${ip}:8545" >/dev/null 2>&1; then BLOCK=$(cast block-number --rpc-url "http://${ip}:8545" 2>/dev/null || echo "N/A") log_success "RPC-$vmid: Online (block: $BLOCK)" ((RPC_ONLINE++)) else log_warn "RPC-$vmid: Not responding yet" fi done # Final summary log_section "Summary" echo "Stopped:" echo " • RPC nodes: $RPC_STOPPED" echo " • Validators: $VALIDATOR_STOPPED" echo "" echo "Cleared:" echo " • RPC nodes: $RPC_CLEARED" echo " • Validators: $VALIDATOR_CLEARED" echo "" echo "Started:" echo " • Validators: $VALIDATOR_STARTED" echo " • RPC nodes: $RPC_STARTED" echo "" echo "Online:" echo " • RPC nodes: $RPC_ONLINE/${#RPC_IPS[@]}" if [ $RPC_ONLINE -eq ${#RPC_IPS[@]} ]; then log_success "\n✅ All nodes cleared and back online!" log_info "Next: Verify pending transactions are cleared and monitor for re-addition" else log_warn "\n⚠️ Some RPC nodes may need more time" log_info "Wait a few minutes and check again" fi log_success "\nThorough transaction pool clear complete!"