proxmox/scripts/clear-transaction-pool-all-nodes-thorough.sh

#!/usr/bin/env bash
# Thorough Transaction Pool Database Clear on All Besu Nodes
# Stops all nodes simultaneously, clears databases, prevents replay

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }
log_section() { echo -e "\n${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}"; echo -e "${CYAN}$1${NC}"; echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${NC}\n"; }

# All nodes
RPC_NODES=(
    "2101:besu-rpc.service"
    "2201:besu-rpc.service"
)

VALIDATORS=(
    "1000:besu-validator.service"
    "1001:besu-validator.service"
    "1002:besu-validator.service"
    "1003:besu-validator.service"
    "1004:besu-validator.service"
)

log_section "Thorough Transaction Pool Database Clear"
log_warn "⚠️  This will stop ALL Besu nodes and clear transaction pools"
log_warn "⚠️  All pending transactions will be permanently lost"
log_info "Proxmox Host: $PROXMOX_HOST"
echo ""

# Step 1: Stop all nodes simultaneously
log_section "Step 1: Stop All Nodes Simultaneously"

log_info "Stopping all RPC nodes..."
RPC_STOPPED=0
for node in "${RPC_NODES[@]}"; do
    IFS=':' read -r vmid service <<< "$node"
    if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then
        log_info "Stopping RPC-$vmid..."
        if ssh root@$PROXMOX_HOST "pct exec $vmid -- systemctl stop $service 2>&1"; then
            ((RPC_STOPPED++))
            log_success "RPC-$vmid stopped"
        fi
        sleep 1
    fi
done

log_info "Stopping all validators..."
VALIDATOR_STOPPED=0
for validator in "${VALIDATORS[@]}"; do
    IFS=':' read -r vmid service <<< "$validator"
    if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then
        log_info "Stopping Validator-$vmid..."
        if ssh root@$PROXMOX_HOST "pct exec $vmid -- systemctl stop $service 2>&1"; then
            ((VALIDATOR_STOPPED++))
            log_success "Validator-$vmid stopped"
        fi
        sleep 1
    fi
done

log_info "Waiting 5 seconds for services to fully stop..."
sleep 5

log_info "Stopped: $RPC_STOPPED RPC nodes, $VALIDATOR_STOPPED validators"

# Step 2: Clear transaction pool databases thoroughly
log_section "Step 2: Clear Transaction Pool Databases Thoroughly"

clear_node_database() {
    local vmid=$1
    local name=$2
    
    log_info "Clearing $name (VMID $vmid)..."
    CLEARED=0
    
    # Clear transaction pool directories
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type d -name '*pool*' -exec rm -rf {} \; 2>/dev/null || true"; then
        ((CLEARED++))
    fi
    
    # Clear transaction pool files
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type f \\( -name '*transaction*' -o -name '*pool*' -o -name '*mempool*' \\) -delete 2>/dev/null || true"; then
        ((CLEARED++))
    fi
    
    # Clear caches
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- rm -rf /data/besu/caches/* 2>/dev/null || true"; then
        ((CLEARED++))
    fi
    
    # Clear any RocksDB transaction pool databases
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type d -name '*rocksdb*' -exec find {} -name '*transaction*' -delete \\; 2>/dev/null || true"; then
        ((CLEARED++))
    fi
    
    # Clear pending transactions from any database
    if ssh root@$PROXMOX_HOST "pct exec $vmid -- find /data/besu -type f -name '*.ldb' -exec rm -f {} \\; 2>/dev/null || true"; then
        ((CLEARED++))
    fi
    
    if [ $CLEARED -gt 0 ]; then
        log_success "$name: Database cleared ($CLEARED operations)"
    else
        log_warn "$name: No database files found (may already be clear)"
    fi
}

# Clear RPC nodes
RPC_CLEARED=0
for node in "${RPC_NODES[@]}"; do
    IFS=':' read -r vmid service <<< "$node"
    if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then
        clear_node_database "$vmid" "RPC-$vmid"
        ((RPC_CLEARED++))
    fi
done

# Clear validators
VALIDATOR_CLEARED=0
for validator in "${VALIDATORS[@]}"; do
    IFS=':' read -r vmid service <<< "$validator"
    if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then
        clear_node_database "$vmid" "Validator-$vmid"
        ((VALIDATOR_CLEARED++))
    fi
done

log_info "Cleared: $RPC_CLEARED RPC nodes, $VALIDATOR_CLEARED validators"

# Step 3: Start validators first (they need to be ready)
log_section "Step 3: Start Validators First"

log_info "Starting validators (network consensus nodes)..."
VALIDATOR_STARTED=0
for validator in "${VALIDATORS[@]}"; do
    IFS=':' read -r vmid service <<< "$validator"
    if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then
        log_info "Starting Validator-$vmid..."
        if ssh root@$PROXMOX_HOST "pct exec $vmid -- systemctl start $service 2>&1"; then
            ((VALIDATOR_STARTED++))
            log_success "Validator-$vmid started"
        fi
        sleep 2
    fi
done

log_info "Waiting 10 seconds for validators to initialize..."
sleep 10

# Step 4: Start RPC nodes
log_section "Step 4: Start RPC Nodes"

log_info "Starting RPC nodes..."
RPC_STARTED=0
for node in "${RPC_NODES[@]}"; do
    IFS=':' read -r vmid service <<< "$node"
    if ssh root@$PROXMOX_HOST "pct status $vmid 2>&1" | grep -q "running"; then
        log_info "Starting RPC-$vmid..."
        if ssh root@$PROXMOX_HOST "pct exec $vmid -- systemctl start $service 2>&1"; then
            ((RPC_STARTED++))
            log_success "RPC-$vmid started"
        fi
        sleep 2
    fi
done

# Step 5: Wait for network to stabilize
log_section "Step 5: Network Stabilization"

log_info "Waiting 20 seconds for network to stabilize and sync..."
sleep 20

# Step 6: Verify nodes are online
log_section "Step 6: Verify Nodes Online"

RPC_IPS=(
    "2101:${RPC_CORE_1}"
    "2201:${RPC_PUBLIC_1}"
)

RPC_ONLINE=0
for rpc in "${RPC_IPS[@]}"; do
    IFS=':' read -r vmid ip <<< "$rpc"
    log_info "Checking RPC-$vmid ($ip)..."
    
    if cast block-number --rpc-url "http://${ip}:8545" >/dev/null 2>&1; then
        BLOCK=$(cast block-number --rpc-url "http://${ip}:8545" 2>/dev/null || echo "N/A")
        log_success "RPC-$vmid: Online (block: $BLOCK)"
        ((RPC_ONLINE++))
    else
        log_warn "RPC-$vmid: Not responding yet"
    fi
done

# Final summary
log_section "Summary"

echo "Stopped:"
echo "  • RPC nodes: $RPC_STOPPED"
echo "  • Validators: $VALIDATOR_STOPPED"
echo ""
echo "Cleared:"
echo "  • RPC nodes: $RPC_CLEARED"
echo "  • Validators: $VALIDATOR_CLEARED"
echo ""
echo "Started:"
echo "  • Validators: $VALIDATOR_STARTED"
echo "  • RPC nodes: $RPC_STARTED"
echo ""
echo "Online:"
echo "  • RPC nodes: $RPC_ONLINE/${#RPC_IPS[@]}"

if [ $RPC_ONLINE -eq ${#RPC_IPS[@]} ]; then
    log_success "\n✅ All nodes cleared and back online!"
    log_info "Next: Verify pending transactions are cleared and monitor for re-addition"
else
    log_warn "\n⚠️  Some RPC nodes may need more time"
    log_info "Wait a few minutes and check again"
fi

log_success "\nThorough transaction pool clear complete!"