#!/usr/bin/env bash # Check for stuck transactions in Besu transaction pool # Usage: ./scripts/check-stuck-transactions.sh [rpc-url] [account-address] set -euo pipefail # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[✓]${NC} $1"; } log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } log_error() { echo -e "${RED}[ERROR]${NC} $1"; } log_detail() { echo -e "${CYAN}[DETAIL]${NC} $1"; } # Configuration RPC_URL="${1:-http://192.168.11.250:8545}" # Use Core RPC (VMID 2500) ACCOUNT_ADDRESS="${2:-}" # Optional: specific account to check echo "=========================================" echo "Check for Stuck Transactions" echo "=========================================" echo "" log_info "RPC URL: $RPC_URL" echo "" # Check if RPC is accessible log_info "Checking RPC connectivity..." RPC_RESPONSE=$(curl -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ "$RPC_URL" 2>/dev/null || echo "") if [ -z "$RPC_RESPONSE" ] || ! echo "$RPC_RESPONSE" | jq -e '.result' >/dev/null 2>&1; then log_error "Cannot connect to RPC endpoint: $RPC_URL" exit 1 fi BLOCK_NUMBER=$(echo "$RPC_RESPONSE" | jq -r '.result' 2>/dev/null | sed 's/0x//' | xargs -I {} printf "%d\n" 0x{} 2>/dev/null || echo "unknown") log_success "RPC is accessible (Current block: $BLOCK_NUMBER)" echo "" # Check if TXPOOL API is enabled log_info "Checking if TXPOOL API is enabled..." RPC_MODULES=$(curl -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","method":"rpc_modules","params":[],"id":1}' \ "$RPC_URL" 2>/dev/null || echo "") TXPOOL_ENABLED=false if echo "$RPC_MODULES" | jq -r '.result | keys[]' 2>/dev/null | grep -qi "txpool"; then TXPOOL_ENABLED=true log_success "TXPOOL API is enabled" else log_warn "TXPOOL API is not enabled - cannot check transaction pool" log_info "Enable TXPOOL in RPC config: rpc-http-api=[\"ETH\",\"NET\",\"WEB3\",\"TXPOOL\"]" exit 1 fi echo "" # Get transaction pool content log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Fetching Transaction Pool" log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Try txpool_besuTransactions (Besu-specific) TXPOOL=$(curl -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","method":"txpool_besuTransactions","params":[],"id":1}' \ "$RPC_URL" 2>/dev/null || echo "") if [ -z "$TXPOOL" ] || ! echo "$TXPOOL" | jq -e '.result' >/dev/null 2>&1; then log_warn "Could not fetch transaction pool using txpool_besuTransactions" # Try txpool_content (standard) TXPOOL=$(curl -s -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","method":"txpool_content","params":[],"id":1}' \ "$RPC_URL" 2>/dev/null || echo "") if [ -z "$TXPOOL" ] || ! echo "$TXPOOL" | jq -e '.result' >/dev/null 2>&1; then log_error "Cannot fetch transaction pool content" exit 1 fi fi # Parse transaction pool TX_COUNT=$(echo "$TXPOOL" | jq -r '[.result | if type == "array" then .[] else . end] | length' 2>/dev/null || echo "0") if [ "$TX_COUNT" = "0" ] || [ "$TX_COUNT" = "null" ]; then log_success "✓ Transaction pool is empty - no stuck transactions" echo "" exit 0 fi log_warn "⚠ Found $TX_COUNT transaction(s) in pool" echo "" # Display all transactions log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Transaction Pool Details" log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Format and display transactions # txpool_besuTransactions returns minimal data, so we fetch full details for each hash TX_HASHES=$(echo "$TXPOOL" | jq -r '.result[]?.hash // empty' 2>/dev/null | grep -v "^null$" | grep -v "^$") if [ -z "$TX_HASHES" ]; then log_warn "Could not extract transaction hashes from pool" log_detail "Raw pool data:" echo "$TXPOOL" | jq '.result' 2>/dev/null | head -20 echo "" else TX_INDEX=0 echo "$TX_HASHES" | while IFS= read -r TX_HASH; do if [ -z "$TX_HASH" ] || [ "$TX_HASH" = "null" ]; then continue fi TX_INDEX=$((TX_INDEX + 1)) # Fetch full transaction details TX_DETAILS=$(curl -s -X POST -H "Content-Type: application/json" \ --data "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getTransactionByHash\",\"params\":[\"$TX_HASH\"],\"id\":1}" \ "$RPC_URL" 2>/dev/null | jq -r '.result' 2>/dev/null) if [ -z "$TX_DETAILS" ] || [ "$TX_DETAILS" = "null" ]; then # Transaction not found in blockchain, still in pool ADDED_AT=$(echo "$TXPOOL" | jq -r --arg hash "$TX_HASH" '.result[] | select(.hash == $hash) | .addedToPoolAt // "N/A"' 2>/dev/null) log_detail "Transaction #$TX_INDEX:" log_detail " Hash: $TX_HASH" log_detail " Status: In pool (not yet mined)" log_detail " Added to pool: $ADDED_AT" echo "" else # Extract details from transaction TX_FROM=$(echo "$TX_DETAILS" | jq -r '.from // "N/A"' 2>/dev/null) TX_TO=$(echo "$TX_DETAILS" | jq -r '.to // "N/A"' 2>/dev/null) TX_NONCE_HEX=$(echo "$TX_DETAILS" | jq -r '.nonce // "0x0"' 2>/dev/null) TX_NONCE=$(printf "%d" $TX_NONCE_HEX 2>/dev/null || echo "N/A") TX_GAS_PRICE_HEX=$(echo "$TX_DETAILS" | jq -r '.gasPrice // "0x0"' 2>/dev/null) TX_GAS_PRICE_WEI=$(printf "%d" $TX_GAS_PRICE_HEX 2>/dev/null || echo "0") TX_GAS_PRICE_GWEI=$(echo "scale=2; $TX_GAS_PRICE_WEI / 1000000000" | bc 2>/dev/null || echo "N/A") TX_GAS=$(echo "$TX_DETAILS" | jq -r '.gas // "N/A"' 2>/dev/null) TX_VALUE_HEX=$(echo "$TX_DETAILS" | jq -r '.value // "0x0"' 2>/dev/null) TX_VALUE=$(printf "%d" $TX_VALUE_HEX 2>/dev/null || echo "0") log_detail "Transaction #$TX_INDEX:" log_detail " Hash: $TX_HASH" log_detail " From: $TX_FROM" log_detail " To: $TX_TO" log_detail " Nonce: $TX_NONCE" log_detail " Gas Price: $TX_GAS_PRICE_GWEI gwei ($TX_GAS_PRICE_WEI wei)" log_detail " Gas Limit: $TX_GAS" log_detail " Value: $TX_VALUE wei" # Check if this transaction matches current nonce (stuck) if [ "$TX_FROM" != "N/A" ] && [ "$TX_NONCE" != "N/A" ]; then CURRENT_NONCE_TX=$(curl -s -X POST -H "Content-Type: application/json" \ --data "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getTransactionCount\",\"params\":[\"$TX_FROM\",\"latest\"],\"id\":1}" \ "$RPC_URL" 2>/dev/null | jq -r '.result' 2>/dev/null | sed 's/0x//' | xargs -I {} printf "%d\n" 0x{} 2>/dev/null || echo "") if [ -n "$CURRENT_NONCE_TX" ] && [ "$TX_NONCE" = "$CURRENT_NONCE_TX" ]; then log_warn " ⚠ STUCK: Nonce $TX_NONCE matches current nonce (transaction blocking subsequent transactions)" elif [ -n "$CURRENT_NONCE_TX" ] && [ "$TX_NONCE" -lt "$CURRENT_NONCE_TX" ]; then log_warn " ⚠ OUTDATED: Nonce $TX_NONCE is below current nonce $CURRENT_NONCE_TX (transaction may be rejected)" fi fi echo "" fi done <<< "$TX_HASHES" fi echo "" # Check specific account if provided if [ -n "$ACCOUNT_ADDRESS" ]; then log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" log_info "Checking Account: $ACCOUNT_ADDRESS" log_info "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" # Normalize address (remove 0x prefix for comparison) NORMALIZED_ACCOUNT=$(echo "$ACCOUNT_ADDRESS" | tr '[:upper:]' '[:lower:]' | sed 's/^0x//') # Find transactions for this account ACCOUNT_TXS=$(echo "$TXPOOL" | jq -r --arg addr "$NORMALIZED_ACCOUNT" \ '.result | if type == "array" then .[] else . end | select((.from // "") | ascii_downcase | gsub("^0x"; "") == $addr)' 2>/dev/null || echo "") if [ -z "$ACCOUNT_TXS" ] || [ "$ACCOUNT_TXS" = "null" ]; then log_success "✓ No transactions found for account $ACCOUNT_ADDRESS" else ACCOUNT_TX_COUNT=$(echo "$ACCOUNT_TXS" | jq -s 'length' 2>/dev/null || echo "1") log_warn "⚠ Found $ACCOUNT_TX_COUNT transaction(s) for account $ACCOUNT_ADDRESS" echo "" # Get current nonce for account CURRENT_NONCE=$(curl -s -X POST -H "Content-Type: application/json" \ --data "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getTransactionCount\",\"params\":[\"$ACCOUNT_ADDRESS\",\"latest\"],\"id\":1}" \ "$RPC_URL" 2>/dev/null | jq -r '.result' 2>/dev/null | sed 's/0x//' | xargs -I {} printf "%d\n" 0x{} 2>/dev/null || echo "unknown") log_info "Current on-chain nonce: $CURRENT_NONCE" echo "" # Display account transactions with nonce info echo "$ACCOUNT_TXS" | jq -r ' "Transaction Details: Hash: \(.hash // .transactionHash // "N/A") From: \(.from // "N/A") Nonce: \(.nonce // "N/A") Gas Price: \(.gasPrice // "N/A") ---"' 2>/dev/null | while IFS= read -r line; do if [[ "$line" == "---" ]]; then echo "" else log_detail "$line" fi done # Check for stuck nonces (nonce matches current nonce) echo "$ACCOUNT_TXS" | jq -r --arg nonce "$CURRENT_NONCE" \ 'select((.nonce // "" | tostring) == $nonce) | .hash // .transactionHash' 2>/dev/null | while read -r hash; do if [ -n "$hash" ] && [ "$hash" != "null" ]; then log_warn "⚠ STUCK TRANSACTION: Hash $hash has nonce $CURRENT_NONCE (matches current nonce)" fi done fi echo "" fi # Summary echo "=========================================" echo "Summary" echo "=========================================" echo "" log_warn "Total transactions in pool: $TX_COUNT" if [ "$TX_COUNT" -gt 0 ]; then log_info "" log_info "Next steps:" log_info " 1. Review transactions above to identify stuck ones" log_info " 2. Flush mempools: ./scripts/flush-all-mempools-proxmox.sh" log_info " 3. Or wait for transactions to be processed" fi echo ""