#!/usr/bin/env bash # Validate Validator Set Script # Validates that all validators are properly configured and can participate in consensus set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" source "$PROJECT_ROOT/lib/common.sh" # Load configuration load_config load_config "$PROJECT_ROOT/config/network.conf" || true # VMID ranges VALIDATORS_START="${VALIDATOR_START:-1000}" VALIDATORS_COUNT="${VALIDATOR_COUNT:-${VALIDATORS_COUNT:-5}}" VALIDATORS_END=$((VALIDATORS_START + VALIDATORS_COUNT - 1)) VALIDATORS=() for ((vmid=VALIDATORS_START; vmid<=VALIDATORS_END; vmid++)); do VALIDATORS+=($vmid) done log_info "=========================================" log_info "Validator Set Validation" log_info "=========================================" log_info "" log_info "Validating ${#VALIDATORS[@]} validators (${VALIDATORS_START}-${VALIDATORS_END})" log_info "" VALIDATION_ERRORS=0 VALIDATION_WARNINGS=0 # Function to check if container is running check_container_running() { local vmid=$1 if pct status "$vmid" 2>/dev/null | grep -q running; then return 0 else return 1 fi } # Function to check if service is running check_service_running() { local vmid=$1 if pct exec "$vmid" -- systemctl is-active --quiet besu-validator 2>/dev/null; then return 0 else return 1 fi } # Function to check validator keys exist check_validator_keys() { local vmid=$1 local keys_dir="/keys/validators" if pct exec "$vmid" -- test -d "$keys_dir" 2>/dev/null; then local key_count key_count=$(pct exec "$vmid" -- find "$keys_dir" -name "*.priv" -o -name "key.pem" 2>/dev/null | wc -l || echo "0") if [[ $key_count -gt 0 ]]; then return 0 fi fi return 1 } # Function to check validator address exists check_validator_address() { local vmid=$1 local address_file="/keys/validators/validator-*/address.txt" if pct exec "$vmid" -- sh -c "test -f $address_file" 2>/dev/null; then local address address=$(pct exec "$vmid" -- sh -c "cat $address_file 2>/dev/null | head -1" 2>/dev/null | tr -d '\n\r ' || echo "") if [[ -n "$address" ]] && [[ ${#address} -eq 42 ]] && [[ "$address" =~ ^0x[0-9a-fA-F]{40}$ ]]; then echo "$address" return 0 fi fi return 1 } # Function to check Besu process is using validator keys check_besu_using_keys() { local vmid=$1 # Check if Besu process is running and has key directory mounted if pct exec "$vmid" -- pgrep -f "besu.*validator" >/dev/null 2>&1; then # Check if Besu config references validator keys if pct exec "$vmid" -- grep -q "validator-keys" /etc/besu/config-validator.toml 2>/dev/null; then return 0 fi fi return 1 } # Function to check consensus participation (via block production) check_consensus_participation() { local vmid=$1 # Try to get block number from RPC (validators typically don't have RPC enabled) # Instead, check logs for consensus activity if pct exec "$vmid" -- journalctl -u besu-validator --no-pager -n 50 2>/dev/null | grep -qiE "(consensus|qbft|block.*produced|proposing)" 2>/dev/null; then return 0 fi # Alternative: Check if process is running and healthy if pct exec "$vmid" -- pgrep -f "besu.*validator" >/dev/null 2>&1; then # If process is running, assume it's participating (may not have produced blocks yet) return 0 fi return 1 } # Function to check validator can connect to peers check_peer_connectivity() { local vmid=$1 # Validators typically don't have RPC, so check if process is running # Peer connectivity is validated during network bootstrap if pct exec "$vmid" -- pgrep -f "besu.*validator" >/dev/null 2>&1; then return 0 fi return 1 } # Main validation loop log_info "=== Validation Phase 1: Container & Service Status ===" for vmid in "${VALIDATORS[@]}"; do log_info "Validating validator $vmid..." # Check container running if check_container_running "$vmid"; then log_success " ✓ Container $vmid is running" else log_error " ✗ Container $vmid is not running" VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1)) continue fi # Check service running if check_service_running "$vmid"; then log_success " ✓ Besu validator service is running" else log_error " ✗ Besu validator service is not running" VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1)) fi done log_info "" log_info "=== Validation Phase 2: Validator Keys ===" for vmid in "${VALIDATORS[@]}"; do if ! check_container_running "$vmid"; then continue fi log_info "Checking validator keys for $vmid..." # Check validator keys exist if check_validator_keys "$vmid"; then log_success " ✓ Validator keys directory exists and contains keys" else log_error " ✗ Validator keys not found in /keys/validators/" VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1)) fi # Check validator address address=$(check_validator_address "$vmid") if [[ -n "$address" ]]; then log_success " ✓ Validator address found: $address" else log_warn " ⚠ Validator address file not found or invalid" VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1)) fi done log_info "" log_info "=== Validation Phase 3: Besu Configuration ===" for vmid in "${VALIDATORS[@]}"; do if ! check_container_running "$vmid"; then continue fi log_info "Checking Besu configuration for $vmid..." # Check config file exists if pct exec "$vmid" -- test -f /etc/besu/config-validator.toml 2>/dev/null; then log_success " ✓ Config file exists" else log_error " ✗ Config file missing: /etc/besu/config-validator.toml" VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1)) continue fi # Check genesis file exists if pct exec "$vmid" -- test -f /etc/besu/genesis.json 2>/dev/null; then log_success " ✓ Genesis file exists" else log_error " ✗ Genesis file missing: /etc/besu/genesis.json" VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1)) fi # Check static-nodes.json exists if pct exec "$vmid" -- test -f /etc/besu/static-nodes.json 2>/dev/null; then log_success " ✓ static-nodes.json exists" else log_warn " ⚠ static-nodes.json missing (may not be critical)" VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1)) fi # Check permissions-nodes.toml exists if pct exec "$vmid" -- test -f /etc/besu/permissions-nodes.toml 2>/dev/null; then log_success " ✓ permissions-nodes.toml exists" else log_warn " ⚠ permissions-nodes.toml missing (may not be critical)" VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1)) fi # Check Besu is using validator keys if check_besu_using_keys "$vmid"; then log_success " ✓ Besu configured with validator keys" else log_warn " ⚠ Cannot verify Besu is using validator keys" VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1)) fi done log_info "" log_info "=== Validation Phase 4: Consensus Participation ===" for vmid in "${VALIDATORS[@]}"; do if ! check_container_running "$vmid"; then continue fi log_info "Checking consensus participation for $vmid..." # Check consensus participation if check_consensus_participation "$vmid"; then log_success " ✓ Validator appears to be participating in consensus" else log_warn " ⚠ Cannot verify consensus participation (may still be starting)" VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1)) fi # Check peer connectivity if check_peer_connectivity "$vmid"; then log_success " ✓ Validator process is running (peer connectivity checked during bootstrap)" else log_warn " ⚠ Validator process not running or not responding" VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1)) fi done # Summary log_info "" log_info "=========================================" log_info "Validation Summary" log_info "=========================================" log_info "Validators checked: ${#VALIDATORS[@]}" log_info "Errors: $VALIDATION_ERRORS" log_info "Warnings: $VALIDATION_WARNINGS" log_info "" if [[ $VALIDATION_ERRORS -eq 0 ]]; then if [[ $VALIDATION_WARNINGS -eq 0 ]]; then log_success "✓ All validators validated successfully!" exit 0 else log_warn "⚠ Validation complete with $VALIDATION_WARNINGS warning(s)" log_info "Warnings are non-critical but should be reviewed" exit 0 fi else log_error "✗ Validation failed with $VALIDATION_ERRORS error(s)" log_info "Please fix the errors before proceeding" exit 1 fi