Files
proxmox/smom-dbis-138-proxmox/scripts/validation/validate-deployment-comprehensive.sh

455 lines
18 KiB
Bash
Executable File

#!/usr/bin/env bash
# Comprehensive Deployment Validation Script
# Validates all aspects of deployment to ensure correctness and completeness
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
source "$PROJECT_ROOT/lib/common.sh"
# Load configuration
load_config "$PROJECT_ROOT/config/proxmox.conf" 2>/dev/null || true
load_config "$PROJECT_ROOT/config/network.conf" 2>/dev/null || true
# Load VMID ranges from config (defaults to new allocation starting at 1000)
VALIDATOR_START="${VMID_VALIDATORS_START:-1000}"
SENTRY_START="${VMID_SENTRIES_START:-1500}"
RPC_START="${VMID_RPC_START:-2500}"
# Support both singular and plural forms from config
VALIDATOR_COUNT="${VALIDATOR_COUNT:-${VALIDATORS_COUNT:-5}}"
SENTRY_COUNT="${SENTRY_COUNT:-${SENTRIES_COUNT:-4}}"
RPC_COUNT="${RPC_COUNT:-3}"
# Build VMID arrays and mappings dynamically
VALIDATORS=()
SENTRIES=()
RPCS=()
declare -A VMID_TO_NODE_TYPE
declare -A VMID_TO_NODE_NAME
# Build validators array and mappings
for ((i=0; i<VALIDATOR_COUNT; i++)); do
vmid=$((VALIDATOR_START + i))
validator_num=$((i + 1))
VALIDATORS+=($vmid)
VMID_TO_NODE_TYPE[$vmid]="validator"
VMID_TO_NODE_NAME[$vmid]="validator-$validator_num"
done
# Build sentries array and mappings
for ((i=0; i<SENTRY_COUNT; i++)); do
vmid=$((SENTRY_START + i))
sentry_num=$((i + 2)) # sentry-2, sentry-3, etc.
SENTRIES+=($vmid)
VMID_TO_NODE_TYPE[$vmid]="sentry"
VMID_TO_NODE_NAME[$vmid]="sentry-$sentry_num"
done
# Build RPC array and mappings
for ((i=0; i<RPC_COUNT; i++)); do
vmid=$((RPC_START + i))
rpc_num=$((i + 1))
RPCS+=($vmid)
VMID_TO_NODE_TYPE[$vmid]="rpc"
VMID_TO_NODE_NAME[$vmid]="rpc-$rpc_num"
done
ALL_BESU=("${VALIDATORS[@]}" "${SENTRIES[@]}" "${RPCS[@]}")
VALIDATION_ERRORS=0
VALIDATION_WARNINGS=0
# Expected file locations by node type
declare -A EXPECTED_FILES_VALIDATOR
EXPECTED_FILES_VALIDATOR["/etc/besu/genesis.json"]="genesis.json"
EXPECTED_FILES_VALIDATOR["/etc/besu/config-validator.toml"]="config-validator.toml"
EXPECTED_FILES_VALIDATOR["/etc/besu/permissions-nodes.toml"]="permissions-nodes.toml"
EXPECTED_FILES_VALIDATOR["/etc/besu/permissions-accounts.toml"]="permissions-accounts.toml"
EXPECTED_FILES_VALIDATOR["/etc/besu/static-nodes.json"]="static-nodes.json"
# Add validator key directories dynamically
for ((i=1; i<=VALIDATOR_COUNT; i++)); do
EXPECTED_FILES_VALIDATOR["/keys/validators/validator-$i"]="validator keys directory"
done
declare -A EXPECTED_FILES_SENTRY
EXPECTED_FILES_SENTRY["/etc/besu/genesis.json"]="genesis.json"
EXPECTED_FILES_SENTRY["/etc/besu/config-sentry.toml"]="config-sentry.toml"
EXPECTED_FILES_SENTRY["/etc/besu/permissions-nodes.toml"]="permissions-nodes.toml"
EXPECTED_FILES_SENTRY["/etc/besu/permissions-accounts.toml"]="permissions-accounts.toml"
EXPECTED_FILES_SENTRY["/etc/besu/static-nodes.json"]="static-nodes.json"
declare -A EXPECTED_FILES_RPC
EXPECTED_FILES_RPC["/etc/besu/genesis.json"]="genesis.json"
EXPECTED_FILES_RPC["/etc/besu/config-rpc-public.toml"]="config-rpc-public.toml"
EXPECTED_FILES_RPC["/etc/besu/permissions-nodes.toml"]="permissions-nodes.toml"
EXPECTED_FILES_RPC["/etc/besu/permissions-accounts.toml"]="permissions-accounts.toml"
EXPECTED_FILES_RPC["/etc/besu/static-nodes.json"]="static-nodes.json"
log_info "========================================="
log_info "Comprehensive Deployment Validation"
log_info "========================================="
log_info ""
# ============================================================================
# Validation 1: Node Count and Types
# ============================================================================
log_info "=== Validation 1: Node Count and Types ==="
EXPECTED_VALIDATORS=$VALIDATOR_COUNT
EXPECTED_SENTRIES=$SENTRY_COUNT
EXPECTED_RPCS=$RPC_COUNT
EXPECTED_TOTAL=$((EXPECTED_VALIDATORS + EXPECTED_SENTRIES + EXPECTED_RPCS))
ACTUAL_VALIDATORS=0
ACTUAL_SENTRIES=0
ACTUAL_RPCS=0
ACTUAL_TOTAL=0
for vmid in "${ALL_BESU[@]}"; do
# Check if container exists: pct status returns 0 if container exists
if pct status "$vmid" &>/dev/null; then
ACTUAL_TOTAL=$((ACTUAL_TOTAL + 1))
node_type="${VMID_TO_NODE_TYPE[$vmid]}"
case "$node_type" in
validator) ACTUAL_VALIDATORS=$((ACTUAL_VALIDATORS + 1)) ;;
sentry) ACTUAL_SENTRIES=$((ACTUAL_SENTRIES + 1)) ;;
rpc) ACTUAL_RPCS=$((ACTUAL_RPCS + 1)) ;;
esac
fi
done
if [[ $ACTUAL_VALIDATORS -eq $EXPECTED_VALIDATORS ]]; then
log_success "✓ Validator count correct: $ACTUAL_VALIDATORS"
else
log_error "✗ Validator count mismatch: expected $EXPECTED_VALIDATORS, found $ACTUAL_VALIDATORS"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
if [[ $ACTUAL_SENTRIES -eq $EXPECTED_SENTRIES ]]; then
log_success "✓ Sentry count correct: $ACTUAL_SENTRIES"
else
log_error "✗ Sentry count mismatch: expected $EXPECTED_SENTRIES, found $ACTUAL_SENTRIES"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
if [[ $ACTUAL_RPCS -eq $EXPECTED_RPCS ]]; then
log_success "✓ RPC count correct: $ACTUAL_RPCS"
else
log_error "✗ RPC count mismatch: expected $EXPECTED_RPCS, found $ACTUAL_RPCS"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
log_info ""
# ============================================================================
# Validation 2: Correct Templates Used
# ============================================================================
log_info "=== Validation 2: Correct Configuration Templates ==="
for vmid in "${VALIDATORS[@]}"; do
if pct status "$vmid" &>/dev/null; then
if pct exec "$vmid" -- test -f /etc/besu/config-validator.toml 2>/dev/null; then
log_success "✓ Container $vmid: config-validator.toml present"
else
log_error "✗ Container $vmid: config-validator.toml missing"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
# Should NOT have sentry or RPC config
if pct exec "$vmid" -- test -f /etc/besu/config-sentry.toml 2>/dev/null; then
log_error "✗ Container $vmid: config-sentry.toml should not exist (validator)"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
if pct exec "$vmid" -- test -f /etc/besu/config-rpc-public.toml 2>/dev/null; then
log_error "✗ Container $vmid: config-rpc-public.toml should not exist (validator)"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
fi
done
for vmid in "${SENTRIES[@]}"; do
if pct status "$vmid" &>/dev/null; then
if pct exec "$vmid" -- test -f /etc/besu/config-sentry.toml 2>/dev/null; then
log_success "✓ Container $vmid: config-sentry.toml present"
else
log_error "✗ Container $vmid: config-sentry.toml missing"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
fi
done
# RPC nodes - check for type-specific config files
declare -A RPC_VMID_TO_CONFIG
RPC_VMID_TO_CONFIG[2500]="config-rpc-core.toml" # Core RPC
RPC_VMID_TO_CONFIG[2501]="config-rpc-perm.toml" # Permissioned RPC
RPC_VMID_TO_CONFIG[2502]="config-rpc-public.toml" # Public RPC
for vmid in "${RPCS[@]}"; do
if pct status "$vmid" &>/dev/null; then
expected_config="${RPC_VMID_TO_CONFIG[$vmid]}"
if [[ -z "$expected_config" ]]; then
log_warn "⚠ Container $vmid: Unknown RPC type, checking for any RPC config"
expected_config="config-rpc-public.toml"
fi
if pct exec "$vmid" -- test -f "/etc/besu/$expected_config" 2>/dev/null; then
log_success "✓ Container $vmid: $expected_config present"
else
log_error "✗ Container $vmid: $expected_config missing"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
fi
done
log_info ""
# ============================================================================
# Validation 3: All Required Files Copied to Correct Locations
# ============================================================================
log_info "=== Validation 3: Required Files in Correct Locations ==="
validate_files_for_node_type() {
local vmid=$1
local node_type=$2
local -n expected_files=$3
for file_path in "${!expected_files[@]}"; do
local file_desc="${expected_files[$file_path]}"
if pct exec "$vmid" -- test -f "$file_path" 2>/dev/null || pct exec "$vmid" -- test -d "$file_path" 2>/dev/null; then
log_success " ✓ Container $vmid: $file_path exists"
else
log_error " ✗ Container $vmid: $file_path missing ($file_desc)"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
done
}
for vmid in "${VALIDATORS[@]}"; do
if pct status "$vmid" &>/dev/null; then
log_info "Checking validator $vmid..."
validate_files_for_node_type "$vmid" "validator" EXPECTED_FILES_VALIDATOR
fi
done
for vmid in "${SENTRIES[@]}"; do
if pct status "$vmid" &>/dev/null; then
log_info "Checking sentry $vmid..."
validate_files_for_node_type "$vmid" "sentry" EXPECTED_FILES_SENTRY
fi
done
for vmid in "${RPCS[@]}"; do
if pct status "$vmid" &>/dev/null; then
log_info "Checking RPC $vmid..."
validate_files_for_node_type "$vmid" "rpc" EXPECTED_FILES_RPC
fi
done
log_info ""
# ============================================================================
# Validation 4: Genesis.json Validation
# ============================================================================
log_info "=== Validation 4: Genesis.json Validation ==="
# Check genesis.json exists on all nodes and is identical
GENESIS_SAMPLE=""
FIRST_VMID=""
for vmid in "${ALL_BESU[@]}"; do
if pct status "$vmid" &>/dev/null; then
if pct exec "$vmid" -- test -f /etc/besu/genesis.json 2>/dev/null; then
if [[ -z "$GENESIS_SAMPLE" ]]; then
FIRST_VMID="$vmid"
GENESIS_SAMPLE=$(pct exec "$vmid" -- cat /etc/besu/genesis.json 2>/dev/null | python3 -m json.tool 2>/dev/null)
GENESIS_HASH=$(echo "$GENESIS_SAMPLE" | sha256sum | cut -d' ' -f1)
log_success "✓ Genesis.json found on container $vmid"
else
CURRENT_GENESIS=$(pct exec "$vmid" -- cat /etc/besu/genesis.json 2>/dev/null | python3 -m json.tool 2>/dev/null)
CURRENT_HASH=$(echo "$CURRENT_GENESIS" | sha256sum | cut -d' ' -f1)
if [[ "$GENESIS_HASH" == "$CURRENT_HASH" ]]; then
log_success "✓ Container $vmid: genesis.json matches"
else
log_error "✗ Container $vmid: genesis.json differs from container $FIRST_VMID"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
fi
else
log_error "✗ Container $vmid: genesis.json missing"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
fi
done
# Validate genesis.json structure
if [[ -n "$GENESIS_SAMPLE" ]]; then
log_info "Validating genesis.json structure..."
# Check for QBFT configuration
if echo "$GENESIS_SAMPLE" | python3 -c "import sys, json; data=json.load(sys.stdin); exit(0 if 'config' in data and 'qbft' in data.get('config', {}) else 1)" 2>/dev/null; then
log_success "✓ QBFT configuration present in genesis.json"
else
log_error "✗ QBFT configuration missing in genesis.json"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
# Check for extraData (should exist but may be empty for dynamic validators)
if echo "$GENESIS_SAMPLE" | python3 -c "import sys, json; data=json.load(sys.stdin); exit(0 if 'extraData' in data else 1)" 2>/dev/null; then
log_success "✓ extraData field present in genesis.json"
# Validate extraData format (should be hex string)
EXTRA_DATA=$(echo "$GENESIS_SAMPLE" | python3 -c "import sys, json; print(json.load(sys.stdin).get('extraData', ''))" 2>/dev/null)
if [[ "$EXTRA_DATA" =~ ^0x[0-9a-fA-F]*$ ]] || [[ -z "$EXTRA_DATA" ]]; then
log_success "✓ extraData format valid"
else
log_error "✗ extraData format invalid: $EXTRA_DATA"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
else
log_error "✗ extraData field missing in genesis.json"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
# For dynamic validators, QBFT config should exist but validators array should NOT
if echo "$GENESIS_SAMPLE" | python3 -c "import sys, json; data=json.load(sys.stdin); qbft=data.get('config', {}).get('qbft', {}); exit(0 if 'validators' not in qbft else 1)" 2>/dev/null; then
log_success "✓ Dynamic validator setup confirmed (no validators array in QBFT config)"
else
log_warn "⚠ Validators array found in QBFT config (expected for dynamic validators)"
VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1))
fi
else
log_error "✗ Could not validate genesis.json (file not found)"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
log_info ""
# ============================================================================
# Validation 5: Validator Keys Validation
# ============================================================================
log_info "=== Validation 5: Validator Keys Validation ==="
for vmid in "${VALIDATORS[@]}"; do
if pct status "$vmid" &>/dev/null; then
validator_name="${VMID_TO_NODE_NAME[$vmid]}"
keys_dir="/keys/validators/$validator_name"
if pct exec "$vmid" -- test -d "$keys_dir" 2>/dev/null; then
log_success "✓ Container $vmid: Validator keys directory exists: $keys_dir"
# Check for required key files
for key_file in "key.pem" "address.txt"; do
if pct exec "$vmid" -- test -f "$keys_dir/$key_file" 2>/dev/null; then
log_success " ✓ Container $vmid: $key_file exists"
else
log_warn " ⚠ Container $vmid: $key_file missing (may not be critical)"
VALIDATION_WARNINGS=$((VALIDATION_WARNINGS + 1))
fi
done
# Validate address format if present
if pct exec "$vmid" -- test -f "$keys_dir/address.txt" 2>/dev/null; then
ADDRESS=$(pct exec "$vmid" -- cat "$keys_dir/address.txt" 2>/dev/null | tr -d '[:space:]')
if [[ "$ADDRESS" =~ ^0x[0-9a-fA-F]{40}$ ]]; then
log_success " ✓ Container $vmid: Validator address format valid: $ADDRESS"
else
log_error " ✗ Container $vmid: Invalid validator address format: $ADDRESS"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
fi
else
log_error "✗ Container $vmid: Validator keys directory missing: $keys_dir"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
fi
done
log_info ""
# ============================================================================
# Validation 6: Configuration File Consistency
# ============================================================================
log_info "=== Validation 6: Configuration File Consistency ==="
# Check that permissions-nodes.toml is consistent across all nodes
PERMISSIONS_SAMPLE=""
FIRST_VMID=""
for vmid in "${ALL_BESU[@]}"; do
if pct status "$vmid" &>/dev/null; then
if pct exec "$vmid" -- test -f /etc/besu/permissions-nodes.toml 2>/dev/null; then
if [[ -z "$PERMISSIONS_SAMPLE" ]]; then
FIRST_VMID="$vmid"
PERMISSIONS_SAMPLE=$(pct exec "$vmid" -- cat /etc/besu/permissions-nodes.toml 2>/dev/null)
PERMISSIONS_HASH=$(echo "$PERMISSIONS_SAMPLE" | sha256sum | cut -d' ' -f1)
log_success "✓ permissions-nodes.toml found on container $vmid"
else
CURRENT_PERMISSIONS=$(pct exec "$vmid" -- cat /etc/besu/permissions-nodes.toml 2>/dev/null)
CURRENT_HASH=$(echo "$CURRENT_PERMISSIONS" | sha256sum | cut -d' ' -f1)
if [[ "$PERMISSIONS_HASH" == "$CURRENT_HASH" ]]; then
log_success "✓ Container $vmid: permissions-nodes.toml matches"
else
log_error "✗ Container $vmid: permissions-nodes.toml differs from container $FIRST_VMID"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
fi
fi
fi
done
# Check static-nodes.json consistency
STATIC_NODES_SAMPLE=""
FIRST_VMID=""
for vmid in "${ALL_BESU[@]}"; do
if pct status "$vmid" &>/dev/null; then
if pct exec "$vmid" -- test -f /etc/besu/static-nodes.json 2>/dev/null; then
if [[ -z "$STATIC_NODES_SAMPLE" ]]; then
FIRST_VMID="$vmid"
STATIC_NODES_SAMPLE=$(pct exec "$vmid" -- cat /etc/besu/static-nodes.json 2>/dev/null | python3 -m json.tool 2>/dev/null)
STATIC_NODES_HASH=$(echo "$STATIC_NODES_SAMPLE" | sha256sum | cut -d' ' -f1)
log_success "✓ static-nodes.json found on container $vmid"
else
CURRENT_STATIC=$(pct exec "$vmid" -- cat /etc/besu/static-nodes.json 2>/dev/null | python3 -m json.tool 2>/dev/null)
CURRENT_HASH=$(echo "$CURRENT_STATIC" | sha256sum | cut -d' ' -f1)
if [[ "$STATIC_NODES_HASH" == "$CURRENT_HASH" ]]; then
log_success "✓ Container $vmid: static-nodes.json matches"
else
log_error "✗ Container $vmid: static-nodes.json differs from container $FIRST_VMID"
VALIDATION_ERRORS=$((VALIDATION_ERRORS + 1))
fi
fi
fi
fi
done
log_info ""
# ============================================================================
# Summary
# ============================================================================
log_info "========================================="
log_info "Validation Summary"
log_info "========================================="
log_info "Errors: $VALIDATION_ERRORS"
log_info "Warnings: $VALIDATION_WARNINGS"
log_info ""
if [[ $VALIDATION_ERRORS -eq 0 ]]; then
if [[ $VALIDATION_WARNINGS -eq 0 ]]; then
log_success "✓ All validations passed!"
exit 0
else
log_warn "⚠ Validations passed with $VALIDATION_WARNINGS warning(s)"
log_info "Review warnings above, but deployment is acceptable"
exit 0
fi
else
log_error "✗ Validation failed with $VALIDATION_ERRORS error(s)"
log_info "Please fix the errors before proceeding"
exit 1
fi