dbd517b279
- Update dbis_core, cross-chain-pmm-lps, explorer-monorepo, metamask-integration, pr-workspace/chains - Omit embedded publish git dirs and empty placeholders from index Made-with: Cursor
506 lines
18 KiB
Bash
Executable File
506 lines
18 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Deploy Validated Set - Complete Deployment Orchestrator
|
|
# Script-based deployment for Besu validated set (no boot node required)
|
|
#
|
|
# This script orchestrates the complete deployment of Besu nodes:
|
|
# 1. Deploy containers (validators, sentries, RPC nodes)
|
|
# 2. Copy configuration files and keys
|
|
# 3. Bootstrap network (generate static-nodes.json)
|
|
# 4. Validate deployment
|
|
#
|
|
# Features:
|
|
# - Progress tracking with ETA
|
|
# - Comprehensive error handling
|
|
# - Rollback support
|
|
# - Detailed logging
|
|
# - Timeout protection
|
|
# - Container status validation
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
|
|
# Source libraries with error handling
|
|
source "$PROJECT_ROOT/lib/common.sh" || {
|
|
echo "ERROR: Failed to load common.sh library" >&2
|
|
exit 1
|
|
}
|
|
|
|
# Optional: Load progress tracking if available
|
|
if [[ -f "$PROJECT_ROOT/lib/progress-tracking.sh" ]]; then
|
|
source "$PROJECT_ROOT/lib/progress-tracking.sh"
|
|
USE_ADVANCED_PROGRESS=true
|
|
else
|
|
USE_ADVANCED_PROGRESS=false
|
|
fi
|
|
|
|
# Parse command line arguments
|
|
SKIP_DEPLOYMENT="${SKIP_DEPLOYMENT:-false}"
|
|
SKIP_CONFIG="${SKIP_CONFIG:-false}"
|
|
SKIP_BOOTSTRAP="${SKIP_BOOTSTRAP:-false}"
|
|
SKIP_VALIDATION="${SKIP_VALIDATION:-false}"
|
|
SOURCE_PROJECT="${SOURCE_PROJECT:-}"
|
|
TIMEOUT_DEPLOYMENT="${TIMEOUT_DEPLOYMENT:-3600}" # 1 hour default
|
|
TIMEOUT_CONFIG="${TIMEOUT_CONFIG:-1800}" # 30 minutes default
|
|
TIMEOUT_BOOTSTRAP="${TIMEOUT_BOOTSTRAP:-300}" # 5 minutes default
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case $1 in
|
|
--skip-deployment)
|
|
SKIP_DEPLOYMENT=true
|
|
shift
|
|
;;
|
|
--skip-config)
|
|
SKIP_CONFIG=true
|
|
shift
|
|
;;
|
|
--skip-bootstrap)
|
|
SKIP_BOOTSTRAP=true
|
|
shift
|
|
;;
|
|
--skip-validation)
|
|
SKIP_VALIDATION=true
|
|
shift
|
|
;;
|
|
--source-project)
|
|
SOURCE_PROJECT="$2"
|
|
shift 2
|
|
;;
|
|
--timeout-deployment)
|
|
TIMEOUT_DEPLOYMENT="$2"
|
|
shift 2
|
|
;;
|
|
--timeout-config)
|
|
TIMEOUT_CONFIG="$2"
|
|
shift 2
|
|
;;
|
|
--help)
|
|
cat << EOF
|
|
Usage: $0 [OPTIONS]
|
|
|
|
Deploy Validated Set - Complete Besu Network Deployment Orchestrator
|
|
|
|
Options:
|
|
--skip-deployment Skip container deployment (assume containers exist)
|
|
--skip-config Skip configuration file copying
|
|
--skip-bootstrap Skip network bootstrap
|
|
--skip-validation Skip validation
|
|
--source-project PATH Path to source project with config files
|
|
(default: ../smom-dbis-138 relative to project root)
|
|
--timeout-deployment SEC Timeout for deployment phase (default: 3600)
|
|
--timeout-config SEC Timeout for config phase (default: 1800)
|
|
--timeout-bootstrap SEC Timeout for bootstrap phase (default: 300)
|
|
--help Show this help message
|
|
|
|
Examples:
|
|
# Full deployment
|
|
$0 --source-project /opt/smom-dbis-138
|
|
|
|
# Skip deployment, only configure
|
|
$0 --skip-deployment --source-project /opt/smom-dbis-138
|
|
|
|
# Only validate existing deployment
|
|
$0 --skip-deployment --skip-config --skip-bootstrap
|
|
EOF
|
|
exit 0
|
|
;;
|
|
*)
|
|
log_error "Unknown option: $1"
|
|
log_info "Use --help for usage information"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# Load configuration
|
|
load_config || {
|
|
log_warn "Failed to load main configuration, using defaults"
|
|
}
|
|
load_config "$PROJECT_ROOT/config/network.conf" || {
|
|
log_warn "network.conf not found, using defaults"
|
|
}
|
|
|
|
# Track deployment start time
|
|
DEPLOYMENT_START_TIME=$(date +%s)
|
|
|
|
# Display header
|
|
log_info "========================================="
|
|
log_info "Deploy Validated Set - Script-Based Approach"
|
|
log_info "========================================="
|
|
log_info "Started: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
log_info ""
|
|
|
|
# Calculate total steps based on what's being executed
|
|
TOTAL_STEPS=0
|
|
[[ "$SKIP_DEPLOYMENT" != "true" ]] && TOTAL_STEPS=$((TOTAL_STEPS + 1))
|
|
[[ "$SKIP_CONFIG" != "true" ]] && TOTAL_STEPS=$((TOTAL_STEPS + 1))
|
|
[[ "$SKIP_BOOTSTRAP" != "true" ]] && TOTAL_STEPS=$((TOTAL_STEPS + 1))
|
|
[[ "$SKIP_VALIDATION" != "true" ]] && TOTAL_STEPS=$((TOTAL_STEPS + 1))
|
|
|
|
CURRENT_STEP=0
|
|
|
|
# Enhanced progress tracking function
|
|
show_progress() {
|
|
local step_name="$1"
|
|
local phase_start_time="${2:-$(date +%s)}"
|
|
CURRENT_STEP=$((CURRENT_STEP + 1))
|
|
local percent=$((CURRENT_STEP * 100 / TOTAL_STEPS))
|
|
|
|
if [[ "$USE_ADVANCED_PROGRESS" == "true" ]] && command_exists update_progress 2>/dev/null; then
|
|
update_progress "$CURRENT_STEP" "$step_name"
|
|
else
|
|
log_info "Progress: [$percent%] [$CURRENT_STEP/$TOTAL_STEPS] - $step_name"
|
|
fi
|
|
}
|
|
|
|
# Time tracking function
|
|
track_phase_time() {
|
|
local phase_name="$1"
|
|
local start_time="$2"
|
|
local end_time=$(date +%s)
|
|
local duration=$((end_time - start_time))
|
|
local minutes=$((duration / 60))
|
|
local seconds=$((duration % 60))
|
|
log_info "$phase_name completed in ${minutes}m ${seconds}s"
|
|
}
|
|
|
|
# Run command with timeout
|
|
run_with_timeout() {
|
|
local timeout_sec="$1"
|
|
shift
|
|
local cmd=("$@")
|
|
|
|
if command_exists timeout; then
|
|
timeout "$timeout_sec" "${cmd[@]}" || {
|
|
local exit_code=$?
|
|
if [[ $exit_code -eq 124 ]]; then
|
|
log_error "Command timed out after ${timeout_sec} seconds: ${cmd[*]}"
|
|
return 124
|
|
fi
|
|
return $exit_code
|
|
}
|
|
else
|
|
# Fallback: run without timeout if timeout command not available
|
|
"${cmd[@]}"
|
|
fi
|
|
}
|
|
|
|
# Check prerequisites
|
|
log_info "=== Pre-Deployment Validation ==="
|
|
|
|
if ! command_exists pct; then
|
|
error_exit "pct command not found. This script must be run on Proxmox host."
|
|
fi
|
|
|
|
if [[ $EUID -ne 0 ]]; then
|
|
error_exit "This script must be run as root"
|
|
fi
|
|
|
|
# Validate project structure
|
|
if [[ ! -d "$PROJECT_ROOT/scripts" ]]; then
|
|
error_exit "Invalid project structure: scripts directory not found"
|
|
fi
|
|
|
|
if [[ ! -d "$PROJECT_ROOT/config" ]]; then
|
|
log_warn "config directory not found, some features may not work"
|
|
fi
|
|
|
|
log_success "Prerequisites checked"
|
|
|
|
# Ensure OS template exists
|
|
log_info "Checking OS template..."
|
|
ensure_os_template "${CONTAINER_OS_TEMPLATE:-local:vztmpl/ubuntu-22.04-standard_22.04-1_amd64.tar.zst}" || {
|
|
error_exit "OS template not available. Please download it first: pveam download local ubuntu-22.04-standard_22.04-1_amd64.tar.zst"
|
|
}
|
|
|
|
# Create log directory early
|
|
mkdir -p "$PROJECT_ROOT/logs"
|
|
LOG_FILE="$PROJECT_ROOT/logs/deploy-validated-set-$(date +%Y%m%d-%H%M%S).log"
|
|
log_info "Deployment log: $LOG_FILE"
|
|
|
|
# Setup logging - preserve stdout/stderr for interactive use, also log to file
|
|
# Use exec to redirect, but preserve original descriptors
|
|
exec 3>&1 4>&2
|
|
exec 1> >(tee -a "$LOG_FILE")
|
|
exec 2> >(tee -a "$LOG_FILE" >&2)
|
|
|
|
# Create snapshot helper function
|
|
create_snapshot_if_needed() {
|
|
local vmid=$1
|
|
local reason=$2
|
|
if command -v pct >/dev/null 2>&1 && [[ -f "$PROJECT_ROOT/scripts/manage/snapshot-before-change.sh" ]]; then
|
|
log_info "Creating snapshot for container $vmid before $reason..."
|
|
"$PROJECT_ROOT/scripts/manage/snapshot-before-change.sh" "$vmid" "pre-$reason-$(date +%Y%m%d-%H%M%S)" || {
|
|
log_warn "Snapshot creation failed (continuing anyway)"
|
|
}
|
|
fi
|
|
}
|
|
|
|
# Initialize rollback tracking
|
|
if command_exists init_rollback 2>/dev/null; then
|
|
ROLLBACK_LOG="$PROJECT_ROOT/logs/rollback-$(date +%Y%m%d-%H%M%S).log"
|
|
init_rollback "$ROLLBACK_LOG"
|
|
set_rollback_trap
|
|
log_info "Rollback tracking enabled: $ROLLBACK_LOG"
|
|
fi
|
|
|
|
# Initialize advanced progress tracking if available
|
|
if [[ "$USE_ADVANCED_PROGRESS" == "true" ]] && command_exists init_progress_tracking 2>/dev/null; then
|
|
init_progress_tracking "$TOTAL_STEPS" "Deployment"
|
|
fi
|
|
|
|
# Resolve SOURCE_PROJECT path early if provided
|
|
if [[ -n "$SOURCE_PROJECT" ]]; then
|
|
# Resolve absolute path for SOURCE_PROJECT if relative
|
|
if [[ "$SOURCE_PROJECT" != /* ]]; then
|
|
SOURCE_PROJECT="$(cd "$PROJECT_ROOT" && cd "$SOURCE_PROJECT" 2>/dev/null && pwd || echo "$PROJECT_ROOT/$SOURCE_PROJECT")"
|
|
fi
|
|
|
|
if [[ ! -d "$SOURCE_PROJECT" ]]; then
|
|
log_error "Source project directory not found: $SOURCE_PROJECT"
|
|
exit 1
|
|
fi
|
|
|
|
log_info "Source project: $SOURCE_PROJECT"
|
|
fi
|
|
|
|
log_info ""
|
|
|
|
# Phase 1: Deploy Containers
|
|
if [[ "$SKIP_DEPLOYMENT" != "true" ]]; then
|
|
PHASE_START_1=$(date +%s)
|
|
show_progress "Deploy Containers" "$PHASE_START_1"
|
|
log_info ""
|
|
log_info "========================================="
|
|
log_info "Phase 1: Deploy Containers"
|
|
log_info "========================================="
|
|
log_info "Timeout: ${TIMEOUT_DEPLOYMENT}s"
|
|
log_info ""
|
|
|
|
if [[ -f "$SCRIPT_DIR/deploy-besu-nodes.sh" ]]; then
|
|
log_info "Deploying Besu nodes..."
|
|
|
|
if ! run_with_timeout "$TIMEOUT_DEPLOYMENT" bash "$SCRIPT_DIR/deploy-besu-nodes.sh"; then
|
|
log_error "Besu nodes deployment failed or timed out"
|
|
log_error "Check logs for details: $LOG_FILE"
|
|
exit 1
|
|
fi
|
|
|
|
track_phase_time "Container deployment" "$PHASE_START_1"
|
|
log_success "Besu nodes deployed"
|
|
else
|
|
log_error "Besu deployment script not found: $SCRIPT_DIR/deploy-besu-nodes.sh"
|
|
exit 1
|
|
fi
|
|
else
|
|
log_info "Skipping container deployment (--skip-deployment)"
|
|
log_info "Assuming containers already exist"
|
|
fi
|
|
|
|
# Phase 2: Copy Configuration Files
|
|
if [[ "$SKIP_CONFIG" != "true" ]]; then
|
|
PHASE_START_2=$(date +%s)
|
|
show_progress "Copy Configuration Files" "$PHASE_START_2"
|
|
log_info ""
|
|
log_info "========================================="
|
|
log_info "Phase 2: Copy Configuration Files"
|
|
log_info "========================================="
|
|
log_info "Timeout: ${TIMEOUT_CONFIG}s"
|
|
log_info ""
|
|
|
|
if [[ -z "$SOURCE_PROJECT" ]]; then
|
|
# Try default location
|
|
DEFAULT_SOURCE="$PROJECT_ROOT/../smom-dbis-138"
|
|
if [[ -d "$DEFAULT_SOURCE" ]]; then
|
|
SOURCE_PROJECT="$DEFAULT_SOURCE"
|
|
log_info "Using default source project: $SOURCE_PROJECT"
|
|
else
|
|
log_warn "Source project not specified and default not found: $DEFAULT_SOURCE"
|
|
log_warn "Skipping configuration copy"
|
|
log_info "Specify with: --source-project PATH"
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "$SOURCE_PROJECT" ]] && [[ -d "$SOURCE_PROJECT" ]]; then
|
|
# Run prerequisites check first
|
|
if [[ -f "$PROJECT_ROOT/scripts/validation/check-prerequisites.sh" ]]; then
|
|
log_info "Checking prerequisites..."
|
|
if ! bash "$PROJECT_ROOT/scripts/validation/check-prerequisites.sh" "$SOURCE_PROJECT"; then
|
|
log_error "Prerequisites check failed"
|
|
log_error "Review the output above and fix issues before continuing"
|
|
exit 1
|
|
fi
|
|
log_success "Prerequisites validated"
|
|
fi
|
|
|
|
export SOURCE_PROJECT
|
|
copy_success=false
|
|
|
|
# Try enhanced copy script first (supports config/nodes/ structure), then fallback
|
|
if [[ -f "$PROJECT_ROOT/scripts/copy-besu-config-with-nodes.sh" ]]; then
|
|
log_info "Copying Besu configuration files (with nodes/ structure support)..."
|
|
if run_with_timeout "$TIMEOUT_CONFIG" bash "$PROJECT_ROOT/scripts/copy-besu-config-with-nodes.sh" "$SOURCE_PROJECT"; then
|
|
copy_success=true
|
|
log_success "Configuration files copied (with nodes/ structure)"
|
|
else
|
|
log_warn "Enhanced copy script failed, trying standard copy script..."
|
|
fi
|
|
fi
|
|
|
|
# Fallback to standard copy script
|
|
if [[ "$copy_success" != "true" ]] && [[ -f "$PROJECT_ROOT/scripts/copy-besu-config.sh" ]]; then
|
|
log_info "Copying Besu configuration files (standard method)..."
|
|
if ! run_with_timeout "$TIMEOUT_CONFIG" bash "$PROJECT_ROOT/scripts/copy-besu-config.sh" "$SOURCE_PROJECT"; then
|
|
log_error "Configuration copy failed"
|
|
log_error "Check logs for details: $LOG_FILE"
|
|
exit 1
|
|
fi
|
|
copy_success=true
|
|
log_success "Configuration files copied"
|
|
fi
|
|
|
|
if [[ "$copy_success" != "true" ]]; then
|
|
log_error "No working copy script available"
|
|
log_error "Checked:"
|
|
log_error " - $PROJECT_ROOT/scripts/copy-besu-config-with-nodes.sh"
|
|
log_error " - $PROJECT_ROOT/scripts/copy-besu-config.sh"
|
|
exit 1
|
|
fi
|
|
|
|
track_phase_time "Configuration copy" "$PHASE_START_2"
|
|
else
|
|
log_warn "Source project not available, skipping configuration copy"
|
|
log_info "You can copy configuration files manually or run:"
|
|
log_info " $PROJECT_ROOT/scripts/copy-besu-config.sh $SOURCE_PROJECT"
|
|
fi
|
|
else
|
|
log_info "Skipping configuration copy (--skip-config)"
|
|
fi
|
|
|
|
# Phase 3: Bootstrap Network
|
|
if [[ "$SKIP_BOOTSTRAP" != "true" ]]; then
|
|
PHASE_START_3=$(date +%s)
|
|
show_progress "Bootstrap Network" "$PHASE_START_3"
|
|
log_info ""
|
|
log_info "========================================="
|
|
log_info "Phase 3: Bootstrap Network"
|
|
log_info "========================================="
|
|
log_info "Timeout: ${TIMEOUT_BOOTSTRAP}s"
|
|
log_info ""
|
|
|
|
if [[ -f "$PROJECT_ROOT/scripts/network/bootstrap-network.sh" ]]; then
|
|
log_info "Bootstrapping network (generating static-nodes.json)..."
|
|
|
|
if ! run_with_timeout "$TIMEOUT_BOOTSTRAP" bash "$PROJECT_ROOT/scripts/network/bootstrap-network.sh"; then
|
|
log_error "Network bootstrap failed or timed out"
|
|
log_error "Check logs for details: $LOG_FILE"
|
|
exit 1
|
|
fi
|
|
|
|
track_phase_time "Network bootstrap" "$PHASE_START_3"
|
|
log_success "Network bootstrapped"
|
|
else
|
|
log_error "Bootstrap script not found: $PROJECT_ROOT/scripts/network/bootstrap-network.sh"
|
|
exit 1
|
|
fi
|
|
else
|
|
log_info "Skipping network bootstrap (--skip-bootstrap)"
|
|
fi
|
|
|
|
# Phase 4: Validate Deployment
|
|
if [[ "$SKIP_VALIDATION" != "true" ]]; then
|
|
PHASE_START_4=$(date +%s)
|
|
show_progress "Validate Deployment" "$PHASE_START_4"
|
|
log_info ""
|
|
log_info "========================================="
|
|
log_info "Phase 4: Validate Deployment"
|
|
log_info "========================================="
|
|
log_info ""
|
|
|
|
# Comprehensive deployment validation
|
|
if [[ -f "$PROJECT_ROOT/scripts/validation/validate-deployment-comprehensive.sh" ]]; then
|
|
log_info "Running comprehensive deployment validation..."
|
|
if ! bash "$PROJECT_ROOT/scripts/validation/validate-deployment-comprehensive.sh"; then
|
|
log_error "Comprehensive validation failed (check output above)"
|
|
log_error "This validation ensures:"
|
|
log_error " 1. All config files are in correct locations"
|
|
log_error " 2. Genesis.json is valid and consistent"
|
|
log_error " 3. Correct number of nodes deployed"
|
|
log_error " 4. Correct templates used for each node type"
|
|
log_error " 5. No inconsistencies or gaps"
|
|
log_info ""
|
|
log_info "Review validation output and fix issues before proceeding"
|
|
exit 1
|
|
fi
|
|
track_phase_time "Deployment validation" "$PHASE_START_4"
|
|
log_success "Comprehensive validation passed"
|
|
else
|
|
log_warn "Comprehensive validation script not found, running basic validation..."
|
|
|
|
# Fallback to basic validation
|
|
if [[ -f "$PROJECT_ROOT/scripts/validation/validate-validator-set.sh" ]]; then
|
|
log_info "Validating validator set..."
|
|
if bash "$PROJECT_ROOT/scripts/validation/validate-validator-set.sh"; then
|
|
log_success "Validator set validation passed"
|
|
else
|
|
log_warn "Validator set validation had issues (check output above)"
|
|
fi
|
|
else
|
|
log_warn "Validator validation script not found, skipping validation"
|
|
fi
|
|
fi
|
|
else
|
|
log_info "Skipping validation (--skip-validation)"
|
|
fi
|
|
|
|
# Calculate total deployment time
|
|
DEPLOYMENT_END_TIME=$(date +%s)
|
|
TOTAL_DURATION=$((DEPLOYMENT_END_TIME - DEPLOYMENT_START_TIME))
|
|
TOTAL_MINUTES=$((TOTAL_DURATION / 60))
|
|
TOTAL_SECONDS=$((TOTAL_DURATION % 60))
|
|
|
|
# Complete progress tracking if using advanced tracking
|
|
if [[ "$USE_ADVANCED_PROGRESS" == "true" ]] && command_exists complete_progress 2>/dev/null; then
|
|
complete_progress
|
|
fi
|
|
|
|
# Final Summary
|
|
log_info ""
|
|
log_info "========================================="
|
|
log_success "Deployment Complete!"
|
|
log_info "========================================="
|
|
log_info "Completed: $(date '+%Y-%m-%d %H:%M:%S')"
|
|
log_info "Total duration: ${TOTAL_MINUTES}m ${TOTAL_SECONDS}s"
|
|
log_info ""
|
|
log_info "Deployment Summary:"
|
|
log_info " - Containers: $([ "$SKIP_DEPLOYMENT" != "true" ] && echo "✓ Deployed" || echo "⊘ Skipped")"
|
|
log_info " - Configuration: $([ "$SKIP_CONFIG" != "true" ] && echo "✓ Copied" || echo "⊘ Skipped")"
|
|
log_info " - Bootstrap: $([ "$SKIP_BOOTSTRAP" != "true" ] && echo "✓ Completed" || echo "⊘ Skipped")"
|
|
log_info " - Validation: $([ "$SKIP_VALIDATION" != "true" ] && echo "✓ Completed" || echo "⊘ Skipped")"
|
|
log_info ""
|
|
log_info "Next Steps:"
|
|
log_info "1. Verify all services are running:"
|
|
log_info " for vmid in 1000 1001 1002 1003 1004 1500 1501 1502 1503 2101 2102 2103 2201 2301 2303 2304 2305 2306 2307 2308 2400 2401 2402 2403 2420 2430 2440 2460 2470 2480; do"
|
|
log_info " pct exec \$vmid -- systemctl status besu-validator besu-sentry besu-rpc 2>/dev/null"
|
|
log_info " done"
|
|
log_info ""
|
|
log_info "2. Start Besu services (if not already started):"
|
|
log_info " $PROJECT_ROOT/scripts/fix-besu-services.sh"
|
|
log_info ""
|
|
log_info "3. Check consensus is active (blocks being produced):"
|
|
log_info " pct exec 1000 -- curl -X POST -H 'Content-Type: application/json' \\"
|
|
log_info " --data '{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":1}' \\"
|
|
log_info " http://localhost:8545"
|
|
log_info ""
|
|
log_info "4. Monitor logs for any issues:"
|
|
log_info " pct exec 1000 -- journalctl -u besu-validator -f"
|
|
log_info ""
|
|
log_info "Deployment log: $LOG_FILE"
|
|
if [[ -n "${ROLLBACK_LOG:-}" ]]; then
|
|
log_info "Rollback log: $ROLLBACK_LOG"
|
|
fi
|
|
log_info ""
|
|
|
|
exit 0
|