fbda1b4beb
Some checks failed
Deploy to Phoenix / deploy (push) Has been cancelled
- ADD_CHAIN138_TO_LEDGER_LIVE: Ledger form done; public code review repo bis-innovations/LedgerLive; init/push commands - CONTRACT_DEPLOYMENT_RUNBOOK: Chain 138 gas price 1 gwei, 36-addr check, TransactionMirror workaround - CONTRACT_*: AddressMapper, MirrorManager deployed 2026-02-12; 36-address on-chain check - NEXT_STEPS_FOR_YOU: Ledger done; steps completable now (no LAN); run-completable-tasks-from-anywhere - MASTER_INDEX, OPERATOR_OPTIONAL, SMART_CONTRACTS_INVENTORY_SIMPLE: updates - LEDGER_BLOCKCHAIN_INTEGRATION_COMPLETE: bis-innovations/LedgerLive reference Co-authored-by: Cursor <cursoragent@cursor.com>
244 lines
8.1 KiB
Bash
Executable File
244 lines
8.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Fix Remaining Migrations - Use API-based method for r630-02
|
|
# Migrate containers to r630-02 using proper storage specification
|
|
|
|
set -euo pipefail
|
|
|
|
# Load IP configuration
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
|
|
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
REPORT_DIR="${PROJECT_ROOT}/reports/status"
|
|
TIMESTAMP=$(date +%Y%m%d_%H%M%S)
|
|
FIX_LOG="${REPORT_DIR}/fix_migrations_${TIMESTAMP}.log"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
MAGENTA='\033[0;35m'
|
|
NC='\033[0m'
|
|
|
|
log_info() { echo -e "${BLUE}[INFO]${NC} $1" | tee -a "$FIX_LOG"; }
|
|
log_success() { echo -e "${GREEN}[✓]${NC} $1" | tee -a "$FIX_LOG"; }
|
|
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1" | tee -a "$FIX_LOG"; }
|
|
log_error() { echo -e "${RED}[✗]${NC} $1" | tee -a "$FIX_LOG"; }
|
|
log_header() { echo -e "${CYAN}=== $1 ===${NC}" | tee -a "$FIX_LOG"; }
|
|
log_section() { echo -e "\n${MAGENTA}>>> $1 <<<${NC}\n" | tee -a "$FIX_LOG"; }
|
|
|
|
mkdir -p "$REPORT_DIR"
|
|
|
|
declare -A NODES
|
|
NODES[ml110]="${PROXMOX_HOST_ML110:-192.168.11.10}:L@kers2010"
|
|
NODES[r630-02]="${PROXMOX_HOST_R630_02:-192.168.11.12}:password"
|
|
|
|
ssh_node() {
|
|
local hostname="$1"
|
|
shift
|
|
local ip="${NODES[$hostname]%%:*}"
|
|
local password="${NODES[$hostname]#*:}"
|
|
|
|
if command -v sshpass >/dev/null 2>&1; then
|
|
sshpass -p "$password" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@"$ip" "$@"
|
|
else
|
|
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 root@"$ip" "$@"
|
|
fi
|
|
}
|
|
|
|
check_node() {
|
|
local hostname="$1"
|
|
local ip="${NODES[$hostname]%%:*}"
|
|
ping -c 1 -W 2 "$ip" >/dev/null 2>&1
|
|
}
|
|
|
|
# Migrate container using API method
|
|
migrate_with_storage() {
|
|
local vmid=$1
|
|
local source_node=$2
|
|
local target_node=$3
|
|
local target_storage=$4
|
|
local container_name=$5
|
|
|
|
log_info "Migrating CT $vmid ($container_name) to $target_node using storage $target_storage..."
|
|
|
|
# Check if container exists and is running
|
|
local status=$(ssh_node "$source_node" "pct status $vmid 2>/dev/null | awk '{print \$2}'" || echo "not_found")
|
|
|
|
if [ "$status" = "not_found" ]; then
|
|
log_error "Container $vmid not found on $source_node"
|
|
return 1
|
|
fi
|
|
|
|
log_info " Current status: $status"
|
|
|
|
# Use API-based migration with storage parameter
|
|
log_info " Using API migration method with storage: $target_storage"
|
|
|
|
local migrate_result=$(ssh_node "$source_node" bash <<ENDSSH
|
|
# Use pvesh API to migrate with storage specification
|
|
pvesh create /nodes/$source_node/lxc/$vmid/migrate \\
|
|
--target $target_node \\
|
|
--storage $target_storage \\
|
|
--online 1 2>&1
|
|
ENDSSH
|
|
)
|
|
|
|
local migrate_exit=$?
|
|
|
|
if echo "$migrate_result" | grep -q "error\|Error\|ERROR\|aborted"; then
|
|
log_error " Migration failed: $migrate_result"
|
|
return 1
|
|
else
|
|
log_success " Migration command executed successfully"
|
|
log_info " Migration output: $migrate_result"
|
|
|
|
# Wait and verify
|
|
log_info " Waiting for migration to complete..."
|
|
sleep 10
|
|
|
|
# Check if container is now on target node
|
|
local target_status=$(ssh_node "$target_node" "pct status $vmid 2>/dev/null | awk '{print \$2}'" || echo "not_found")
|
|
|
|
if [ "$target_status" != "not_found" ]; then
|
|
log_success " Container $vmid is now on $target_node (status: $target_status)"
|
|
return 0
|
|
else
|
|
log_warn " Container not yet visible on target node (migration may still be in progress)"
|
|
return 0
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Fix thin2 capacity by migrating containers
|
|
fix_thin2_capacity() {
|
|
log_section "Fixing thin2 Capacity Issue"
|
|
|
|
local hostname="r630-02"
|
|
if ! check_node "$hostname"; then
|
|
log_error "$hostname is not reachable"
|
|
return 1
|
|
fi
|
|
|
|
# Available storage with good capacity
|
|
local target_storage="thin1-r630-02" # 0.34% used, 225.36 GiB available
|
|
|
|
# Migrate CT 5000 (blockscout-1)
|
|
log_info "Migrating CT 5000 (blockscout-1) from thin2 to $target_storage..."
|
|
|
|
local migrate_result=$(ssh_node "$hostname" bash <<ENDSSH
|
|
# Get container config to determine size
|
|
size=\$(pct config 5000 2>/dev/null | grep "^rootfs:" | grep -oP 'size=\K[^,]+' || echo "200G")
|
|
echo "Container size: \$size"
|
|
|
|
# Stop container first for safer migration
|
|
pct stop 5000 2>&1 || echo "Already stopped or stop failed"
|
|
sleep 3
|
|
|
|
# Use API to migrate within same node but change storage
|
|
# Note: This requires using move-disk or changing storage
|
|
# Let's check if we can use move-disk
|
|
pvesh create /nodes/$hostname/lxc/5000/migrate \\
|
|
--target $hostname \\
|
|
--storage $target_storage \\
|
|
--online 0 2>&1
|
|
ENDSSH
|
|
)
|
|
|
|
log_info "Migration result: $migrate_result"
|
|
|
|
# For same-node storage migration, we might need to use move-disk instead
|
|
log_info "Attempting storage move using move-disk API..."
|
|
|
|
local move_result=$(ssh_node "$hostname" bash <<ENDSSH
|
|
# Get current disk
|
|
disk=\$(pct config 5000 2>/dev/null | grep "^rootfs:" | awk '{print \$2}' | cut -d: -f2)
|
|
echo "Current disk: \$disk"
|
|
|
|
# Move disk to new storage
|
|
pvesh create /nodes/$hostname/lxc/5000/move_disk \\
|
|
--disk rootfs \\
|
|
--storage $target_storage \\
|
|
--delete 1 2>&1 || echo "Move disk failed"
|
|
ENDSSH
|
|
)
|
|
|
|
log_info "Move disk result: $move_result"
|
|
log_success "thin2 capacity fix attempted."
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
log_header "Fixing Remaining Migrations to r630-02"
|
|
echo "Log file: $FIX_LOG" | tee -a "$FIX_LOG"
|
|
echo "Timestamp: $(date)" | tee -a "$FIX_LOG"
|
|
echo "" | tee -a "$FIX_LOG"
|
|
|
|
local source_node="ml110"
|
|
local target_node="r630-02"
|
|
local target_storage="thin1-r630-02" # Best available storage on r630-02
|
|
|
|
if ! check_node "$source_node" || ! check_node "$target_node"; then
|
|
log_error "One or more nodes are not reachable"
|
|
return 1
|
|
fi
|
|
|
|
# Containers to migrate
|
|
declare -A containers
|
|
containers[1003]="besu-validator-4"
|
|
containers[1004]="besu-validator-5"
|
|
containers[1503]="besu-sentry-4"
|
|
containers[1504]="besu-sentry-ali"
|
|
containers[2201]="besu-rpc-public-1"
|
|
containers[2303]="besu-rpc-ali-0x8a"
|
|
containers[2401]="besu-rpc-thirdweb-0x8a-1"
|
|
|
|
log_section "Migrating Containers to r630-02"
|
|
|
|
local success_count=0
|
|
local fail_count=0
|
|
|
|
for vmid in "${!containers[@]}"; do
|
|
local name="${containers[$vmid]}"
|
|
|
|
if migrate_with_storage "$vmid" "$source_node" "$target_node" "$target_storage" "$name"; then
|
|
((success_count++))
|
|
sleep 5 # Wait between migrations
|
|
else
|
|
((fail_count++))
|
|
log_warn "Will retry $vmid later"
|
|
fi
|
|
done
|
|
|
|
log_section "Migration Summary"
|
|
log_info "Successfully migrated: $success_count containers"
|
|
log_info "Failed: $fail_count containers"
|
|
|
|
# Fix thin2 capacity
|
|
fix_thin2_capacity
|
|
|
|
# Verify final state
|
|
log_section "Final System State"
|
|
|
|
for hostname in "$source_node" "$target_node"; do
|
|
if check_node "$hostname"; then
|
|
local container_count=$(ssh_node "$hostname" "pct list 2>/dev/null | tail -n +2 | wc -l" || echo "0")
|
|
local running_count=$(ssh_node "$hostname" "pct list 2>/dev/null | grep running | wc -l" || echo "0")
|
|
local cpu_usage=$(ssh_node "$hostname" "top -bn1 | grep 'Cpu(s)' | awk '{print \$2}' | sed 's/%us,//' || echo 'N/A'")
|
|
|
|
log_info "$hostname: $container_count containers ($running_count running), CPU: $cpu_usage%"
|
|
fi
|
|
done
|
|
|
|
log_header "Fix Execution Complete"
|
|
log_info "Full log saved to: $FIX_LOG"
|
|
log_success "Remaining migrations have been attempted!"
|
|
}
|
|
|
|
main "$@"
|