proxmox/scripts/migrate-2-containers-to-pve2-thin1.sh

#!/usr/bin/env bash
# Migrate 2 containers to pve2 using thin1 LVM storage
# This uses the thin1 LVM thin pool created on pve2

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"
PROXMOX_PASS="${PROXMOX_PASS:-L@kers2010}"
SOURCE_NODE="ml110"
TARGET_NODE="pve2"
TARGET_STORAGE="thin1"  # Use thin1 LVM storage

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }
log_header() { echo -e "${CYAN}[$1]${NC} $2"; }

# SSH helper
ssh_proxmox() {
    sshpass -p "$PROXMOX_PASS" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 root@"$PROXMOX_HOST" "$@"
}

# Check if container exists and get its status
check_container() {
    local vmid=$1
    local node=$2
    ssh_proxmox "pvesh get /nodes/$node/lxc/$vmid/status/current --output-format json" 2>&1 | python3 -c "import sys, json; d=json.load(sys.stdin); print(d.get('status', 'unknown'))" 2>/dev/null || echo "not_found"
}

# Migrate a single container using thin1 storage
migrate_container() {
    local vmid=$1
    local name=$2
    
    log_info "Migrating container $vmid ($name) from $SOURCE_NODE to $TARGET_NODE"
    log_info "  Using storage: $TARGET_STORAGE (LVM thin pool)"
    
    # Check source container status
    source_status=$(check_container "$vmid" "$SOURCE_NODE")
    if [[ "$source_status" == "not_found" ]]; then
        log_error "  Container $vmid not found on $SOURCE_NODE"
        return 1
    fi
    
    log_info "  Current status on $SOURCE_NODE: $source_status"
    
    # Perform migration with storage specification
    log_info "  Starting migration (this may take several minutes)..."
    
    if ssh_proxmox "pct migrate $vmid $TARGET_NODE --storage $TARGET_STORAGE --restart" 2>&1; then
        log_success "  Migration command completed"
        
        # Wait and verify
        log_info "  Waiting for migration to complete and verifying..."
        sleep 10
        
        # Check multiple times as migration might take time
        for i in {1..6}; do
            sleep 5
            target_status=$(check_container "$vmid" "$TARGET_NODE")
            if [[ "$target_status" != "not_found" ]]; then
                log_success "  Container $vmid is now on $TARGET_NODE (status: $target_status)"
                log_info "  Storage: $TARGET_STORAGE"
                return 0
            fi
            log_info "  Still migrating... (attempt $i/6)"
        done
        
        log_warn "  Migration may have succeeded but container not yet visible on target"
        log_info "  Please verify manually: ssh root@$PROXMOX_HOST 'pvesh get /nodes/$TARGET_NODE/lxc'"
        return 0
    else
        log_error "  Migration failed for container $vmid"
        return 1
    fi
}

# Main execution
main() {
    echo "========================================="
    log_header "MIGRATION" "2 Containers to pve2 using thin1 storage"
    echo "========================================="
    echo ""
    
    # Verify cluster status
    log_info "Verifying cluster status..."
    cluster_status=$(ssh_proxmox "pvecm status" 2>&1 | grep -c "Quorate:.*Yes" || echo "0")
    if [[ "$cluster_status" == "0" ]]; then
        log_error "Cluster is not quorate. Cannot proceed with migration."
        exit 1
    fi
    log_success "Cluster is quorate"
    echo ""
    
    # Verify target node is online
    log_info "Verifying target node ($TARGET_NODE) is online..."
    target_online=$(ssh_proxmox "pvesh get /nodes --output-format json" 2>&1 | python3 -c "import sys, json; nodes=json.load(sys.stdin); target=[n for n in nodes if n['node']=='$TARGET_NODE' and n['status']=='online']; print('online' if target else 'offline')" 2>/dev/null || echo "unknown")
    if [[ "$target_online" != "online" ]]; then
        log_error "Target node $TARGET_NODE is not online (status: $target_online)"
        exit 1
    fi
    log_success "Target node is online"
    
    # Verify target storage exists on pve2 directly (it's node-specific storage)
    log_info "Verifying target storage ($TARGET_STORAGE) exists on $TARGET_NODE..."
    # Check directly on pve2 since it's node-specific storage
    storage_check=$(sshpass -p 'password' ssh -o StrictHostKeyChecking=no root@${PROXMOX_HOST_R630_02:-192.168.11.12} "pvesm status 2>/dev/null | grep -i $TARGET_STORAGE" 2>&1 | head -1)
    if echo "$storage_check" | grep -qi "$TARGET_STORAGE.*active"; then
        log_success "Target storage $TARGET_STORAGE is available and active on $TARGET_NODE"
        echo "  $storage_check"
    else
        log_warn "Target storage $TARGET_STORAGE may be node-specific (not visible from ml110)"
        log_info "Checking storage.cfg shows thin1 is configured for pve2 node"
        log_info "Proceeding with migration - pct migrate will use storage on target node"
    fi
    echo ""
    
    # Select 2 containers to migrate (smaller/less critical ones first for testing)
    # Let's start with 2 sentry containers as they're less resource-intensive
    TEST_CONTAINERS=(
        "1500:besu-sentry-1"
        "1501:besu-sentry-2"
    )
    
    log_info "Will migrate 2 test containers to $TARGET_NODE using $TARGET_STORAGE storage:"
    for container in "${TEST_CONTAINERS[@]}"; do
        vmid="${container%%:*}"
        name="${container#*:}"
        echo "  - $vmid: $name"
    done
    echo ""
    
    log_warn "Starting migrations. Each migration may take 2-5 minutes."
    echo ""
    
    failed=0
    success=0
    total=${#TEST_CONTAINERS[@]}
    
    for container in "${TEST_CONTAINERS[@]}"; do
        vmid="${container%%:*}"
        name="${container#*:}"
        
        if migrate_container "$vmid" "$name"; then
            success=$((success + 1))
        else
            failed=$((failed + 1))
        fi
        
        # Small delay between migrations
        if [[ $success -lt $total ]]; then
            log_info "Waiting 15 seconds before next migration..."
            sleep 15
        fi
        echo ""
    done
    
    echo ""
    log_info "========================================="
    log_info "Migration Summary"
    log_info "========================================="
    log_success "Successful: $success/$total"
    if [[ $failed -gt 0 ]]; then
        log_warn "Failed: $failed/$total"
    fi
    echo ""
    
    if [[ $success -eq $total ]]; then
        log_success "All test migrations completed successfully!"
        log_info "You can now proceed with migrating more containers using thin1 storage."
    fi
    
    echo ""
    log_info "Verify containers on target node:"
    echo "  ssh root@$PROXMOX_HOST 'pvesh get /nodes/$TARGET_NODE/lxc'"
    echo ""
    log_info "Check container storage:"
    echo "  ssh root@$PROXMOX_HOST 'pvesh get /nodes/$TARGET_NODE/lxc/<VMID>/config' | grep rootfs"
    echo ""
}

main "$@"