proxmox/scripts/archive/consolidated/verify/check-container-2101.sh

#!/usr/bin/env bash
# Check Container 2101 (besu-rpc-core-1) Deployment Readiness
# Verifies container is running, service is healthy, and ready for config deployment

set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

# Configuration
PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"
SSH_KEY="${SSH_KEY:-~/.ssh/id_ed25519_proxmox}"
VMID=2101
CONTAINER_NAME="besu-rpc-core-1"

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

echo -e "${BLUE}╔══════════════════════════════════════════════════════════════╗${NC}"
echo -e "${BLUE}║  Container 2101 Deployment Readiness Check                  ║${NC}"
echo -e "${BLUE}╚══════════════════════════════════════════════════════════════╝${NC}"
echo ""

CHECKS_PASSED=0
CHECKS_FAILED=0
CHECKS_WARN=0

# Check SSH access
log_info "Checking SSH access to Proxmox host ($PROXMOX_HOST)..."
if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "echo 'SSH OK'" &>/dev/null 2>&1; then
    log_success "SSH access confirmed"
    ((CHECKS_PASSED++))
    SSH_AVAILABLE=1
else
    log_error "Cannot access Proxmox host via SSH"
    log_error "  Host: $PROXMOX_HOST"
    log_error "  SSH Key: ${SSH_KEY}"
    log_error "  Run this script from a machine with SSH access to Proxmox host"
    ((CHECKS_FAILED++))
    SSH_AVAILABLE=0
    exit 1
fi

echo ""

# Check container status
log_info "Checking container status (VMID $VMID)..."
CONTAINER_STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "pct status $VMID 2>&1" || echo "not found")

if echo "$CONTAINER_STATUS" | grep -q "running"; then
    log_success "Container $VMID is running"
    ((CHECKS_PASSED++))
    CONTAINER_RUNNING=1
elif echo "$CONTAINER_STATUS" | grep -q "stopped"; then
    log_warn "Container $VMID is stopped"
    ((CHECKS_WARN++))
    CONTAINER_RUNNING=0
else
    log_error "Container $VMID not found or status unknown"
    log_error "  Status: $CONTAINER_STATUS"
    ((CHECKS_FAILED++))
    CONTAINER_RUNNING=0
    exit 1
fi

echo ""

# If container is running, check service
if [ "$CONTAINER_RUNNING" -eq 1 ]; then
    # Check Besu service status
    log_info "Checking Besu RPC service status..."
    SERVICE_STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "pct exec $VMID -- systemctl is-active besu-rpc.service 2>&1" || echo "unknown")
    
    if [ "$SERVICE_STATUS" = "active" ]; then
        log_success "Besu RPC service is active"
        ((CHECKS_PASSED++))
        SERVICE_ACTIVE=1
    else
        log_warn "Besu RPC service is not active (status: $SERVICE_STATUS)"
        ((CHECKS_WARN++))
        SERVICE_ACTIVE=0
    fi
    
    echo ""
    
    # Check service enabled
    log_info "Checking if service is enabled..."
    SERVICE_ENABLED=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "pct exec $VMID -- systemctl is-enabled besu-rpc.service 2>&1" || echo "unknown")
    
    if [ "$SERVICE_ENABLED" = "enabled" ]; then
        log_success "Service is enabled (will start on boot)"
        ((CHECKS_PASSED++))
    else
        log_warn "Service is not enabled (status: $SERVICE_ENABLED)"
        ((CHECKS_WARN++))
    fi
    
    echo ""
    
    # Check current config file
    log_info "Checking current configuration file..."
    CONFIG_FILE=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "pct exec $VMID -- grep -E 'config-file=|--config-file=' /etc/systemd/system/besu-rpc.service 2>/dev/null | grep -oE 'config-[^[:space:]]+' | head -1" || echo "")
    
    if [ -n "$CONFIG_FILE" ]; then
        log_info "  Current config: $CONFIG_FILE.toml"
        EXPECTED_CONFIG="config-rpc-core.toml"
        if [ "$CONFIG_FILE" = "config-rpc-core" ]; then
            log_success "  Config file matches expected: $EXPECTED_CONFIG"
            ((CHECKS_PASSED++))
        else
            log_warn "  Config file differs from expected ($EXPECTED_CONFIG)"
            ((CHECKS_WARN++))
        fi
        
        # Check if config file exists on container
        CONFIG_EXISTS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "pct exec $VMID -- test -f /etc/besu/$EXPECTED_CONFIG && echo 'exists' || echo 'missing'")
        if [ "$CONFIG_EXISTS" = "exists" ]; then
            log_success "  Config file exists on container: /etc/besu/$EXPECTED_CONFIG"
            ((CHECKS_PASSED++))
        else
            log_warn "  Config file not found: /etc/besu/$EXPECTED_CONFIG"
            ((CHECKS_WARN++))
        fi
    else
        log_warn "Could not determine config file from service"
        ((CHECKS_WARN++))
    fi
    
    echo ""
    
    # Check for recent errors in logs
    if [ "$SERVICE_ACTIVE" -eq 1 ]; then
        log_info "Checking for recent errors in service logs..."
        RECENT_ERRORS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "pct exec $VMID -- journalctl -u besu-rpc.service --since '10 minutes ago' --no-pager 2>&1 | grep -i 'error\|failed\|exception' | head -5" || echo "")
        
        if [ -z "$RECENT_ERRORS" ]; then
            log_success "No recent errors in service logs"
            ((CHECKS_PASSED++))
        else
            log_warn "Recent errors found in logs:"
            echo "$RECENT_ERRORS" | while IFS= read -r line; do
                echo "    $line"
            done
            ((CHECKS_WARN++))
        fi
        
        echo ""
        
        # Check RPC endpoint (if accessible)
        log_info "Checking RPC endpoint availability..."
        CONTAINER_IP=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -i "${SSH_KEY/#\~/$HOME}" "root@${PROXMOX_HOST}" "pct exec $VMID -- hostname -I | awk '{print \$1}'" || echo "")
        
        if [ -n "$CONTAINER_IP" ]; then
            log_info "  Container IP: $CONTAINER_IP"
            RPC_RESPONSE=$(curl -s -m 5 -X POST "http://${CONTAINER_IP}:8545" \
                -H "Content-Type: application/json" \
                -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' 2>&1 || echo "unreachable")
            
            if echo "$RPC_RESPONSE" | grep -q "result"; then
                log_success "RPC endpoint responding (eth_blockNumber successful)"
                ((CHECKS_PASSED++))
            else
                log_warn "RPC endpoint not responding or unreachable"
                log_warn "  Response: $(echo "$RPC_RESPONSE" | head -1)"
                ((CHECKS_WARN++))
            fi
        else
            log_warn "Could not determine container IP"
            ((CHECKS_WARN++))
        fi
    fi
fi

echo ""

# Summary
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
echo -e "${BLUE}Summary${NC}"
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
echo ""

TOTAL_CHECKS=$((CHECKS_PASSED + CHECKS_FAILED + CHECKS_WARN))

echo "Checks passed:  $CHECKS_PASSED"
echo "Checks failed:  $CHECKS_FAILED"
echo "Warnings:       $CHECKS_WARN"
echo "Total checks:   $TOTAL_CHECKS"
echo ""

if [ "$CHECKS_FAILED" -eq 0 ]; then
    if [ "$CONTAINER_RUNNING" -eq 1 ] && [ "$SERVICE_ACTIVE" -eq 1 ]; then
        log_success "✅ Container 2101 is fully running and ready for deployment!"
        echo ""
        echo "Container Status:"
        echo "  VMID:           $VMID"
        echo "  Name:           $CONTAINER_NAME"
        echo "  Status:         Running"
        echo "  Service:        Active"
        echo "  Config:         config-rpc-core.toml"
        echo ""
        echo "Ready to deploy cleaned configuration."
        exit 0
    else
        log_warn "⚠️  Container is running but service may need attention"
        exit 0
    fi
else
    log_error "❌ Container not ready for deployment"
    echo ""
    echo "Issues found:"
    echo "  - Fix failed checks before deploying"
    exit 1
fi