proxmox/scripts/diagnose-npmplus-backend-services.sh

#!/usr/bin/env bash
# Diagnose backend services for NPMplus
set -euo pipefail

# Load IP configuration
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true


RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; }
log_error() { echo -e "${RED}[✗]${NC} $1"; }

PROXMOX_HOST="${1:-192.168.11.11}"
CONTAINER_ID="${2:-10233}"

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "🔍 NPMplus Backend Services Diagnosis"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Backend service mapping
declare -A BACKEND_SERVICES=(
    ["${IP_BLOCKSCOUT:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-${IP_DEVICE_14:-192.168.11.14}}}}}0}:80"]="VMID 5000 (blockscout-1)"
    ["${IP_DBIS_FRONTEND:-${IP_SERVICE_13:-${IP_SERVICE_13:-${IP_SERVICE_13:-${IP_SERVICE_13:-${IP_SERVICE_13:-192.168.11.13}}}}}0}:80"]="VMID 10130 (dbis-frontend)"
    ["${IP_DBIS_API:-${IP_DBIS_API:-192.168.11.155}}:3000"]="VMID 10150 (dbis-api-primary)"
    ["${IP_DBIS_API_2:-${IP_DBIS_API_2:-192.168.11.156}}:3000"]="VMID 10151 (dbis-api-secondary)"
    ["${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-${IP_SERVICE_36:-192.168.11.36}}}}}}:80"]="VMID 7811 (mim-api-1)"
    ["${RPC_CORE_1:-${IP_SERVICE_21:-${IP_SERVICE_21:-${IP_SERVICE_21:-${IP_SERVICE_21:-${IP_SERVICE_21:-192.168.11.21}}}}}1}:443"]="VMID 2101 (besu-rpc-core-1)"
    ["${RPC_PUBLIC_1:-${RPC_PUBLIC_1:-192.168.11.221}}:443"]="VMID 2201 (besu-rpc-public-1)"
    ["${RPC_PRIVATE_1:-${RPC_PRIVATE_1:-192.168.11.232}}:443"]="VMID 2301 (besu-rpc-private-1)"
    # Note: VMID 2302 (besu-rpc-private-2) - not in latest mapping, may need different IP or is new service
)

# Check 1: Test from local machine
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Check 1: Testing Backend Services (from local)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

working_count=0
failed_count=0

for backend in "${!BACKEND_SERVICES[@]}"; do
    service_info="${BACKEND_SERVICES[$backend]}"
    ip_port="${backend%%:*}"
    port="${backend##*:}"
    
    log_info "Testing: $service_info ($backend)"
    
    if [ "$port" = "443" ]; then
        response=$(curl -s -o /dev/null -w "%{http_code}" -I -k --connect-timeout 5 "https://$backend" 2>/dev/null || echo "000")
    else
        response=$(curl -s -o /dev/null -w "%{http_code}" -I --connect-timeout 5 "http://$backend" 2>/dev/null || echo "000")
    fi
    
    if [ "$response" != "000" ] && [ "$response" != "" ]; then
        log_success "  ✓ Responding (HTTP $response)"
        working_count=$((working_count + 1))
    else
        log_error "  ✗ Not responding"
        failed_count=$((failed_count + 1))
    fi
    echo ""
done

# Check 2: Test from NPMplus container
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Check 2: Testing from NPMplus Container"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

npm_working=0
npm_failed=0

for backend in "${!BACKEND_SERVICES[@]}"; do
    service_info="${BACKEND_SERVICES[$backend]}"
    port="${backend##*:}"
    
    log_info "Testing: $service_info"
    
    if [ "$port" = "443" ]; then
        response=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- curl -s -o /dev/null -w '%{http_code}' -I -k --connect-timeout 5 'https://$backend' 2>/dev/null || echo '000'")
    else
        response=$(ssh root@"$PROXMOX_HOST" "pct exec $CONTAINER_ID -- curl -s -o /dev/null -w '%{http_code}' -I --connect-timeout 5 'http://$backend' 2>/dev/null || echo '000'")
    fi
    
    if [ "$response" != "000" ] && [ "$response" != "" ]; then
        log_success "  ✓ Accessible (HTTP $response)"
        npm_working=$((npm_working + 1))
    else
        log_error "  ✗ Not accessible"
        npm_failed=$((npm_failed + 1))
    fi
    echo ""
done

# Check 3: VMID status
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Check 3: VMID Status"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

declare -A VMID_HOSTS=(
    ["5000"]="${PROXMOX_HOST_R630_01}"
    ["10130"]="${PROXMOX_HOST_ML110}"
    ["10150"]="${PROXMOX_HOST_ML110}"
    ["10151"]="${PROXMOX_HOST_ML110}"
    ["7811"]="${PROXMOX_HOST_R630_01}"
    ["2501"]="${PROXMOX_HOST_ML110}"
    ["2502"]="${PROXMOX_HOST_ML110}"
)

stopped_list=()

for vmid in "${!VMID_HOSTS[@]}"; do
    host="${VMID_HOSTS[$vmid]}"
    status=$(ssh root@"$host" "pct status $vmid 2>/dev/null || qm status $vmid 2>/dev/null || echo 'not found'")
    
    if echo "$status" | grep -q "running"; then
        log_success "VMID $vmid on $host: Running"
    elif echo "$status" | grep -q "stopped"; then
        log_warn "VMID $vmid on $host: Stopped"
        stopped_list+=("$vmid|$host")
    else
        log_error "VMID $vmid on $host: $status"
        stopped_list+=("$vmid|$host")
    fi
done

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
log_info "Summary:"
echo "  Local Tests: $working_count working, $failed_count failed"
echo "  NPMplus Tests: $npm_working working, $npm_failed failed"
echo "  Stopped VMIDs: ${#stopped_list[@]}"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

if [ ${#stopped_list[@]} -gt 0 ]; then
    log_warn "Stopped VMIDs found:"
    for entry in "${stopped_list[@]}"; do
        IFS='|' read -r vmid host <<< "$entry"
        log_info "  VMID $vmid on $host"
    done
    echo ""
    log_info "To start stopped services:"
    log_info "  bash scripts/fix-npmplus-backend-services.sh $PROXMOX_HOST $CONTAINER_ID true"
    echo ""
fi