smom-dbis-138/terraform/phases/phase1/scripts/test-services.sh

#!/bin/bash
# Service Testing Script
# Tests services running on VMs (requires SSH access)

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PHASE1_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"

# Colors
GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

echo "=========================================="
echo "Service Testing"
echo "=========================================="
echo ""

cd "$PHASE1_DIR"

# Get Nginx proxy IP
NGINX_IP=$(terraform output -json 2>/dev/null | jq -r '.nginx_proxy.value.public_ip // empty' || echo "")

if [ -z "$NGINX_IP" ] || [ "$NGINX_IP" = "null" ]; then
    echo -e "${RED}✗${NC} Nginx proxy IP not found"
    exit 1
fi

echo -e "${BLUE}Testing Nginx Proxy Services ($NGINX_IP)${NC}"
echo ""

# Test SSH and services on Nginx proxy
if ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes besuadmin@$NGINX_IP "echo 'Connected'" > /dev/null 2>&1; then
    echo -e "${GREEN}✓${NC} SSH connection successful"
    
    # Test Docker
    DOCKER_VERSION=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes besuadmin@$NGINX_IP "docker --version 2>/dev/null" || echo "")
    if [ -n "$DOCKER_VERSION" ]; then
        echo -e "${GREEN}✓${NC} Docker installed: $DOCKER_VERSION"
    else
        echo -e "${YELLOW}⊘${NC} Docker not installed or not accessible"
    fi
    
    # Test Nginx
    NGINX_VERSION=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes besuadmin@$NGINX_IP "nginx -v 2>&1" || echo "")
    if [ -n "$NGINX_VERSION" ]; then
        echo -e "${GREEN}✓${NC} Nginx installed: $NGINX_VERSION"
        
        # Test Nginx status
        NGINX_STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes besuadmin@$NGINX_IP "systemctl is-active nginx 2>/dev/null || echo 'inactive'" || echo "unknown")
        if [ "$NGINX_STATUS" = "active" ]; then
            echo -e "${GREEN}✓${NC} Nginx service is running"
        else
            echo -e "${YELLOW}⊘${NC} Nginx service is $NGINX_STATUS"
        fi
    else
        echo -e "${YELLOW}⊘${NC} Nginx not installed"
    fi
    
    # Test Cloudflared
    CLOUDFLARED_VERSION=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes besuadmin@$NGINX_IP "cloudflared --version 2>/dev/null | head -1" || echo "")
    if [ -n "$CLOUDFLARED_VERSION" ]; then
        echo -e "${GREEN}✓${NC} Cloudflared installed: $CLOUDFLARED_VERSION"
        
        # Test Cloudflared service
        CLOUDFLARED_STATUS=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes besuadmin@$NGINX_IP "systemctl is-active cloudflared 2>/dev/null || echo 'inactive'" || echo "unknown")
        if [ "$CLOUDFLARED_STATUS" = "active" ]; then
            echo -e "${GREEN}✓${NC} Cloudflared service is running"
        else
            echo -e "${YELLOW}⊘${NC} Cloudflared service is $CLOUDFLARED_STATUS (not configured yet)"
        fi
    else
        echo -e "${YELLOW}⊘${NC} Cloudflared not installed"
    fi
    
    # Test system resources
    echo ""
    echo "System Resources:"
    UPTIME=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes besuadmin@$NGINX_IP "uptime" || echo "")
    if [ -n "$UPTIME" ]; then
        echo -e "  ${BLUE}ℹ${NC} $UPTIME"
    fi
    
    MEMORY=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes besuadmin@$NGINX_IP "free -h | grep Mem | awk '{print \$3\"/\"\$2}'" || echo "")
    if [ -n "$MEMORY" ]; then
        echo -e "  ${BLUE}ℹ${NC} Memory: $MEMORY"
    fi
    
    DISK=$(ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no -o BatchMode=yes besuadmin@$NGINX_IP "df -h / | tail -1 | awk '{print \$3\"/\"\$2\" (\"\$5\" used)\"}'" || echo "")
    if [ -n "$DISK" ]; then
        echo -e "  ${BLUE}ℹ${NC} Disk: $DISK"
    fi
else
    echo -e "${RED}✗${NC} SSH connection failed"
    echo -e "${YELLOW}⊘${NC} Cannot test services without SSH access"
fi

# Test Backend VMs (if accessible via VPN/Bastion)
echo ""
echo "=========================================="
echo "Backend VM Services"
echo "=========================================="
echo -e "${YELLOW}⊘${NC} Backend VMs use private IPs - testing requires VPN/Bastion access"
echo ""
echo "To test backend VMs, SSH via VPN/Bastion and run:"
echo "  - docker ps (check Besu containers)"
echo "  - systemctl status besu.service"
echo "  - curl http://localhost:8545 (test RPC)"
echo "  - curl http://localhost:9545/metrics (test metrics)"

echo ""
echo "=========================================="
echo "Service Test Complete"
echo "=========================================="