Sankofa/scripts/comprehensive-status-check.sh

#!/bin/bash
# Comprehensive Status Check
# Verifies all systems, checks for issues, and reports status

set -e

GREEN='\033[0;32m'
RED='\033[0;31m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

PASSED=0
FAILED=0
WARNINGS=0

echo "=========================================="
echo "Comprehensive Status Check"
echo "=========================================="
echo ""

# 1. Provider Status
echo -e "${BLUE}1. Provider Status${NC}"
POD_STATUS=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "NOT_FOUND")
if [[ "$POD_STATUS" == "Running" ]]; then
    READY=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].ready}' 2>/dev/null)
    if [[ "$READY" == "true" ]]; then
        echo -e "${GREEN}✓${NC} Provider pod running and ready"
        ((PASSED++))
    else
        echo -e "${YELLOW}⚠${NC} Provider pod running but not ready"
        ((WARNINGS++))
    fi
else
    echo -e "${RED}✗${NC} Provider pod not running (status: $POD_STATUS)"
    ((FAILED++))
fi
echo ""

# 2. Provider Logs - Critical Errors
echo -e "${BLUE}2. Provider Logs - Critical Errors${NC}"
CRITICAL_ERRORS=$(kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=100 2>&1 | grep -i -E "error.*timeout|error.*authentication|error.*connection" | grep -v "CRD" | wc -l)
if [[ $CRITICAL_ERRORS -eq 0 ]]; then
    echo -e "${GREEN}✓${NC} No critical errors in logs"
    ((PASSED++))
else
    echo -e "${YELLOW}⚠${NC} $CRITICAL_ERRORS critical errors found (may be transient)"
    ((WARNINGS++))
fi
echo ""

# 3. VM Status
echo -e "${BLUE}3. VM Status${NC}"
TOTAL_VMS=$(kubectl get proxmoxvm -A --no-headers 2>&1 | wc -l)
FAILED_VMS=$(kubectl get proxmoxvm -A -o json 2>&1 | jq -r '.items[] | select(.status.conditions[]?.type=="Failed" or .status.conditions[]?.type=="ValidationFailed") | .metadata.name' 2>/dev/null | wc -l)
VMS_WITH_ID=$(kubectl get proxmoxvm -A -o jsonpath='{range .items[*]}{.status.vmId}{"\n"}{end}' 2>&1 | grep -v "^$" | wc -l)

echo "  Total VMs: $TOTAL_VMS"
echo "  VMs with VMID: $VMS_WITH_ID"
echo "  Failed VMs: $FAILED_VMS"

if [[ $FAILED_VMS -eq 0 ]]; then
    echo -e "${GREEN}✓${NC} No failed VMs"
    ((PASSED++))
else
    echo -e "${RED}✗${NC} $FAILED_VMS VMs have failed"
    ((FAILED++))
fi

if [[ $VMS_WITH_ID -gt 0 ]]; then
    echo -e "${GREEN}✓${NC} VMs are being created on Proxmox"
    ((PASSED++))
else
    echo -e "${YELLOW}⚠${NC} VMs pending creation (provider processing)"
    ((WARNINGS++))
fi
echo ""

# 4. Connectivity
echo -e "${BLUE}4. Network Connectivity${NC}"
if ping -c 2 192.168.11.10 &>/dev/null; then
    echo -e "${GREEN}✓${NC} ML110-01 reachable"
    ((PASSED++))
else
    echo -e "${RED}✗${NC} ML110-01 not reachable"
    ((FAILED++))
fi

if ping -c 2 192.168.11.11 &>/dev/null; then
    echo -e "${GREEN}✓${NC} R630-01 reachable"
    ((PASSED++))
else
    echo -e "${RED}✗${NC} R630-01 not reachable"
    ((FAILED++))
fi
echo ""

# 5. Credentials
echo -e "${BLUE}5. Credentials${NC}"
if kubectl get secret proxmox-credentials -n crossplane-system &>/dev/null; then
    KEYS=$(kubectl get secret proxmox-credentials -n crossplane-system -o jsonpath='{.data}' 2>&1 | jq -r 'keys[]' 2>/dev/null || echo "")
    if [[ "$KEYS" == *"token"* && "$KEYS" == *"tokenid"* ]]; then
        echo -e "${GREEN}✓${NC} Credentials secret exists with token format"
        ((PASSED++))
    else
        echo -e "${YELLOW}⚠${NC} Credentials format may be incorrect"
        ((WARNINGS++))
    fi
else
    echo -e "${RED}✗${NC} Credentials secret not found"
    ((FAILED++))
fi
echo ""

# 6. Provider Config
echo -e "${BLUE}6. Provider Configuration${NC}"
if kubectl get providerconfig proxmox-provider-config -n crossplane-system &>/dev/null; then
    SITES=$(kubectl get providerconfig proxmox-provider-config -n crossplane-system -o jsonpath='{.spec.sites[*].name}' 2>/dev/null)
    if [[ "$SITES" == *"site-1"* && "$SITES" == *"site-2"* ]]; then
        echo -e "${GREEN}✓${NC} ProviderConfig exists with both sites"
        ((PASSED++))
    else
        echo -e "${RED}✗${NC} Sites not correctly configured: $SITES"
        ((FAILED++))
    fi
else
    echo -e "${RED}✗${NC} ProviderConfig not found"
    ((FAILED++))
fi
echo ""

# 7. YAML Syntax
echo -e "${BLUE}7. YAML Syntax Validation${NC}"
YAML_ERRORS=0
for file in examples/production/phoenix/*.yaml; do
    if ! kubectl apply --dry-run=client -f "$file" &>/dev/null; then
        ((YAML_ERRORS++))
    fi
done

if [[ $YAML_ERRORS -eq 0 ]]; then
    echo -e "${GREEN}✓${NC} All YAML files valid"
    ((PASSED++))
else
    echo -e "${RED}✗${NC} $YAML_ERRORS YAML files have syntax errors"
    ((FAILED++))
fi
echo ""

# 8. CRD Warnings
echo -e "${BLUE}8. CRD Warnings (Non-Critical)${NC}"
CRD_WARNINGS=$(kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 2>&1 | grep -c "CRD.*should be installed" || echo "0")
if [[ $CRD_WARNINGS -gt 0 ]]; then
    echo -e "${YELLOW}⚠${NC} $CRD_WARNINGS CRD warnings (non-critical, log noise only)"
    ((WARNINGS++))
else
    echo -e "${GREEN}✓${NC} No CRD warnings"
    ((PASSED++))
fi
echo ""

# Summary
echo "=========================================="
echo "Summary"
echo "=========================================="
echo -e "${GREEN}Passed:${NC} $PASSED"
echo -e "${YELLOW}Warnings:${NC} $WARNINGS"
echo -e "${RED}Failed:${NC} $FAILED"
echo ""

if [[ $FAILED -eq 0 && $WARNINGS -eq 0 ]]; then
    echo -e "${GREEN}✓ All checks passed!${NC}"
    exit 0
elif [[ $FAILED -eq 0 ]]; then
    echo -e "${YELLOW}⚠ All critical checks passed with minor warnings${NC}"
    exit 0
else
    echo -e "${RED}✗ Critical issues found${NC}"
    exit 1
fi