Sankofa/scripts/diagnose-ceph-issues.sh

#!/bin/bash
# Comprehensive Ceph diagnostic script
# Run on R630-01 as root

set -e

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

echo "=========================================="
echo "Ceph Cluster Diagnostic"
echo "=========================================="
echo ""

# Check if running as root
if [ "$EUID" -ne 0 ]; then 
    echo -e "${RED}ERROR: Please run as root${NC}"
    exit 1
fi

echo -e "${BLUE}=== 1. Ceph Cluster Health ===${NC}"
echo "-----------------------------------"
if command -v ceph &> /dev/null; then
    echo "Ceph Health:"
    ceph health 2>&1 || echo -e "${RED}Cannot get health${NC}"
    echo ""
    echo "Ceph Health Detail:"
    ceph health detail 2>&1 | head -20 || echo -e "${RED}Cannot get health detail${NC}"
else
    echo -e "${RED}ceph command not found${NC}"
fi
echo ""

echo -e "${BLUE}=== 2. Ceph Monitor Status ===${NC}"
echo "-----------------------------------"
if command -v ceph &> /dev/null; then
    echo "Monitor Status:"
    ceph mon stat 2>&1 || echo -e "${RED}Cannot get monitor status${NC}"
    echo ""
    echo "Monitor Dump:"
    ceph mon dump 2>&1 | head -30 || echo -e "${RED}Cannot get monitor dump${NC}"
else
    echo -e "${RED}ceph command not found${NC}"
fi
echo ""

echo -e "${BLUE}=== 3. Ceph OSD Status ===${NC}"
echo "-----------------------------------"
if command -v ceph &> /dev/null; then
    echo "OSD Tree:"
    ceph osd tree 2>&1 || echo -e "${RED}Cannot get OSD tree${NC}"
    echo ""
    echo "OSD Details:"
    ceph osd df 2>&1 || echo -e "${RED}Cannot get OSD details${NC}"
else
    echo -e "${RED}ceph command not found${NC}"
fi
echo ""

echo -e "${BLUE}=== 4. Bootstrap Keyring Check ===${NC}"
echo "-----------------------------------"
echo "Standard location (/var/lib/ceph/bootstrap-osd/):"
if [ -d "/var/lib/ceph/bootstrap-osd" ]; then
    ls -la /var/lib/ceph/bootstrap-osd/ 2>/dev/null || echo "Directory exists but empty"
else
    echo -e "${YELLOW}Directory does not exist${NC}"
fi
echo ""

echo "Proxmox location (/etc/pve/priv/):"
if [ -d "/etc/pve/priv" ]; then
    ls -la /etc/pve/priv/ceph.client.bootstrap-osd.keyring 2>/dev/null || echo -e "${YELLOW}Bootstrap keyring not found${NC}"
    echo "All Ceph keyrings in /etc/pve/priv/:"
    ls -la /etc/pve/priv/ceph* 2>/dev/null || echo "No Ceph keyrings found"
else
    echo -e "${YELLOW}/etc/pve/priv does not exist${NC}"
fi
echo ""

echo "Bootstrap client in auth list:"
if command -v ceph &> /dev/null; then
    ceph auth list 2>&1 | grep -i bootstrap || echo -e "${YELLOW}No bootstrap client found${NC}"
else
    echo -e "${RED}ceph command not found${NC}"
fi
echo ""

echo -e "${BLUE}=== 5. Ceph Services Status ===${NC}"
echo "-----------------------------------"
echo "Ceph Target:"
systemctl status ceph.target --no-pager 2>&1 | head -10 || echo "Service not found"
echo ""

echo "Ceph Monitor Services:"
systemctl list-units --type=service | grep ceph-mon || echo "No monitor services found"
for mon in $(systemctl list-units --type=service | grep ceph-mon | awk '{print $1}'); do
    echo "  $mon:"
    systemctl status $mon --no-pager 2>&1 | head -5
done
echo ""

echo "Ceph OSD Services:"
systemctl list-units --type=service | grep ceph-osd | head -5 || echo "No OSD services found"
echo ""

echo -e "${BLUE}=== 6. Network Connectivity ===${NC}"
echo "-----------------------------------"
echo "Ceph Monitor Ports:"
netstat -tlnp 2>/dev/null | grep 6789 || ss -tlnp 2>/dev/null | grep 6789 || echo "No listeners on port 6789"
echo ""

echo "Monitor Addresses (from mon dump):"
if command -v ceph &> /dev/null; then
    MON_ADDRS=$(ceph mon dump 2>/dev/null | grep -E "addr|rank" | head -10)
    if [ ! -z "$MON_ADDRS" ]; then
        echo "$MON_ADDRS"
        echo ""
        echo "Testing connectivity to monitors:"
        echo "$MON_ADDRS" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+' | while read addr; do
            ip=$(echo $addr | cut -d: -f1)
            port=$(echo $addr | cut -d: -f2)
            echo -n "  Testing $ip:$port ... "
            if timeout 2 bash -c "echo > /dev/tcp/$ip/$port" 2>/dev/null; then
                echo -e "${GREEN}✓ Reachable${NC}"
            else
                echo -e "${RED}✗ Not reachable${NC}"
            fi
        done
    else
        echo -e "${YELLOW}Cannot get monitor addresses${NC}"
    fi
else
    echo -e "${RED}ceph command not found${NC}"
fi
echo ""

echo -e "${BLUE}=== 7. Proxmox Ceph Integration ===${NC}"
echo "-----------------------------------"
echo "pveceph command:"
which pveceph || echo -e "${YELLOW}pveceph not found${NC}"
echo ""

if command -v pveceph &> /dev/null; then
    echo "pveceph status:"
    pveceph status 2>&1 || echo "pveceph status failed"
fi
echo ""

echo -e "${BLUE}=== 8. Ceph Configuration ===${NC}"
echo "-----------------------------------"
echo "Ceph config file:"
if [ -f "/etc/ceph/ceph.conf" ]; then
    echo "  /etc/ceph/ceph.conf exists"
    cat /etc/ceph/ceph.conf 2>/dev/null | head -20
elif [ -f "/etc/pve/ceph.conf" ]; then
    echo "  /etc/pve/ceph.conf exists"
    cat /etc/pve/ceph.conf 2>/dev/null | head -20
else
    echo -e "${YELLOW}No Ceph config file found${NC}"
fi
echo ""

echo "Ceph cluster name:"
if command -v ceph &> /dev/null; then
    ceph --version 2>&1 || true
    # Try to get cluster name
    ceph config dump 2>&1 | grep "cluster" | head -5 || true
fi
echo ""

echo "=========================================="
echo -e "${GREEN}Diagnostic Complete${NC}"
echo "=========================================="
echo ""
echo "Summary of findings:"
echo "  - Review Ceph health status above"
echo "  - Check bootstrap keyring locations"
echo "  - Verify monitor connectivity"
echo "  - Check if pveceph is available"
echo ""