Sankofa/scripts/fix-ceph-services.sh

#!/bin/bash
# Fix failed Ceph services and retry OSD creation
# Run on R630-01 as root

set -e

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")

echo "=========================================="
echo "Fix Ceph Services and Create OSDs"
echo "=========================================="
echo ""

# Check if running as root
if [ "$EUID" -ne 0 ]; then 
    echo -e "${RED}ERROR: Please run as root${NC}"
    exit 1
fi

echo -e "${BLUE}=== Step 1: Check Failed Services ===${NC}"
echo "-----------------------------------"

echo "Monitor service status:"
systemctl status ceph-mon@r630-01.service --no-pager | head -15 || true
echo ""

echo "OSD service status:"
systemctl status ceph-osd@1.service --no-pager | head -15 || true
echo ""

echo -e "${BLUE}=== Step 2: Check Service Logs ===${NC}"
echo "-----------------------------------"

echo "Monitor service logs (last 20 lines):"
journalctl -u ceph-mon@r630-01.service -n 20 --no-pager | tail -10 || true
echo ""

echo "OSD service logs (last 20 lines):"
journalctl -u ceph-osd@1.service -n 20 --no-pager | tail -10 || true
echo ""

echo -e "${BLUE}=== Step 3: Attempt to Start Services ===${NC}"
echo "-----------------------------------"

echo "Starting monitor service..."
if systemctl start ceph-mon@r630-01.service 2>&1; then
    sleep 3
    if systemctl is-active --quiet ceph-mon@r630-01.service; then
        echo -e "${GREEN}  ✓ Monitor service started${NC}"
    else
        echo -e "${YELLOW}  ⚠ Monitor service started but not active${NC}"
    fi
else
    echo -e "${RED}  ✗ Failed to start monitor service${NC}"
    echo "  Check logs above for errors"
fi
echo ""

echo "Starting OSD service..."
if systemctl start ceph-osd@1.service 2>&1; then
    sleep 3
    if systemctl is-active --quiet ceph-osd@1.service; then
        echo -e "${GREEN}  ✓ OSD service started${NC}"
    else
        echo -e "${YELLOW}  ⚠ OSD service started but not active${NC}"
    fi
else
    echo -e "${RED}  ✗ Failed to start OSD service${NC}"
    echo "  Check logs above for errors"
fi
echo ""

echo -e "${BLUE}=== Step 4: Test Cluster Connectivity ===${NC}"
echo "-----------------------------------"

echo "Testing cluster connectivity (5 second timeout)..."
if timeout 5 ceph health 2>&1; then
    echo -e "${GREEN}  ✓ Cluster is accessible${NC}"
    echo ""
    echo "Ceph health:"
    ceph health
    echo ""
    echo "OSD tree:"
    ceph osd tree
else
    echo -e "${YELLOW}  ⚠ Cluster still not accessible${NC}"
    echo "  Services may need more time or have other issues"
fi
echo ""

echo -e "${BLUE}=== Step 5: Create OSDs (if cluster accessible) ===${NC}"
echo "-----------------------------------"

# Test if cluster is accessible
if timeout 5 ceph health &>/dev/null; then
    echo "Cluster is accessible. Creating OSDs..."
    echo ""
    
    osd_count=0
    for drive in "${DRIVES[@]}"; do
        if [ ! -b "/dev/$drive" ]; then
            continue
        fi
        
        echo -e "${BLUE}  Creating OSD on /dev/$drive...${NC}"
        
        # Try pveceph first (Proxmox tool)
        if command -v pveceph &> /dev/null; then
            if pveceph create "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
                echo -e "${GREEN}    ✓ OSD created on /dev/$drive (using pveceph)${NC}"
                osd_count=$((osd_count + 1))
                continue
            fi
        fi
        
        # Fallback to ceph-volume
        if ceph-volume lvm create --data "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
            echo -e "${GREEN}    ✓ OSD created on /dev/$drive${NC}"
            osd_count=$((osd_count + 1))
        else
            echo -e "${YELLOW}    ⚠ OSD creation had issues${NC}"
        fi
        echo ""
    done
    
    echo "Created $osd_count OSD(s) out of ${#DRIVES[@]} drives"
    echo ""
    
    echo "Verifying OSDs:"
    ceph osd tree
    echo ""
    ceph health
else
    echo -e "${YELLOW}Cluster not accessible. Cannot create OSDs yet.${NC}"
    echo ""
    echo "Next steps:"
    echo "1. Fix service issues (check logs above)"
    echo "2. Ensure services are running"
    echo "3. Test cluster connectivity"
    echo "4. Retry OSD creation"
fi
echo ""

echo "=========================================="
echo -e "${GREEN}Process Complete${NC}"
echo "=========================================="
echo ""