Sankofa/scripts/fix-ceph-bootstrap-and-create-osds.sh

#!/bin/bash
# Fix Ceph bootstrap keyring issue and create OSDs
# Run on R630-01 as root

set -e

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")

echo "=========================================="
echo "Fix Ceph Bootstrap and Create OSDs"
echo "=========================================="
echo ""

# Check if running as root
if [ "$EUID" -ne 0 ]; then 
    echo -e "${RED}ERROR: Please run as root${NC}"
    exit 1
fi

echo -e "${BLUE}=== Step 1: Check Ceph Bootstrap Keyring ===${NC}"
echo "-----------------------------------"

# Check for bootstrap keyring in standard locations
BOOTSTRAP_KEYRING=""
if [ -f "/var/lib/ceph/bootstrap-osd/ceph.keyring" ]; then
    BOOTSTRAP_KEYRING="/var/lib/ceph/bootstrap-osd/ceph.keyring"
    echo -e "${GREEN}  Found: $BOOTSTRAP_KEYRING${NC}"
elif [ -f "/etc/pve/priv/ceph.client.bootstrap-osd.keyring" ]; then
    BOOTSTRAP_KEYRING="/etc/pve/priv/ceph.client.bootstrap-osd.keyring"
    echo -e "${GREEN}  Found: $BOOTSTRAP_KEYRING${NC}"
else
    echo -e "${YELLOW}  Bootstrap keyring not found in standard locations${NC}"
    echo "  Searching for keyring files..."
    find /var/lib/ceph /etc/pve -name "*bootstrap*keyring" 2>/dev/null | head -5
fi
echo ""

echo -e "${BLUE}=== Step 2: Check Ceph Cluster Status ===${NC}"
echo "-----------------------------------"

# Check if Ceph is accessible
if command -v ceph &> /dev/null; then
    echo "Testing Ceph connectivity..."
    if ceph health &>/dev/null; then
        echo -e "${GREEN}  ✓ Ceph cluster is accessible${NC}"
        ceph health
    else
        echo -e "${RED}  ✗ Ceph cluster not accessible${NC}"
        echo "  Checking Ceph services..."
        systemctl status ceph.target 2>/dev/null || echo "  Ceph services not running"
    fi
else
    echo -e "${YELLOW}  ceph command not found${NC}"
fi
echo ""

echo -e "${BLUE}=== Step 3: Check Ceph Monitors ===${NC}"
echo "-----------------------------------"
if command -v ceph &> /dev/null; then
    ceph mon stat 2>/dev/null || echo -e "${YELLOW}  Cannot get monitor status${NC}"
    ceph mon dump 2>/dev/null || echo -e "${YELLOW}  Cannot get monitor dump${NC}"
fi
echo ""

echo -e "${BLUE}=== Step 4: Try Creating OSD with Different Method ===${NC}"
echo "-----------------------------------"

# Try using pveceph if available (Proxmox Ceph management)
if command -v pveceph &> /dev/null; then
    echo "Using pveceph to create OSDs..."
    for drive in "${DRIVES[@]}"; do
        if [ ! -b "/dev/$drive" ]; then
            continue
        fi
        echo -e "${BLUE}  Creating OSD on /dev/$drive using pveceph...${NC}"
        pveceph create /dev/$drive 2>&1 || echo -e "${YELLOW}    pveceph method failed, will try ceph-volume${NC}"
    done
else
    echo -e "${YELLOW}  pveceph not available${NC}"
fi
echo ""

echo -e "${BLUE}=== Step 5: Create OSDs with ceph-volume (with fixes) ===${NC}"
echo "-----------------------------------"

# Try to fix bootstrap keyring issue
if [ -z "$BOOTSTRAP_KEYRING" ] && [ -d "/var/lib/ceph/bootstrap-osd" ]; then
    echo "Checking if we need to create bootstrap keyring..."
    # Check if we can get it from the cluster
    if command -v ceph &> /dev/null && ceph auth get client.bootstrap-osd &>/dev/null; then
        echo "  Getting bootstrap keyring from cluster..."
        mkdir -p /var/lib/ceph/bootstrap-osd
        ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring 2>/dev/null || true
    fi
fi

# Try creating OSDs
osd_count=0
for drive in "${DRIVES[@]}"; do
    if [ ! -b "/dev/$drive" ]; then
        continue
    fi
    
    echo -e "${BLUE}  Creating OSD on /dev/$drive...${NC}"
    
    # Try with explicit keyring path
    if [ ! -z "$BOOTSTRAP_KEYRING" ]; then
        export CEPH_ARGS="--keyring $BOOTSTRAP_KEYRING"
    fi
    
    # Try creating OSD
    if ceph-volume lvm create --data "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
        echo -e "${GREEN}    ✓ OSD created on /dev/$drive${NC}"
        osd_count=$((osd_count + 1))
    else
        echo -e "${YELLOW}    ⚠ OSD creation had issues, checking status...${NC}"
        # Check if OSD was partially created
        if ceph osd tree 2>/dev/null | grep -q "osd\."; then
            echo "    Checking if OSD exists..."
            ceph osd tree 2>/dev/null | tail -5
        fi
    fi
    echo ""
done

echo "=========================================="
echo -e "${GREEN}Process Complete!${NC}"
echo "=========================================="
echo ""
echo "Created $osd_count OSD(s) out of ${#DRIVES[@]} drives"
echo ""

echo "Current OSD status:"
ceph osd tree 2>/dev/null || echo "Cannot get OSD tree"
echo ""

echo "Ceph health:"
ceph health 2>/dev/null || echo "Cannot get health"
echo ""