Sankofa/scripts/fix-and-create-all-osds.sh

#!/bin/bash
# Complete script to fix bootstrap keyring and create OSDs on all 6 drives
# Run on R630-01 as root

set -e

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")

echo "=========================================="
echo "Fix Bootstrap Keyring and Create All OSDs"
echo "=========================================="
echo ""

# Check if running as root
if [ "$EUID" -ne 0 ]; then 
    echo -e "${RED}ERROR: Please run as root${NC}"
    exit 1
fi

echo -e "${BLUE}=== Step 1: Check Ceph Cluster Connectivity ===${NC}"
echo "-----------------------------------"

if ! command -v ceph &> /dev/null; then
    echo -e "${RED}Ceph command not found. Please install ceph-common.${NC}"
    exit 1
fi

echo "Testing Ceph connectivity..."
if ceph health &>/dev/null; then
    echo -e "${GREEN}  ✓ Ceph cluster is accessible${NC}"
    ceph health
else
    echo -e "${YELLOW}  ⚠ Ceph cluster may not be fully accessible${NC}"
    echo "  Attempting to continue anyway..."
fi
echo ""

echo -e "${BLUE}=== Step 2: Fix Bootstrap Keyring ===${NC}"
echo "-----------------------------------"

BOOTSTRAP_DIR="/var/lib/ceph/bootstrap-osd"
BOOTSTRAP_KEYRING="$BOOTSTRAP_DIR/ceph.keyring"

# Create directory
mkdir -p "$BOOTSTRAP_DIR"

# Try to get bootstrap keyring from cluster
echo "Attempting to get bootstrap keyring from cluster..."
if ceph auth get client.bootstrap-osd -o "$BOOTSTRAP_KEYRING" 2>/dev/null; then
    echo -e "${GREEN}  ✓ Retrieved bootstrap keyring from cluster${NC}"
elif ceph auth list 2>/dev/null | grep -q "client.bootstrap-osd"; then
    echo "  Bootstrap keyring exists in cluster, exporting..."
    ceph auth export client.bootstrap-osd -o "$BOOTSTRAP_KEYRING" 2>/dev/null || \
    ceph auth get client.bootstrap-osd -o "$BOOTSTRAP_KEYRING" 2>/dev/null || true
else
    echo -e "${YELLOW}  ⚠ Bootstrap keyring not found in cluster${NC}"
    echo "  Attempting to create it..."
    ceph auth add client.bootstrap-osd mon 'allow profile bootstrap-osd' -i "$BOOTSTRAP_KEYRING" 2>/dev/null || \
    echo -e "${YELLOW}    Could not create bootstrap keyring automatically${NC}"
fi

# Check if keyring exists now
if [ -f "$BOOTSTRAP_KEYRING" ]; then
    echo -e "${GREEN}  ✓ Bootstrap keyring exists: $BOOTSTRAP_KEYRING${NC}"
    ls -lh "$BOOTSTRAP_KEYRING"
else
    echo -e "${YELLOW}  ⚠ Bootstrap keyring still not found${NC}"
    echo "  Will try alternative methods..."
fi
echo ""

echo -e "${BLUE}=== Step 3: Check for pveceph (Proxmox Ceph Tool) ===${NC}"
echo "-----------------------------------"

USE_PVECEPH=false
if command -v pveceph &> /dev/null; then
    echo -e "${GREEN}  ✓ pveceph found - will use Proxmox Ceph tool${NC}"
    USE_PVECEPH=true
else
    echo -e "${YELLOW}  pveceph not found - will use ceph-volume${NC}"
fi
echo ""

echo -e "${BLUE}=== Step 4: Create OSDs on All Drives ===${NC}"
echo "-----------------------------------"

osd_count=0
failed_drives=()

for drive in "${DRIVES[@]}"; do
    if [ ! -b "/dev/$drive" ]; then
        echo -e "${YELLOW}  /dev/$drive not found, skipping...${NC}"
        continue
    fi
    
    echo -e "${BLUE}  Creating OSD on /dev/$drive...${NC}"
    
    # Try pveceph first if available
    if [ "$USE_PVECEPH" = true ]; then
        if pveceph create "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
            echo -e "${GREEN}    ✓ OSD created on /dev/$drive (using pveceph)${NC}"
            osd_count=$((osd_count + 1))
            continue
        else
            echo -e "${YELLOW}    pveceph failed, trying ceph-volume...${NC}"
        fi
    fi
    
    # Try ceph-volume
    if ceph-volume lvm create --data "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
        echo -e "${GREEN}    ✓ OSD created on /dev/$drive${NC}"
        osd_count=$((osd_count + 1))
    else
        echo -e "${YELLOW}    ⚠ OSD creation had issues${NC}"
        failed_drives+=("$drive")
        
        # Check if OSD was partially created
        sleep 2
        if ceph osd tree 2>/dev/null | grep -q "osd\."; then
            echo "    Checking if OSD exists in cluster..."
            # Count OSDs before and after to see if one was created
        fi
    fi
    echo ""
done

echo "=========================================="
echo -e "${GREEN}OSD Creation Summary${NC}"
echo "=========================================="
echo ""
echo "Successfully created: $osd_count OSD(s) out of ${#DRIVES[@]} drives"

if [ ${#failed_drives[@]} -gt 0 ]; then
    echo -e "${YELLOW}Failed drives: ${failed_drives[@]}${NC}"
    echo "Check logs in /tmp/ceph-osd-*.log for details"
fi
echo ""

echo -e "${BLUE}=== Step 5: Verify Ceph Status ===${NC}"
echo "-----------------------------------"

echo "OSD Tree:"
ceph osd tree 2>/dev/null || echo -e "${YELLOW}Cannot get OSD tree${NC}"
echo ""

echo "Ceph Health:"
ceph health 2>/dev/null || echo -e "${YELLOW}Cannot get health${NC}"
echo ""

echo "OSD Details:"
ceph osd df 2>/dev/null || echo -e "${YELLOW}Cannot get OSD details${NC}"
echo ""

echo "Current OSD count:"
OSD_COUNT=$(ceph osd tree 2>/dev/null | grep -c "osd\." || echo "0")
echo "  Total OSDs: $OSD_COUNT"

if [ "$OSD_COUNT" -ge 3 ]; then
    echo -e "${GREEN}  ✓ SUCCESS: Have 3+ OSDs (fixes TOO_FEW_OSDS error)${NC}"
else
    echo -e "${YELLOW}  ⚠ Still need more OSDs (currently have $OSD_COUNT, need 3+)${NC}"
fi
echo ""

echo "=========================================="
if [ "$osd_count" -gt 0 ]; then
    echo -e "${GREEN}Process Complete!${NC}"
    echo ""
    echo "Next steps:"
    echo "1. Monitor Ceph health: ceph health detail"
    echo "2. Check for remaining issues: ceph health"
    echo "3. If some OSDs failed, check logs: /tmp/ceph-osd-*.log"
else
    echo -e "${YELLOW}No OSDs were created${NC}"
    echo ""
    echo "Troubleshooting:"
    echo "1. Check Ceph cluster connectivity: ceph health"
    echo "2. Check bootstrap keyring: ls -la $BOOTSTRAP_KEYRING"
    echo "3. Check logs: /tmp/ceph-osd-*.log"
    echo "4. Try manual creation: ceph-volume lvm create --data /dev/sdc"
fi
echo "=========================================="
echo ""