Sankofa/scripts/setup-ceph-osds-r630.sh

#!/bin/bash
# Setup Ceph OSDs on R630-01 available disks
# Creates OSDs on sdc-sdh (6x 232.9G SSDs) and optionally sdb (279.4G)
# Run on R630-01 as root

set +e  # Don't exit on error - we want to continue even if some OSDs fail

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
NC='\033[0m'

# Configuration
PRIMARY_DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")  # 6x SSDs
SECONDARY_DRIVE="sdb"  # Optional 279.4G disk
INCLUDE_SDB="${INCLUDE_SDB:-false}"  # Set to "true" to include sdb

echo "=========================================="
echo -e "${CYAN}Ceph OSD Setup for R630-01${NC}"
echo "=========================================="
echo ""

# Check if running as root
if [ "$EUID" -ne 0 ]; then 
    echo -e "${RED}ERROR: Please run as root${NC}"
    exit 1
fi

# Check if Ceph is installed
echo -e "${BLUE}=== Step 1: Prerequisites Check ===${NC}"
echo "-----------------------------------"

if ! command -v ceph &>/dev/null; then
    echo -e "${RED}ERROR: Ceph is not installed${NC}"
    echo "Please install Ceph first"
    exit 1
fi
echo -e "${GREEN}✓ Ceph is installed${NC}"

if ! command -v ceph-volume &>/dev/null; then
    echo -e "${RED}ERROR: ceph-volume is not installed${NC}"
    exit 1
fi
echo -e "${GREEN}✓ ceph-volume is installed${NC}"

# Check cluster connectivity
echo ""
echo "Checking Ceph cluster connectivity..."
if ceph quorum_status &>/dev/null 2>&1; then
    echo -e "${GREEN}✓ Ceph cluster is accessible${NC}"
    CLUSTER_ACCESSIBLE=true
else
    echo -e "${YELLOW}⚠ Warning: Cannot access Ceph cluster${NC}"
    echo "  This might be normal if this is the first OSD"
    CLUSTER_ACCESSIBLE=false
fi

# Show current OSD status
if [ "$CLUSTER_ACCESSIBLE" = true ]; then
    echo ""
    echo "Current OSD status:"
    ceph osd tree 2>/dev/null || echo -e "${YELLOW}  Cannot get OSD tree${NC}"
    echo ""
    CURRENT_OSD_COUNT=$(ceph osd tree 2>/dev/null | grep -c "osd\." || echo "0")
    echo "Current OSD count: $CURRENT_OSD_COUNT"
fi
echo ""

# Verify disks are available
echo -e "${BLUE}=== Step 2: Disk Verification ===${NC}"
echo "-----------------------------------"

available_disks=()
unavailable_disks=()

# Check primary drives (sdc-sdh)
echo "Checking primary drives (sdc-sdh)..."
for disk in "${PRIMARY_DRIVES[@]}"; do
    if [ ! -b "/dev/$disk" ]; then
        echo -e "  ${RED}✗ /dev/$disk not found${NC}"
        unavailable_disks+=("$disk")
        continue
    fi
    
    # Check if already has OSD
    if ceph-volume lvm list /dev/$disk 2>/dev/null | grep -q "osd\."; then
        echo -e "  ${YELLOW}⚠ /dev/$disk already has an OSD, skipping${NC}"
        unavailable_disks+=("$disk")
        continue
    fi
    
    # Check if has partitions or is mounted
    partitions=$(lsblk -n /dev/$disk 2>/dev/null | grep -c "part" || echo "0")
    partitions=$(echo "$partitions" | tr -d '\n\r ' | head -1)
    mountpoint=$(lsblk -d -n -o MOUNTPOINT /dev/$disk 2>/dev/null | tr -d '\n' || echo "")
    
    if [ "${partitions:-0}" -eq 0 ] && [ -z "$mountpoint" ]; then
        size=$(lsblk -d -n -o SIZE /dev/$disk 2>/dev/null || echo "unknown")
        echo -e "  ${GREEN}✓ /dev/$disk ($size) - Available${NC}"
        available_disks+=("$disk")
    else
        echo -e "  ${YELLOW}⚠ /dev/$disk has partitions or is mounted, skipping${NC}"
        unavailable_disks+=("$disk")
    fi
done

# Check secondary drive (sdb) if requested
if [ "$INCLUDE_SDB" = "true" ]; then
    echo ""
    echo "Checking secondary drive (sdb)..."
    if [ ! -b "/dev/$SECONDARY_DRIVE" ]; then
        echo -e "  ${RED}✗ /dev/$SECONDARY_DRIVE not found${NC}"
    elif ceph-volume lvm list /dev/$SECONDARY_DRIVE 2>/dev/null | grep -q "osd\."; then
        echo -e "  ${YELLOW}⚠ /dev/$SECONDARY_DRIVE already has an OSD, skipping${NC}"
    else
        # Check if it has LVM signature (needs wiping)
        if pvs 2>/dev/null | grep -q "/dev/$SECONDARY_DRIVE"; then
            echo -e "  ${YELLOW}⚠ /dev/$SECONDARY_DRIVE has LVM signature (will be wiped)${NC}"
        fi
        size=$(lsblk -d -n -o SIZE /dev/$SECONDARY_DRIVE 2>/dev/null || echo "unknown")
        echo -e "  ${GREEN}✓ /dev/$SECONDARY_DRIVE ($size) - Will be prepared${NC}"
        available_disks+=("$SECONDARY_DRIVE")
    fi
fi

echo ""
echo "Summary:"
echo "  Available disks: ${#available_disks[@]}"
echo "  Unavailable/skipped: ${#unavailable_disks[@]}"

if [ ${#available_disks[@]} -eq 0 ]; then
    echo -e "${RED}ERROR: No available disks to create OSDs on${NC}"
    exit 1
fi

echo ""
read -p "Continue with OSD creation on ${#available_disks[@]} disk(s)? (yes/no): " confirm
if [ "$confirm" != "yes" ]; then
    echo "Aborted by user"
    exit 0
fi
echo ""

# Prepare disks
echo -e "${BLUE}=== Step 3: Prepare Disks ===${NC}"
echo "-----------------------------------"

for disk in "${available_disks[@]}"; do
    echo "Preparing /dev/$disk..."
    
    # Unmount any partitions
    umount /dev/${disk}* 2>/dev/null || true
    
    # For sdb, remove LVM signature if present
    if [ "$disk" = "$SECONDARY_DRIVE" ]; then
        if pvs 2>/dev/null | grep -q "/dev/$disk"; then
            echo "  Removing LVM signature..."
            pvremove -f /dev/$disk 2>/dev/null || true
        fi
    fi
    
    # Wipe filesystem signatures (but keep partition table if needed)
    echo "  Wiping filesystem signatures..."
    wipefs -a /dev/$disk 2>/dev/null || true
    
    echo -e "  ${GREEN}✓ /dev/$disk prepared${NC}"
done
echo ""

# Create OSDs
echo -e "${BLUE}=== Step 4: Create Ceph OSDs ===${NC}"
echo "-----------------------------------"

osd_count=0
failed_disks=()
successful_osds=()

for disk in "${available_disks[@]}"; do
    echo -e "${CYAN}Creating OSD on /dev/$disk...${NC}"
    
    # Create OSD with logging
    log_file="/tmp/ceph-osd-${disk}.log"
    if ceph-volume lvm create --data /dev/$disk 2>&1 | tee "$log_file"; then
        # Try to extract OSD ID
        osd_id=$(grep -oP "osd\.\K\d+" "$log_file" 2>/dev/null | head -1)
        if [ -n "$osd_id" ]; then
            echo -e "${GREEN}  ✓ OSD $osd_id created on /dev/$disk${NC}"
            successful_osds+=("osd.$osd_id")
            
            # Enable and start service
            if systemctl enable ceph-osd@$osd_id 2>/dev/null; then
                if systemctl start ceph-osd@$osd_id 2>/dev/null; then
                    echo -e "${GREEN}  ✓ OSD $osd_id service started${NC}"
                else
                    echo -e "${YELLOW}  ⚠ OSD $osd_id service start had issues${NC}"
                fi
            fi
        else
            echo -e "${GREEN}  ✓ OSD created on /dev/$disk (ID not extracted)${NC}"
        fi
        osd_count=$((osd_count + 1))
    else
        echo -e "${RED}  ✗ Failed to create OSD on /dev/$disk${NC}"
        failed_disks+=("$disk")
        echo "  Check $log_file for details"
    fi
    echo ""
done

# Summary
echo "=========================================="
echo -e "${CYAN}OSD Creation Summary${NC}"
echo "=========================================="
echo ""
echo "Successfully created: $osd_count OSD(s)"
echo "Successful OSDs: ${successful_osds[@]}"

if [ ${#failed_disks[@]} -gt 0 ]; then
    echo -e "${YELLOW}Failed disks: ${failed_disks[@]}${NC}"
fi
echo ""

# Verify Ceph status
echo -e "${BLUE}=== Step 5: Verify Ceph Status ===${NC}"
echo "-----------------------------------"

if [ "$CLUSTER_ACCESSIBLE" = true ] || ceph quorum_status &>/dev/null 2>&1; then
    echo "Ceph Health:"
    ceph health 2>/dev/null || echo -e "${YELLOW}Cannot get health status${NC}"
    echo ""
    
    echo "OSD Tree:"
    ceph osd tree 2>/dev/null || echo -e "${YELLOW}Cannot get OSD tree${NC}"
    echo ""
    
    echo "OSD Details:"
    ceph osd df 2>/dev/null || echo -e "${YELLOW}Cannot get OSD details${NC}"
    echo ""
    
    NEW_OSD_COUNT=$(ceph osd tree 2>/dev/null | grep -c "osd\." || echo "0")
    echo "Total OSD count: $NEW_OSD_COUNT"
    
    if [ "$NEW_OSD_COUNT" -ge 3 ]; then
        echo -e "${GREEN}  ✓ SUCCESS: Have 3+ OSDs (should resolve TOO_FEW_OSDS if present)${NC}"
    elif [ "$NEW_OSD_COUNT" -ge 1 ]; then
        echo -e "${YELLOW}  ⚠ Have $NEW_OSD_COUNT OSD(s). Need 3+ for production use.${NC}"
    fi
else
    echo -e "${YELLOW}Ceph cluster not accessible. OSDs may need time to join cluster.${NC}"
fi
echo ""

# Check OSD services
echo -e "${BLUE}=== Step 6: OSD Service Status ===${NC}"
echo "-----------------------------------"
systemctl list-units --type=service | grep ceph-osd | head -10 || echo "No OSD services found"
echo ""

echo "=========================================="
echo -e "${GREEN}Setup Complete!${NC}"
echo "=========================================="
echo ""
echo "Next steps:"
echo "1. Monitor Ceph health: ceph health"
echo "2. Check OSD status: ceph osd tree"
echo "3. Verify OSD services: systemctl status ceph-osd@<id>"
echo "4. If needed, create more OSDs on other nodes"
echo ""