Sankofa/scripts/fix-ceph-rbd-storage.sh

#!/bin/bash
# Fix Ceph RBD storage configuration on r630-01
# This script fixes the "No such file or directory" error when accessing RBD storage

set -e

echo "═══════════════════════════════════════════════════════════════════════════════"
echo "  FIX CEPH RBD STORAGE CONFIGURATION"
echo "═══════════════════════════════════════════════════════════════════════════════"
echo ""

# Check if running on r630-01
HOSTNAME=$(hostname)
if [ "$HOSTNAME" != "r630-01" ]; then
    echo "⚠️  WARNING: This script should be run on r630-01"
    echo "   Current hostname: $HOSTNAME"
    echo ""
    echo "To run remotely via SSH:"
    echo "  ssh root@192.168.11.11 'bash -s' < $0"
    echo ""
    read -p "Continue anyway? (y/N) " -n 1 -r
    echo
    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
        exit 1
    fi
fi

echo "Step 1: Checking Ceph cluster status..."
if ! command -v ceph &> /dev/null; then
    echo "❌ ERROR: ceph command not found. Ceph packages may not be installed."
    exit 1
fi

CEPH_HEALTH=$(ceph health detail)
echo "$CEPH_HEALTH"
if [[ "$CEPH_HEALTH" == *"HEALTH_ERR"* ]]; then
    echo "❌ ERROR: Ceph cluster is in HEALTH_ERR state. Please fix Ceph issues first."
    exit 1
fi
echo ""

echo "Step 2: Checking RBD pool 'rbd'..."
if ! ceph osd pool ls | grep -q "^rbd$"; then
    echo "  Creating RBD pool 'rbd'..."
    ceph osd pool create rbd 32 32 || { echo "❌ Failed to create RBD pool 'rbd'"; exit 1; }
    echo "✅ RBD pool 'rbd' created."
else
    echo "✅ RBD pool 'rbd' exists."
fi
echo ""

echo "Step 3: Initializing RBD pool..."
if ! rbd pool init rbd 2>/dev/null; then
    echo "⚠️  RBD pool 'rbd' already initialized or initialization failed."
else
    echo "✅ RBD pool 'rbd' initialized."
fi
echo ""

echo "Step 4: Enabling RBD application on pool..."
if ! ceph osd pool application get rbd 2>/dev/null | grep -q "rbd"; then
    echo "  Enabling RBD application..."
    ceph osd pool application enable rbd rbd || { echo "❌ Failed to enable RBD application"; exit 1; }
    echo "✅ RBD application enabled."
else
    echo "✅ RBD application already enabled."
fi
echo ""

echo "Step 5: Checking RBD pool status..."
if rbd ls -p rbd &>/dev/null; then
    echo "✅ RBD pool is accessible."
    echo "   Images in pool:"
    rbd ls -p rbd | head -5 || echo "   (no images yet)"
else
    echo "⚠️  RBD pool access test failed, but pool exists."
fi
echo ""

echo "Step 6: Checking Proxmox RBD storage configuration..."
if pvesm status 2>/dev/null | grep -q "ceph-rbd"; then
    echo "✅ ceph-rbd storage exists in Proxmox."
    echo ""
    echo "Storage details:"
    pvesm status | grep "ceph-rbd"
    echo ""
    
    # Check if storage needs to be removed and re-added
    echo "Step 7: Verifying storage configuration..."
    if pvesm status | grep "ceph-rbd" | grep -q "inactive"; then
        echo "⚠️  Storage is inactive. Attempting to fix..."
        
        # Get storage config
        STORAGE_CFG="/etc/pve/storage.cfg"
        if [ -f "$STORAGE_CFG" ]; then
            echo "   Current ceph-rbd storage config:"
            grep -A 10 "^rbd: ceph-rbd" "$STORAGE_CFG" || echo "   (not found in config)"
            echo ""
            
            echo "   Removing and re-adding ceph-rbd storage..."
            # Remove storage
            pvesm remove ceph-rbd 2>/dev/null || echo "   (storage may not exist or already removed)"
            
            # Re-add storage
            echo "   Re-adding ceph-rbd storage..."
            pvesm add rbd ceph-rbd \
                --nodes r630-01 \
                --pool rbd \
                --content images \
                --monhost 192.168.11.10:6789,192.168.11.11:6789 \
                --username admin 2>&1 || {
                echo "❌ Failed to re-add ceph-rbd storage."
                echo "   Try manually via Web UI: Datacenter → Storage → Add → RBD"
                exit 1
            }
            echo "✅ ceph-rbd storage re-added."
        fi
    else
        echo "✅ Storage appears to be active."
    fi
else
    echo "⚠️  ceph-rbd storage not found in Proxmox."
    echo "   Adding ceph-rbd storage..."
    pvesm add rbd ceph-rbd \
        --nodes r630-01 \
        --pool rbd \
        --content images \
        --monhost 192.168.11.10:6789,192.168.11.11:6789 \
        --username admin 2>&1 || {
        echo "❌ Failed to add ceph-rbd storage."
        exit 1
    }
    echo "✅ ceph-rbd storage added."
fi
echo ""

echo "Step 8: Testing RBD storage access..."
if pvesm list ceph-rbd &>/dev/null; then
    echo "✅ RBD storage is accessible via Proxmox."
    echo "   Storage content:"
    pvesm list ceph-rbd | head -5 || echo "   (empty - no images yet)"
else
    echo "⚠️  RBD storage listing failed, but storage exists."
    echo "   Error details:"
    pvesm list ceph-rbd 2>&1 | head -3
fi
echo ""

echo "═══════════════════════════════════════════════════════════════════════════════"
echo "  VERIFICATION"
echo "═══════════════════════════════════════════════════════════════════════════════"
echo "Checking storage status..."
pvesm status | grep "ceph-rbd" || echo "Storage 'ceph-rbd' not found in pvesm status"
echo ""

echo "✅ Fix complete! RBD storage should now be working."
echo ""
echo "If errors persist, check:"
echo "  1. Ceph cluster health: ceph health"
echo "  2. RBD pool exists: ceph osd pool ls | grep rbd"
echo "  3. RBD pool is initialized: rbd pool init rbd"
echo "  4. Ceph keyring is accessible: ls -la /etc/pve/priv/ceph/ceph-rbd.keyring"