Sankofa/scripts/fix-r630-mon-quorum.sh

#!/bin/bash
# Fix R630-01 monitor to join quorum
set -e

echo "=== Fixing R630-01 Monitor Quorum ==="

# Check current status
echo "1. Checking monitor service status..."
systemctl status ceph-mon@r630-01 --no-pager | head -15

echo ""
echo "2. Checking monitor directory..."
ls -la /var/lib/ceph/mon/ceph-r630-01/ 2>&1 || echo "Monitor directory missing!"

echo ""
echo "3. Checking monitor logs..."
journalctl -u ceph-mon@r630-01 -n 20 --no-pager | tail -20

echo ""
echo "4. Checking if monitor is listening..."
ss -ltnp | grep -E "(:3300|:6789)" || echo "No monitor listeners found"

echo ""
echo "5. Attempting to add monitor to cluster..."
# Get monmap from ml110-01
echo "Fetching monmap from ml110-01..."
ceph mon getmap -o /tmp/monmap 2>&1 || echo "Failed to get monmap"

if [ -f /tmp/monmap ]; then
    echo "Monmap retrieved, checking contents..."
    monmaptool --print /tmp/monmap 2>&1
    
    # Check if r630-01 is in monmap
    if ! monmaptool --print /tmp/monmap 2>&1 | grep -q "r630-01"; then
        echo "r630-01 not in monmap, adding..."
        # Get cluster FSID
        FSID=$(ceph config get mon fsid 2>/dev/null || grep "fsid" /etc/pve/ceph.conf | awk '{print $3}')
        if [ -z "$FSID" ]; then
            FSID=$(grep "fsid" /etc/pve/ceph.conf | awk '{print $3}')
        fi
        
        if [ -n "$FSID" ]; then
            echo "Adding r630-01 to monmap with FSID: $FSID"
            monmaptool --add r630-01 192.168.11.11 --fsid $FSID /tmp/monmap 2>&1
            # Inject updated monmap
            ceph mon setmap -i /tmp/monmap 2>&1 || echo "Failed to set monmap"
        fi
    else
        echo "r630-01 already in monmap"
    fi
fi

echo ""
echo "6. Checking if monitor directory needs creation..."
if [ ! -d "/var/lib/ceph/mon/ceph-r630-01" ] || [ -z "$(ls -A /var/lib/ceph/mon/ceph-r630-01 2>/dev/null)" ]; then
    echo "Monitor directory missing or empty, creating..."
    
    # Get keyring
    if [ -f "/var/lib/ceph/bootstrap-osd/ceph.keyring" ]; then
        echo "Using bootstrap keyring to create monitor..."
        mkdir -p /var/lib/ceph/mon/ceph-r630-01
        chown ceph:ceph /var/lib/ceph/mon/ceph-r630-01
        
        # Get monmap and keyring
        if [ -f /tmp/monmap ]; then
            ceph-authtool --create-keyring /var/lib/ceph/mon/ceph-r630-01/keyring --gen-key -n mon. 2>&1 || true
            ceph mon getmap -o /tmp/monmap 2>&1 || true
            
            if [ -f /tmp/monmap ] && [ -f /var/lib/ceph/mon/ceph-r630-01/keyring ]; then
                echo "Creating monitor filesystem..."
                ceph-mon --mkfs -i r630-01 --monmap /tmp/monmap --keyring /var/lib/ceph/mon/ceph-r630-01/keyring 2>&1
                chown -R ceph:ceph /var/lib/ceph/mon/ceph-r630-01
            fi
        fi
    else
        echo "Bootstrap keyring not found, using pveceph..."
        pveceph mon create 2>&1 || echo "pveceph mon create failed"
    fi
fi

echo ""
echo "7. Restarting monitor service..."
systemctl restart ceph-mon@r630-01
sleep 5
systemctl status ceph-mon@r630-01 --no-pager | head -15

echo ""
echo "8. Waiting for quorum..."
sleep 10
ceph quorum_status 2>&1 | head -10

echo ""
echo "=== Done ==="
echo "Check quorum with: ceph quorum_status"
echo "Check cluster with: ceph -s"