Sankofa/scripts/fresh-ceph-install-r630.sh

#!/bin/bash
# Fresh Ceph installation for R630-01 (r630-01.sankofa.nexus)
set -e

HOSTNAME="r630-01"
FQDN="r630-01.sankofa.nexus"
PUBLIC_NETWORK="192.168.11.0/24"
CLUSTER_NETWORK="192.168.11.0/24"
FSID="5fb968ae-12ab-405f-b05f-0df29a168328"

echo "=== Fresh Ceph Installation for $FQDN ==="
echo ""

# Verify we're on the right host
CURRENT_HOST=$(hostname)
if [ "$CURRENT_HOST" != "$HOSTNAME" ]; then
    echo "WARNING: Hostname is $CURRENT_HOST, expected $HOSTNAME"
    read -p "Continue anyway? (y/N) " -n 1 -r
    echo
    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
        exit 1
    fi
fi

echo "=== Step 1: Installing Ceph packages ==="
apt-get update
apt-get install -y ceph ceph-common ceph-base ceph-volume

echo ""
echo "=== Step 2: Cleaning up any existing Ceph data ==="
systemctl stop ceph-*.service 2>/dev/null || true
systemctl disable ceph-*.service 2>/dev/null || true
rm -rf /var/lib/ceph/*
rm -rf /var/log/ceph/*
rm -rf /etc/ceph/*
rm -rf /etc/pve/ceph.conf
rm -rf /run/ceph/*

# Unmount any OSD directories
for osd_dir in /var/lib/ceph/osd/ceph-*; do
    if mountpoint -q "$osd_dir" 2>/dev/null; then
        umount -f "$osd_dir" 2>/dev/null || true
    fi
done

# Remove Ceph LVM volumes
for vg in $(vgs --noheadings -o vg_name | grep ceph); do
    vgremove -f "$vg" 2>/dev/null || true
done

# Wipe Ceph drives
for disk in sdc sdd sde sdf sdg sdh; do
    if [ -e "/dev/$disk" ]; then
        echo "Wiping /dev/$disk..."
        wipefs -a /dev/$disk 2>/dev/null || true
        dd if=/dev/zero of=/dev/$disk bs=1M count=100 2>/dev/null || true
    fi
done

echo ""
echo "=== Step 3: Creating Ceph configuration ==="
cat > /etc/pve/ceph.conf <<EOF
[global]
	auth_client_required = cephx
	auth_cluster_required = cephx
	auth_service_required = cephx
	cluster_network = $CLUSTER_NETWORK
	fsid = $FSID
	mon_allow_pool_delete = true
	mon_host = 192.168.11.10 192.168.11.11
	ms_bind_ipv4 = true
	ms_bind_ipv6 = false
	osd_pool_default_min_size = 2
	osd_pool_default_size = 3
	public_network = $PUBLIC_NETWORK

[client]
	keyring = /etc/pve/priv/\$cluster.\$name.keyring

[client.crash]
	keyring = /etc/pve/ceph/\$cluster.\$name.keyring

[mds]
	keyring = /var/lib/ceph/mds/ceph-\$id/keyring

[mds.ml110-01]
	host = ml110-01
	mds_standby_for_name = pve

[mds.r630-01]
	host = r630-01
	mds_standby_for_name = pve

[mon.ml110-01]
	public_addr = 192.168.11.10

[mon.r630-01]
	public_addr = 192.168.11.11
EOF

# Create symlink
ln -sf /etc/pve/ceph.conf /etc/ceph/ceph.conf

echo "Configuration created"

echo ""
echo "=== Step 4: Initializing Ceph cluster (if not already done) ==="
# Check if cluster is already initialized by checking for bootstrap keyring
if [ ! -f "/var/lib/ceph/bootstrap-osd/ceph.keyring" ]; then
    echo "Cluster not initialized. Initializing..."
    pveceph init --network $PUBLIC_NETWORK --cluster-network $CLUSTER_NETWORK 2>&1 || {
        echo "Init may have already been done or failed. Continuing..."
    }
else
    echo "Cluster appears to be initialized (bootstrap keyring exists)"
fi

echo ""
echo "=== Step 5: Creating monitor on $HOSTNAME ==="
# Remove any existing monitor directory
rm -rf /var/lib/ceph/mon/ceph-$HOSTNAME/*

# Create monitor
pveceph mon create 2>&1 || {
    echo "Monitor creation failed or already exists. Checking status..."
    systemctl status ceph-mon@$HOSTNAME --no-pager | head -10 || true
}

echo ""
echo "=== Step 6: Starting monitor service ==="
systemctl enable ceph-mon@$HOSTNAME
systemctl start ceph-mon@$HOSTNAME
sleep 5

echo ""
echo "=== Step 7: Verifying monitor status ==="
systemctl status ceph-mon@$HOSTNAME --no-pager | head -20

echo ""
echo "=== Step 8: Checking quorum status ==="
sleep 10
if ceph quorum_status 2>&1 | head -20; then
    echo "Quorum established!"
else
    echo "Quorum not yet established. This is normal if ml110-01 monitor is not running."
    echo "Monitor is running and will join quorum when both monitors are up."
fi

echo ""
echo "=== Step 9: Preparing drives for OSD creation ==="
echo "Available drives:"
lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE | grep -E "(sd[cd]|sd[ef]|sd[gh]|NAME)"

echo ""
echo "=== COMPLETE ==="
echo "Ceph monitor installed and configured on $FQDN"
echo ""
echo "Next steps:"
echo "1. Verify monitor is running: systemctl status ceph-mon@$HOSTNAME"
echo "2. Check quorum: ceph quorum_status"
echo "3. Create OSDs: ceph-volume lvm create --data /dev/<disk>"
echo "4. Check cluster health: ceph -s"