Sankofa/scripts/complete-ceph-reinstall.sh

#!/bin/bash
# Complete Ceph uninstall, cleanup, and reinstall
set -e

echo "=== CEPH COMPLETE REINSTALL SCRIPT ==="
echo "WARNING: This will destroy all Ceph data and configuration!"
echo ""

# Backup critical configs first
echo "=== Step 1: Backing up critical configurations ==="
BACKUP_DIR="/root/ceph-backup-$(date +%Y%m%d-%H%M%S)"
mkdir -p "$BACKUP_DIR"
cp -a /etc/ceph/ "$BACKUP_DIR/ceph-config/" 2>/dev/null || true
cp -a /etc/pve/ceph.conf "$BACKUP_DIR/" 2>/dev/null || true
cp -a /var/lib/ceph/bootstrap-osd/ "$BACKUP_DIR/bootstrap-osd/" 2>/dev/null || true
echo "Backup created at: $BACKUP_DIR"

echo ""
echo "=== Step 2: Stopping all Ceph services ==="
systemctl stop ceph-osd@*.service 2>/dev/null || true
systemctl stop ceph-mon@*.service 2>/dev/null || true
systemctl stop ceph-mgr@*.service 2>/dev/null || true
systemctl stop ceph-mds@*.service 2>/dev/null || true
systemctl stop ceph.target 2>/dev/null || true
sleep 3

echo ""
echo "=== Step 3: Disabling all Ceph services ==="
systemctl disable ceph-osd@*.service 2>/dev/null || true
systemctl disable ceph-mon@*.service 2>/dev/null || true
systemctl disable ceph-mgr@*.service 2>/dev/null || true
systemctl disable ceph-mds@*.service 2>/dev/null || true
systemctl disable ceph.target 2>/dev/null || true

echo ""
echo "=== Step 4: Unmounting OSD directories ==="
# Unmount all OSD tmpfs mounts
for osd_dir in /var/lib/ceph/osd/ceph-*; do
    if mountpoint -q "$osd_dir" 2>/dev/null; then
        echo "Unmounting $osd_dir..."
        umount -f "$osd_dir" 2>/dev/null || true
    fi
done
sleep 2

echo ""
echo "=== Step 5: Removing Ceph packages (Ceph only, not Proxmox) ==="
# Only remove Ceph packages, not Proxmox dependencies
apt-get remove --purge -y ceph ceph-base ceph-common ceph-mgr ceph-mon ceph-osd ceph-mds ceph-fuse ceph-volume 2>/dev/null || true
apt-get remove --purge -y libcephfs2 librados2 librbd1 python3-cephfs python3-rados python3-rbd 2>/dev/null || true
apt-get autoremove -y 2>/dev/null || true

echo ""
echo "=== Step 6: Cleaning up Ceph directories ==="
# Force remove OSD directories
for osd_dir in /var/lib/ceph/osd/ceph-*; do
    if [ -d "$osd_dir" ]; then
        umount -f "$osd_dir" 2>/dev/null || true
        rm -rf "$osd_dir" 2>/dev/null || true
    fi
done
rm -rf /var/lib/ceph/mon/*
rm -rf /var/lib/ceph/mgr/*
rm -rf /var/lib/ceph/mds/*
rm -rf /var/lib/ceph/bootstrap-osd/*
rm -rf /var/log/ceph/*
rm -rf /etc/ceph/*
rm -rf /etc/pve/ceph.conf
rm -rf /run/ceph/*

echo ""
echo "=== Step 7: Removing Ceph LVM volumes ==="
# Find and remove all Ceph LVs
for vg in $(vgs --noheadings -o vg_name | grep ceph); do
    echo "Removing volume group: $vg"
    vgremove -f "$vg" 2>/dev/null || true
done

# Remove Ceph physical volumes
for pv in $(pvs --noheadings -o pv_name | grep -E "(sdc|sdd|sde|sdf|sdg|sdh)"); do
    echo "Removing physical volume: $pv"
    pvremove -f "$pv" 2>/dev/null || true
done

echo ""
echo "=== Step 8: Wiping Ceph drives ==="
for disk in sdc sdd sde sdf sdg sdh; do
    if [ -e "/dev/$disk" ]; then
        echo "Wiping /dev/$disk..."
        wipefs -a /dev/$disk 2>/dev/null || true
        dd if=/dev/zero of=/dev/$disk bs=1M count=100 2>/dev/null || true
    fi
done

echo ""
echo "=== Step 9: Flushing kernel caches ==="
sync
echo 3 > /proc/sys/vm/drop_caches

echo ""
echo "=== Step 10: Reinstalling Ceph ==="
apt-get update
apt-get install -y ceph ceph-common ceph-base

echo ""
echo "=== Step 11: Restoring bootstrap keyring if available ==="
if [ -d "$BACKUP_DIR/bootstrap-osd" ]; then
    mkdir -p /var/lib/ceph/bootstrap-osd/ceph/
    cp -a "$BACKUP_DIR/bootstrap-osd/"* /var/lib/ceph/bootstrap-osd/ceph/ 2>/dev/null || true
fi

echo ""
echo "=== Step 12: Restoring Ceph configuration if available ==="
if [ -d "$BACKUP_DIR/ceph-config" ]; then
    mkdir -p /etc/ceph/
    cp -a "$BACKUP_DIR/ceph-config/"* /etc/ceph/ 2>/dev/null || true
fi

if [ -f "$BACKUP_DIR/ceph.conf" ]; then
    cp "$BACKUP_DIR/ceph.conf" /etc/pve/ceph.conf 2>/dev/null || true
fi

echo ""
echo "=== COMPLETE ==="
echo "Ceph has been uninstalled, cleaned, and reinstalled."
echo "Backup location: $BACKUP_DIR"
echo ""
echo "Next steps:"
echo "1. Review /etc/ceph/ceph.conf and /etc/pve/ceph.conf"
echo "2. Reinitialize Ceph cluster if needed: pveceph init"
echo "3. Create monitors: pveceph mon create <node>"
echo "4. Create OSDs: ceph-volume lvm create --data /dev/<disk>"