#!/bin/bash set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true # Fix Storage Issues on r630-02 # Implements recommendations from storage review set -e HOST="r630-02" HOST_IP="${PROXMOX_HOST_R630_02}" echo "🔧 Storage Fix Script for r630-02" echo "" # Check connectivity if ! ping -c 1 -W 2 $HOST_IP >/dev/null 2>&1; then echo "❌ Host $HOST_IP is not reachable" exit 1 fi echo "✅ Host is reachable" echo "" # Check if running as root or with sudo if [ "$EUID" -ne 0 ]; then echo "⚠️ This script requires root access for some operations" echo " Some fixes will be done via SSH to r630-02" fi echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "1️⃣ EXPANDING CONTAINER 7811 DISK (CRITICAL)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" echo "📋 Current status of container 7811:" ssh root@$HOST_IP "pct df 7811 | grep rootfs" echo "" read -p "Expand container 7811 disk from 30GB to 50GB? (y/N): " -n 1 -r echo "" if [[ $REPLY =~ ^[Yy]$ ]]; then echo "🔄 Expanding container 7811 disk..." ssh root@$HOST_IP "pct resize 7811 rootfs +20G" if [ $? -eq 0 ]; then echo "✅ Container 7811 disk expanded successfully" echo "" echo "📊 New status:" ssh root@$HOST_IP "pct df 7811 | grep rootfs" else echo "❌ Failed to expand container 7811 disk" fi else echo "⏭️ Skipping container 7811 disk expansion" fi echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "2️⃣ VERIFYING DUPLICATE VOLUMES ON THIN1" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" echo "📋 Checking for duplicate volumes on thin1:" DUPLICATES=$(ssh root@$HOST_IP "lvs -o lv_name,vg_name --units g | grep -E 'vm-5000-disk-0.*thin1|vm-6200-disk-0.*thin1|vm-6201-disk-0.*thin1' | wc -l") if [ "$DUPLICATES" -gt 0 ]; then echo "⚠️ Found $DUPLICATES duplicate volumes on thin1:" ssh root@$HOST_IP "lvs -o lv_name,vg_name,lv_size --units g | grep -E 'vm-5000-disk-0.*thin1|vm-6200-disk-0.*thin1|vm-6201-disk-0.*thin1'" echo "" echo "🔍 Verifying containers are using thin2 (not thin1):" for CTID in 5000 6200 6201; do echo "" echo "Container $CTID:" ssh root@$HOST_IP "pct config $CTID | grep rootfs" done echo "" read -p "Remove duplicate volumes from thin1? (y/N): " -n 1 -r echo "" if [[ $REPLY =~ ^[Yy]$ ]]; then echo "🗑️ Removing duplicate volumes..." for CTID in 5000 6200 6201; do echo "Removing vm-$CTID-disk-0 from thin1..." ssh root@$HOST_IP "lvremove -f /dev/thin1/vm-$CTID-disk-0" 2>/dev/null && echo "✅ Removed" || echo "⚠️ Failed or already removed" done echo "" echo "📊 Updated thin1 pool status:" ssh root@$HOST_IP "pvesm status | grep thin1-r630-02" else echo "⏭️ Skipping duplicate volume removal" fi else echo "✅ No duplicate volumes found on thin1" fi echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "3️⃣ REVIEWING OLD BACKUP FILES" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" echo "📋 Backup files in /var/lib/vz/dump:" BACKUP_COUNT=$(ssh root@$HOST_IP "find /var/lib/vz/dump -name '*.tar.gz' 2>/dev/null | wc -l") BACKUP_SIZE=$(ssh root@$HOST_IP "du -sh /var/lib/vz/dump 2>/dev/null | awk '{print \$1}'") echo " Count: $BACKUP_COUNT files" echo " Total Size: $BACKUP_SIZE" echo "" echo "📋 Listing backup files:" ssh root@$HOST_IP "ls -lh /var/lib/vz/dump/*.tar.gz 2>/dev/null | awk '{print \$9, \$5}' | head -10" echo "" read -p "Archive backups older than 30 days? (y/N): " -n 1 -r echo "" if [[ $REPLY =~ ^[Yy]$ ]]; then echo "📦 Archiving old backups..." ssh root@$HOST_IP "mkdir -p /var/lib/vz/archive && find /var/lib/vz/dump -name '*.tar.gz' -mtime +30 -exec mv {} /var/lib/vz/archive/ \;" ARCHIVED=$(ssh root@$HOST_IP "find /var/lib/vz/archive -name '*.tar.gz' 2>/dev/null | wc -l") echo "✅ Archived $ARCHIVED backup files" else echo "⏭️ Skipping backup archival" fi echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "4️⃣ MOVING CONTAINER TO EMPTY POOL (OPTIONAL)" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" echo "📋 Container 6201 (firefly-ali-1) is currently stopped and on thin2" echo " Moving it to thin3 would free space on thin2" echo "" read -p "Move container 6201 to thin3? (y/N): " -n 1 -r echo "" if [[ $REPLY =~ ^[Yy]$ ]]; then echo "🔄 Moving container 6201 to thin3..." echo " This requires moving the volume, which may take time" ssh root@$HOST_IP "pct move-volume 6201 rootfs thin3" if [ $? -eq 0 ]; then echo "✅ Container 6201 moved to thin3 successfully" echo "" echo "📊 Updated container config:" ssh root@$HOST_IP "pct config 6201 | grep rootfs" else echo "❌ Failed to move container 6201" fi else echo "⏭️ Skipping container move" fi echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "✅ STORAGE FIX SCRIPT COMPLETE" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" echo "📊 Final Storage Status:" ssh root@$HOST_IP "pvesm status | grep -E 'thin1-r630-02|thin2|thin3|thin4' | awk '{printf \" %-20s %6s used\n\", \$1, \$7}'" echo "" echo "💡 Next Steps:" echo " 1. Monitor container 7811 disk usage" echo " 2. Monitor thin pool usage trends" echo " 3. Review storage review document: docs/04-configuration/R630-02_STORAGE_REVIEW.md" echo ""