b3a8fe4496
Some checks failed
Deploy to Phoenix / deploy (push) Has been cancelled
- Config, docs, scripts, and backup manifests - Submodule refs unchanged (m = modified content in submodules) Made-with: Cursor
117 lines
4.9 KiB
Bash
117 lines
4.9 KiB
Bash
#!/usr/bin/env bash
|
|
# Fix high storage: r630-01 data/local-lvm (72%) and r630-02 thin5 (84.6%).
|
|
# - thin5: Only VMID 5000 (Blockscout) uses it; prune logs/Docker inside 5000. Optional: migrate 5000 to thin2/thin6.
|
|
# - r630-01 data: Prune logs/journal in CTs on local-lvm to free space.
|
|
#
|
|
# Usage:
|
|
# ./scripts/maintenance/fix-storage-r630-01-and-thin5.sh [--dry-run] [--prune-only] [--migrate-5000 TARGET]
|
|
# --dry-run Print actions only.
|
|
# --prune-only Only prune logs (no migration).
|
|
# --migrate-5000 Migrate VMID 5000 from thin5 to TARGET (e.g. thin2 or thin6). Does backup/restore.
|
|
# Requires: SSH key-based access to r630-01 and r630-02. Run from project root (LAN).
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
[[ -f "${PROJECT_ROOT}/config/ip-addresses.conf" ]] && source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
|
|
|
|
R630_01="${PROXMOX_HOST_R630_01:-192.168.11.11}"
|
|
R630_02="${PROXMOX_HOST_R630_02:-192.168.11.12}"
|
|
SSH_OPTS="-o ConnectTimeout=10 -o StrictHostKeyChecking=no"
|
|
|
|
DRY_RUN=false
|
|
PRUNE_ONLY=true
|
|
MIGRATE_5000_TARGET=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--dry-run) DRY_RUN=true; shift ;;
|
|
--prune-only) PRUNE_ONLY=true; shift ;;
|
|
--migrate-5000) MIGRATE_5000_TARGET="${2:-}"; PRUNE_ONLY=false; shift 2 ;;
|
|
*) shift ;;
|
|
esac
|
|
done
|
|
|
|
run_r630_01() { ssh $SSH_OPTS "root@$R630_01" "$@"; }
|
|
run_r630_02() { ssh $SSH_OPTS "root@$R630_02" "$@"; }
|
|
|
|
log_info() { echo -e "\033[0;34m[INFO]\033[0m $1"; }
|
|
log_ok() { echo -e "\033[0;32m[✓]\033[0m $1"; }
|
|
log_warn() { echo -e "\033[0;33m[⚠]\033[0m $1"; }
|
|
|
|
echo ""
|
|
echo "=== Fix storage: r630-01 data (72%) + r630-02 thin5 (84.6%) ==="
|
|
echo " dry-run=$DRY_RUN prune-only=$PRUNE_ONLY migrate-5000=${MIGRATE_5000_TARGET:-no}"
|
|
echo ""
|
|
|
|
# ----- Phase 1: Prune VMID 5000 (thin5) on r630-02 -----
|
|
echo "-------- Phase 1: Free space in VMID 5000 (thin5 on r630-02) --------"
|
|
if $DRY_RUN; then
|
|
echo " Would run: vmid5000-free-disk-and-logs.sh (journal vacuum, Docker prune, logrotate)"
|
|
else
|
|
if bash "${SCRIPT_DIR}/vmid5000-free-disk-and-logs.sh" 2>&1; then
|
|
log_ok "VMID 5000 prune done."
|
|
else
|
|
log_warn "VMID 5000 prune had warnings or failed (check above)."
|
|
fi
|
|
fi
|
|
echo ""
|
|
|
|
# ----- Phase 2: Prune logs in CTs on r630-01 (data) -----
|
|
echo "-------- Phase 2: Prune logs in CTs on r630-01 (local-lvm/data) --------"
|
|
VMIDS_R630_01=$(run_r630_01 "pct list 2>/dev/null | awk 'NR>1 && \$2==\"running\" {print \$1}'" 2>/dev/null || true)
|
|
if [[ -z "$VMIDS_R630_01" ]]; then
|
|
log_warn "Could not list running CTs on r630-01 (SSH failed?)."
|
|
else
|
|
for vmid in $VMIDS_R630_01; do
|
|
if $DRY_RUN; then
|
|
echo " Would prune in VMID $vmid: journalctl vacuum-time=3d vacuum-size=200M, logrotate -f"
|
|
continue
|
|
fi
|
|
run_r630_01 "pct exec $vmid -- journalctl --vacuum-time=3d 2>/dev/null; pct exec $vmid -- journalctl --vacuum-size=200M 2>/dev/null" 2>/dev/null || true
|
|
run_r630_01 "pct exec $vmid -- logrotate -f /etc/logrotate.conf 2>/dev/null" 2>/dev/null || true
|
|
run_r630_01 "pct exec $vmid -- sh -c ': > /var/log/syslog 2>/dev/null'" 2>/dev/null || true
|
|
log_ok "Pruned VMID $vmid"
|
|
done
|
|
fi
|
|
echo ""
|
|
|
|
# ----- Phase 3: Optional migrate 5000 from thin5 to thin2/thin6 -----
|
|
if [[ -n "$MIGRATE_5000_TARGET" ]]; then
|
|
echo "-------- Phase 3: Migrate VMID 5000 from thin5 to $MIGRATE_5000_TARGET --------"
|
|
if $DRY_RUN; then
|
|
echo " Would: stop 5000 -> vzdump to local -> destroy -> pct restore 5000 --storage $MIGRATE_5000_TARGET -> start"
|
|
echo " Target must be thin2 or thin6 (have free space on r630-02)."
|
|
else
|
|
run_r630_02 "pct stop 5000" 2>/dev/null || true
|
|
sleep 3
|
|
BACKUP_OUT=$(run_r630_02 "vzdump 5000 --storage local --compress gzip --mode stop --remove 0 2>&1" || true)
|
|
if echo "$BACKUP_OUT" | grep -qE "error|Error|failed|Failed"; then
|
|
log_warn "Backup failed: $BACKUP_OUT"
|
|
run_r630_02 "pct start 5000" 2>/dev/null || true
|
|
else
|
|
BACKUP_FILE=$(run_r630_02 "ls -t /var/lib/vz/dump/vzdump-lxc-5000-*.tar.gz 2>/dev/null | head -1" || true)
|
|
if [[ -z "$BACKUP_FILE" ]]; then
|
|
log_warn "Backup file not found."
|
|
run_r630_02 "pct start 5000" 2>/dev/null || true
|
|
else
|
|
run_r630_02 "pct destroy 5000 --force 2>/dev/null" || true
|
|
sleep 2
|
|
run_r630_02 "pct restore 5000 $BACKUP_FILE --storage $MIGRATE_5000_TARGET 2>&1" || { log_warn "Restore failed"; run_r630_02 "pct start 5000" 2>/dev/null; exit 1; }
|
|
run_r630_02 "rm -f $BACKUP_FILE"
|
|
run_r630_02 "pct start 5000"
|
|
log_ok "Migrated 5000 to $MIGRATE_5000_TARGET and started."
|
|
fi
|
|
fi
|
|
fi
|
|
echo ""
|
|
fi
|
|
|
|
echo "=== Re-check storage (run audit after a few minutes for thin pool reclaim) ==="
|
|
if ! $DRY_RUN; then
|
|
echo " r630-01: ssh root@$R630_01 'pvesm status | grep -E \"data|local-lvm\"'"
|
|
echo " r630-02: ssh root@$R630_02 'pvesm status | grep thin5'"
|
|
echo " Or: bash scripts/audit-proxmox-rpc-storage.sh"
|
|
fi
|
|
echo ""
|