proxmox/scripts/cloudflare-tunnels/scripts/monitor-tunnels.sh.bak

#!/usr/bin/env bash
# Continuous health monitoring for Cloudflare tunnels

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

# Colors
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
log_success() { echo -e "${GREEN}[✓]${NC} $1"; }
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
log_error() { echo -e "${RED}[ERROR]${NC} $1"; }

# Configuration
PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"
VMID="${VMID:-102}"
TUNNELS=("ml110" "r630-01" "r630-02")
CHECK_INTERVAL="${CHECK_INTERVAL:-60}"  # seconds
LOG_FILE="${LOG_FILE:-/var/log/cloudflared-monitor.log}"
ALERT_SCRIPT="$SCRIPT_DIR/alert-tunnel-failure.sh"

# Check if running as daemon
DAEMON=false
if [[ "${1:-}" == "--daemon" ]]; then
    DAEMON=true
fi

# Check if running on Proxmox host
if command -v pct &> /dev/null; then
    RUN_LOCAL=true
else
    RUN_LOCAL=false
fi

exec_in_container() {
    local cmd="$1"
    if [ "$RUN_LOCAL" = true ]; then
        pct exec "$VMID" -- bash -c "$cmd"
    else
        ssh "root@${PROXMOX_HOST}" "pct exec $VMID -- bash -c '$cmd'"
    fi
}

# Check tunnel health
check_tunnel() {
    local tunnel="$1"
    local service="cloudflared-${tunnel}"
    
    # Check if service is active
    if exec_in_container "systemctl is-active --quiet $service"; then
        # Check if service is actually running (not just enabled)
        if exec_in_container "systemctl is-active $service 2>/dev/null | grep -q active"; then
            return 0
        fi
    fi
    return 1
}

# Check tunnel connectivity
check_connectivity() {
    local tunnel="$1"
    local domain=""
    
    case "$tunnel" in
        ml110) domain="ml110-01.d-bis.org" ;;
        r630-01) domain="r630-01.d-bis.org" ;;
        r630-02) domain="r630-02.d-bis.org" ;;
        *) return 1 ;;
    esac
    
    # Try to connect via HTTPS (should get Cloudflare Access page or redirect)
    if curl -s -o /dev/null -w "%{http_code}" --max-time 10 "https://${domain}" | grep -qE "^(200|302|403|401)"; then
        return 0
    fi
    return 1
}

# Log message
log_message() {
    local level="$1"
    local message="$2"
    local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
    
    if [ "$DAEMON" = true ]; then
        echo "[$timestamp] [$level] $message" >> "$LOG_FILE"
    else
        case "$level" in
            INFO) log_info "$message" ;;
            SUCCESS) log_success "$message" ;;
            WARN) log_warn "$message" ;;
            ERROR) log_error "$message" ;;
        esac
    fi
}

# Monitor loop
monitor_loop() {
    local failed_tunnels=()
    
    while true; do
        for tunnel in "${TUNNELS[@]}"; do
            # Check service status
            if ! check_tunnel "$tunnel"; then
                log_message "ERROR" "Tunnel $tunnel service is not running"
                
                # Alert if not already alerted
                if [[ ! " ${failed_tunnels[@]} " =~ " ${tunnel} " ]]; then
                    failed_tunnels+=("$tunnel")
                    if [ -f "$ALERT_SCRIPT" ]; then
                        "$ALERT_SCRIPT" "$tunnel" "service_down"
                    fi
                    
                    # Attempt restart
                    log_message "INFO" "Attempting to restart tunnel $tunnel"
                    exec_in_container "systemctl restart cloudflared-${tunnel}.service" || true
                    sleep 5
                fi
            else
                # Service is running, check connectivity
                if ! check_connectivity "$tunnel"; then
                    log_message "WARN" "Tunnel $tunnel service is running but connectivity check failed"
                else
                    log_message "SUCCESS" "Tunnel $tunnel is healthy"
                    
                    # Remove from failed list if it was there
                    failed_tunnels=("${failed_tunnels[@]/$tunnel}")
                fi
            fi
        done
        
        # Sleep before next check
        sleep "$CHECK_INTERVAL"
    done
}

# Main
main() {
    log_message "INFO" "Starting tunnel monitoring"
    log_message "INFO" "Monitoring tunnels: ${TUNNELS[*]}"
    log_message "INFO" "Check interval: ${CHECK_INTERVAL}s"
    
    if [ "$DAEMON" = true ]; then
        log_message "INFO" "Running in daemon mode. Logs: $LOG_FILE"
        # Redirect output to log file
        monitor_loop >> "$LOG_FILE" 2>&1 &
        echo $! > /tmp/cloudflared-monitor.pid
        log_message "INFO" "Monitor started with PID: $(cat /tmp/cloudflared-monitor.pid)"
    else
        log_message "INFO" "Running in foreground mode. Press Ctrl+C to stop."
        monitor_loop
    fi
}

# Handle signals
trap 'log_message "INFO" "Monitor stopped"; exit 0' SIGINT SIGTERM

main