#!/bin/bash set -euo pipefail # Proxmox Review and Deployment Planning Script # This script connects to both Proxmox instances, reviews configurations, # checks status, and generates a deployment plan with task list SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" OUTPUT_DIR="${PROJECT_ROOT}/docs/proxmox-review" TIMESTAMP=$(date +%Y%m%d_%H%M%S) # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color log() { echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $*" } log_success() { echo -e "${GREEN}[$(date +'%Y-%m-%d %H:%M:%S')] ✅${NC} $*" } log_warning() { echo -e "${YELLOW}[$(date +'%Y-%m-%d %H:%M:%S')] ⚠️${NC} $*" } log_error() { echo -e "${RED}[$(date +'%Y-%m-%d %H:%M:%S')] ❌${NC} $*" } error() { log_error "$*" exit 1 } # Load environment variables load_env() { if [ -f "${PROJECT_ROOT}/.env" ]; then source "${PROJECT_ROOT}/.env" log "Loaded environment variables from .env" else log_warning ".env file not found, using defaults from ENV_EXAMPLES.md" fi # Set defaults if not provided PROXMOX_1_API_URL="${PROXMOX_1_API_URL:-https://192.168.11.10:8006}" PROXMOX_1_USER="${PROXMOX_1_USER:-root}" PROXMOX_1_PASS="${PROXMOX_1_PASS:-}" PROXMOX_1_API_TOKEN="${PROXMOX_1_API_TOKEN:-}" PROXMOX_1_INSECURE_SKIP_TLS_VERIFY="${PROXMOX_1_INSECURE_SKIP_TLS_VERIFY:-false}" PROXMOX_2_API_URL="${PROXMOX_2_API_URL:-https://192.168.11.11:8006}" PROXMOX_2_USER="${PROXMOX_2_USER:-root}" PROXMOX_2_PASS="${PROXMOX_2_PASS:-}" PROXMOX_2_API_TOKEN="${PROXMOX_2_API_TOKEN:-}" PROXMOX_2_INSECURE_SKIP_TLS_VERIFY="${PROXMOX_2_INSECURE_SKIP_TLS_VERIFY:-false}" } # Check prerequisites check_prerequisites() { log "Checking prerequisites..." if ! command -v curl &> /dev/null; then error "curl is required but not installed" fi if ! command -v jq &> /dev/null; then log_warning "jq is not installed. JSON parsing will be limited." JQ_AVAILABLE=false else JQ_AVAILABLE=true fi if [ -z "${PROXMOX_1_PASS}" ] && [ -z "${PROXMOX_1_API_TOKEN}" ]; then log_warning "PROXMOX_1_PASS or PROXMOX_1_API_TOKEN not set" fi if [ -z "${PROXMOX_2_PASS}" ] && [ -z "${PROXMOX_2_API_TOKEN}" ]; then log_warning "PROXMOX_2_PASS or PROXMOX_2_API_TOKEN not set" fi log_success "Prerequisites check completed" } # Create output directory create_output_dir() { mkdir -p "${OUTPUT_DIR}" log "Output directory: ${OUTPUT_DIR}" } # Authenticate to Proxmox API proxmox_auth() { local api_url=$1 local username=$2 local password=$3 local token=$4 local insecure=$5 local auth_url="${api_url}/api2/json/access/ticket" local curl_opts=() if [ "${insecure}" = "true" ]; then curl_opts+=("-k") fi if [ -n "${token}" ]; then # Token authentication echo "${token}" return 0 fi # Password authentication local response response=$(curl -s "${curl_opts[@]}" -X POST \ -d "username=${username}&password=${password}" \ "${auth_url}" 2>/dev/null || echo "") if [ -z "${response}" ]; then echo "" return 1 fi if [ "${JQ_AVAILABLE}" = "true" ]; then echo "${response}" | jq -r '.data.ticket // empty' else echo "${response}" | grep -o '"ticket":"[^"]*' | cut -d'"' -f4 fi } # Get Proxmox API ticket and CSRF token get_proxmox_ticket() { local api_url=$1 local username=$2 local password=$3 local token=$4 local insecure=$5 local auth_url="${api_url}/api2/json/access/ticket" local curl_opts=() if [ "${insecure}" = "true" ]; then curl_opts+=("-k") fi if [ -n "${token}" ]; then # For token auth, we need to split token into user:token echo "TOKEN:${token}" return 0 fi local response response=$(curl -s "${curl_opts[@]}" -X POST \ -d "username=${username}&password=${password}" \ "${auth_url}" 2>/dev/null || echo "") if [ -z "${response}" ]; then echo "" return 1 fi if [ "${JQ_AVAILABLE}" = "true" ]; then local ticket=$(echo "${response}" | jq -r '.data.ticket // empty') local csrf=$(echo "${response}" | jq -r '.data.CSRFPreventionToken // empty') echo "${ticket}:${csrf}" else local ticket=$(echo "${response}" | grep -o '"ticket":"[^"]*' | cut -d'"' -f4) local csrf=$(echo "${response}" | grep -o '"CSRFPreventionToken":"[^"]*' | cut -d'"' -f4) echo "${ticket}:${csrf}" fi } # Call Proxmox API proxmox_api_call() { local api_url=$1 local endpoint=$2 local ticket=$3 local csrf=$4 local insecure=$5 local curl_opts=(-s -f) local full_url="${api_url}/api2/json${endpoint}" if [ "${insecure}" = "true" ]; then curl_opts+=("-k") fi if [[ "${ticket}" == TOKEN:* ]]; then # Token authentication local token="${ticket#TOKEN:}" curl_opts+=(-H "Authorization: PVEAuthCookie=${token}") else # Ticket authentication curl_opts+=(-b "PVEAuthCookie=${ticket}") if [ -n "${csrf}" ]; then curl_opts+=(-H "CSRFPreventionToken: ${csrf}") fi fi curl "${curl_opts[@]}" "${full_url}" 2>/dev/null || echo "" } # Connect to Proxmox instance connect_proxmox() { local instance_num=$1 local api_url=$2 local username=$3 local password=$4 local token=$5 local insecure=$6 log "Connecting to Proxmox Instance ${instance_num} (${api_url})..." local auth_result auth_result=$(get_proxmox_ticket "${api_url}" "${username}" "${password}" "${token}" "${insecure}") if [ -z "${auth_result}" ]; then log_error "Failed to authenticate to Proxmox Instance ${instance_num}" return 1 fi local ticket="${auth_result%%:*}" local csrf="${auth_result##*:}" if [ "${ticket}" = "TOKEN" ]; then ticket="${auth_result}" csrf="" fi log_success "Authenticated to Proxmox Instance ${instance_num}" # Get cluster status log " Fetching cluster status..." local cluster_status cluster_status=$(proxmox_api_call "${api_url}" "/cluster/status" "${ticket}" "${csrf}" "${insecure}") # Get nodes log " Fetching nodes..." local nodes nodes=$(proxmox_api_call "${api_url}" "/nodes" "${ticket}" "${csrf}" "${insecure}") # Get version log " Fetching version..." local version version=$(proxmox_api_call "${api_url}" "/version" "${ticket}" "${csrf}" "${insecure}") # Get storage log " Fetching storage..." local storage storage=$(proxmox_api_call "${api_url}" "/storage" "${ticket}" "${csrf}" "${insecure}") # Save results local output_file="${OUTPUT_DIR}/proxmox-${instance_num}-status-${TIMESTAMP}.json" { echo "{" echo " \"instance\": ${instance_num}," echo " \"api_url\": \"${api_url}\"," echo " \"timestamp\": \"$(date -Iseconds)\"," echo " \"cluster_status\": ${cluster_status:-null}," echo " \"nodes\": ${nodes:-null}," echo " \"version\": ${version:-null}," echo " \"storage\": ${storage:-null}" echo "}" } > "${output_file}" log_success "Status saved to ${output_file}" # Display summary if [ "${JQ_AVAILABLE}" = "true" ]; then log " Cluster Summary:" if [ -n "${version}" ]; then local pve_version=$(echo "${version}" | jq -r '.data.version // "unknown"') log " Version: ${pve_version}" fi if [ -n "${nodes}" ]; then local node_count=$(echo "${nodes}" | jq '.data | length') log " Nodes: ${node_count}" echo "${nodes}" | jq -r '.data[]? | " - \(.node) (status: \(.status // "unknown"))"' || true fi fi echo "${ticket}:${csrf}" } # Review configurations review_configurations() { log "Reviewing configurations..." local config_file="${OUTPUT_DIR}/configuration-review-${TIMESTAMP}.md" { echo "# Proxmox Configuration Review" echo "" echo "Generated: $(date -Iseconds)" echo "" echo "## Environment Configuration" echo "" echo "### Proxmox Instance 1" echo "- API URL: ${PROXMOX_1_API_URL}" echo "- User: ${PROXMOX_1_USER}" echo "- Password: $([ -n "${PROXMOX_1_PASS}" ] && echo "***SET***" || echo "NOT SET")" echo "- API Token: $([ -n "${PROXMOX_1_API_TOKEN}" ] && echo "***SET***" || echo "NOT SET")" echo "- Insecure Skip TLS: ${PROXMOX_1_INSECURE_SKIP_TLS_VERIFY}" echo "" echo "### Proxmox Instance 2" echo "- API URL: ${PROXMOX_2_API_URL}" echo "- User: ${PROXMOX_2_USER}" echo "- Password: $([ -n "${PROXMOX_2_PASS}" ] && echo "***SET***" || echo "NOT SET")" echo "- API Token: $([ -n "${PROXMOX_2_API_TOKEN}" ] && echo "***SET***" || echo "NOT SET")" echo "- Insecure Skip TLS: ${PROXMOX_2_INSECURE_SKIP_TLS_VERIFY}" echo "" echo "## Crossplane Provider Configuration" echo "" echo "### Provider Config" if [ -f "${PROJECT_ROOT}/crossplane-provider-proxmox/examples/provider-config.yaml" ]; then echo "\`\`\`yaml" cat "${PROJECT_ROOT}/crossplane-provider-proxmox/examples/provider-config.yaml" echo "\`\`\`" else echo "Provider config file not found" fi echo "" echo "## Cloudflare Tunnel Configurations" echo "" for site_config in "${PROJECT_ROOT}/cloudflare/tunnel-configs/proxmox-site-"*.yaml; do if [ -f "${site_config}" ]; then echo "### $(basename "${site_config}")" echo "\`\`\`yaml" head -20 "${site_config}" echo "\`\`\`" echo "" fi done } > "${config_file}" log_success "Configuration review saved to ${config_file}" } # Generate deployment plan generate_deployment_plan() { log "Generating deployment plan..." local plan_file="${OUTPUT_DIR}/deployment-plan-${TIMESTAMP}.md" { echo "# Proxmox Deployment Plan" echo "" echo "Generated: $(date -Iseconds)" echo "" echo "## Current Status" echo "" echo "### Proxmox Instances" echo "- **Instance 1**: ${PROXMOX_1_API_URL}" echo "- **Instance 2**: ${PROXMOX_2_API_URL}" echo "" echo "### Configuration Sites" echo "- **us-east-1**: https://pve1.sankofa.nexus:8006 (node: pve1)" echo "- **eu-west-1**: https://pve4.sankofa.nexus:8006 (node: pve4)" echo "- **apac-1**: https://pve7.sankofa.nexus:8006 (node: pve7)" echo "" echo "## Deployment Phases" echo "" echo "### Phase 1: Connection and Validation" echo "" echo "1. **Verify Connectivity**" echo " - [ ] Test connection to Instance 1" echo " - [ ] Test connection to Instance 2" echo " - [ ] Verify API authentication" echo " - [ ] Check network connectivity" echo "" echo "2. **Status Review**" echo " - [ ] Review cluster status for both instances" echo " - [ ] Check node health and availability" echo " - [ ] Review storage configuration" echo " - [ ] Check network configuration" echo " - [ ] Review existing VMs and resources" echo "" echo "### Phase 2: Configuration Alignment" echo "" echo "1. **Site Mapping**" echo " - [ ] Map Instance 1 to appropriate site (us-east-1?)" echo " - [ ] Map Instance 2 to appropriate site (eu-west-1?)" echo " - [ ] Verify DNS/hostname configuration" echo " - [ ] Update provider-config.yaml with actual endpoints" echo "" echo "2. **Authentication Setup**" echo " - [ ] Create API tokens for Instance 1" echo " - [ ] Create API tokens for Instance 2" echo " - [ ] Update credentials in Kubernetes secrets" echo " - [ ] Test token authentication" echo "" echo "3. **Cloudflare Tunnel Configuration**" echo " - [ ] Review tunnel configs for all sites" echo " - [ ] Update hostnames in tunnel configs" echo " - [ ] Verify tunnel credentials" echo " - [ ] Test tunnel connectivity" echo "" echo "### Phase 3: Crossplane Provider Deployment" echo "" echo "1. **Provider Installation**" echo " - [ ] Build Crossplane provider" echo " - [ ] Deploy CRDs" echo " - [ ] Deploy provider controller" echo " - [ ] Verify provider health" echo "" echo "2. **Provider Configuration**" echo " - [ ] Create ProviderConfig resource" echo " - [ ] Configure credentials secret" echo " - [ ] Test provider connectivity to both instances" echo " - [ ] Verify site configuration" echo "" echo "### Phase 4: Infrastructure Deployment" echo "" echo "1. **Initial VM Deployment**" echo " - [ ] Deploy test VM on Instance 1" echo " - [ ] Deploy test VM on Instance 2" echo " - [ ] Verify VM creation via Crossplane" echo " - [ ] Test VM lifecycle operations" echo "" echo "2. **Monitoring Setup**" echo " - [ ] Deploy Prometheus exporters" echo " - [ ] Configure Grafana dashboards" echo " - [ ] Set up alerts" echo " - [ ] Verify metrics collection" echo "" echo "3. **Backup and Recovery**" echo " - [ ] Configure backup schedules" echo " - [ ] Test backup procedures" echo " - [ ] Test recovery procedures" echo "" echo "### Phase 5: Production Readiness" echo "" echo "1. **Security Hardening**" echo " - [ ] Review and update firewall rules" echo " - [ ] Enable TLS certificate validation" echo " - [ ] Rotate API tokens" echo " - [ ] Review access controls" echo "" echo "2. **Documentation**" echo " - [ ] Document deployment procedures" echo " - [ ] Create runbooks" echo " - [ ] Update architecture diagrams" echo "" echo "3. **Testing and Validation**" echo " - [ ] End-to-end testing" echo " - [ ] Load testing" echo " - [ ] Disaster recovery testing" echo " - [ ] Performance validation" echo "" } > "${plan_file}" log_success "Deployment plan saved to ${plan_file}" } # Generate task list generate_task_list() { log "Generating detailed task list..." local task_file="${OUTPUT_DIR}/task-list-${TIMESTAMP}.md" { echo "# Proxmox Deployment Task List" echo "" echo "Generated: $(date -Iseconds)" echo "" echo "## Immediate Tasks (Priority: High)" echo "" echo "### Connection and Authentication" echo "" echo "- [ ] **TASK-001**: Verify network connectivity to ${PROXMOX_1_API_URL}" echo " - Command: \`curl -k ${PROXMOX_1_API_URL}/api2/json/version\`" echo " - Expected: JSON response with Proxmox version" echo "" echo "- [ ] **TASK-002**: Verify network connectivity to ${PROXMOX_2_API_URL}" echo " - Command: \`curl -k ${PROXMOX_2_API_URL}/api2/json/version\`" echo " - Expected: JSON response with Proxmox version" echo "" echo "- [ ] **TASK-003**: Test authentication to Instance 1" echo " - Verify credentials or create API token" echo " - Test API access" echo "" echo "- [ ] **TASK-004**: Test authentication to Instance 2" echo " - Verify credentials or create API token" echo " - Test API access" echo "" echo "### Configuration Review" echo "" echo "- [ ] **TASK-005**: Review current provider-config.yaml" echo " - File: \`crossplane-provider-proxmox/examples/provider-config.yaml\`" echo " - Verify endpoints match actual Proxmox instances" echo " - Update if necessary" echo "" echo "- [ ] **TASK-006**: Review Cloudflare tunnel configurations" echo " - Files: \`cloudflare/tunnel-configs/proxmox-site-*.yaml\`" echo " - Verify hostnames and endpoints" echo " - Update domain names if needed" echo "" echo "- [ ] **TASK-007**: Map Proxmox instances to sites" echo " - Determine which instance corresponds to which site" echo " - Update documentation" echo "" echo "## Short-term Tasks (Priority: Medium)" echo "" echo "### Crossplane Provider" echo "" echo "- [ ] **TASK-008**: Complete Proxmox API client implementation" echo " - File: \`crossplane-provider-proxmox/pkg/proxmox/client.go\`" echo " - Implement actual API calls (currently TODOs)" echo " - Add proper HTTP client with authentication" echo "" echo "- [ ] **TASK-009**: Build and test Crossplane provider" echo " - Run: \`cd crossplane-provider-proxmox && make build\`" echo " - Test provider locally" echo "" echo "- [ ] **TASK-010**: Deploy Crossplane provider to Kubernetes" echo " - Apply CRDs: \`kubectl apply -f config/crd/bases/\`" echo " - Deploy provider: \`kubectl apply -f config/provider.yaml\`" echo "" echo "- [ ] **TASK-011**: Create ProviderConfig resource" echo " - Update \`examples/provider-config.yaml\` with actual values" echo " - Create credentials secret" echo " - Apply ProviderConfig" echo "" echo "### Infrastructure Setup" echo "" echo "- [ ] **TASK-012**: Deploy Prometheus exporters to Proxmox nodes" echo " - Use script: \`scripts/setup-proxmox-agents.sh\`" echo " - Configure metrics collection" echo "" echo "- [ ] **TASK-013**: Configure Cloudflare tunnels" echo " - Deploy tunnel configs to Proxmox nodes" echo " - Verify tunnel connectivity" echo " - Test access via Cloudflare" echo "" echo "- [ ] **TASK-014**: Set up monitoring dashboards" echo " - Import Grafana dashboards" echo " - Configure alerts" echo "" echo "## Long-term Tasks (Priority: Low)" echo "" echo "### Testing and Validation" echo "" echo "- [ ] **TASK-015**: Deploy test VMs via Crossplane" echo " - Create test VM on Instance 1" echo " - Create test VM on Instance 2" echo " - Verify VM lifecycle operations" echo "" echo "- [ ] **TASK-016**: End-to-end testing" echo " - Test VM creation from portal" echo " - Test VM management operations" echo " - Test multi-site deployments" echo "" echo "- [ ] **TASK-017**: Performance testing" echo " - Load test API endpoints" echo " - Test concurrent VM operations" echo " - Measure response times" echo "" echo "### Documentation and Operations" echo "" echo "- [ ] **TASK-018**: Create operational runbooks" echo " - VM provisioning procedures" echo " - Troubleshooting guides" echo " - Disaster recovery procedures" echo "" echo "- [ ] **TASK-019**: Set up backup procedures" echo " - Configure automated backups" echo " - Test backup and restore" echo "" echo "- [ ] **TASK-020**: Security audit" echo " - Review access controls" echo " - Enable TLS validation" echo " - Rotate credentials" echo "" } > "${task_file}" log_success "Task list saved to ${task_file}" } # Main execution main() { log "Starting Proxmox Review and Deployment Planning..." log "==================================================" load_env check_prerequisites create_output_dir log "" log "=== Phase 1: Connecting to Proxmox Instances ===" local instance1_auth="" local instance2_auth="" # Connect to Instance 1 if instance1_auth=$(connect_proxmox 1 \ "${PROXMOX_1_API_URL}" \ "${PROXMOX_1_USER}" \ "${PROXMOX_1_PASS}" \ "${PROXMOX_1_API_TOKEN}" \ "${PROXMOX_1_INSECURE_SKIP_TLS_VERIFY}"); then log_success "Successfully connected to Proxmox Instance 1" else log_error "Failed to connect to Proxmox Instance 1" fi log "" # Connect to Instance 2 if instance2_auth=$(connect_proxmox 2 \ "${PROXMOX_2_API_URL}" \ "${PROXMOX_2_USER}" \ "${PROXMOX_2_PASS}" \ "${PROXMOX_2_API_TOKEN}" \ "${PROXMOX_2_INSECURE_SKIP_TLS_VERIFY}"); then log_success "Successfully connected to Proxmox Instance 2" else log_error "Failed to connect to Proxmox Instance 2" fi log "" log "=== Phase 2: Reviewing Configurations ===" review_configurations log "" log "=== Phase 3: Generating Deployment Plan ===" generate_deployment_plan log "" log "=== Phase 4: Generating Task List ===" generate_task_list log "" log "==================================================" log_success "Review and planning completed!" log "" log "Output files:" log " - Configuration Review: ${OUTPUT_DIR}/configuration-review-${TIMESTAMP}.md" log " - Deployment Plan: ${OUTPUT_DIR}/deployment-plan-${TIMESTAMP}.md" log " - Task List: ${OUTPUT_DIR}/task-list-${TIMESTAMP}.md" log " - Status JSONs: ${OUTPUT_DIR}/proxmox-*-status-${TIMESTAMP}.json" log "" } main "$@"