Sankofa/scripts/complete-cleanup-and-deploy.sh

#!/bin/bash
set -e

# Check if resuming from step 12 (pod ready check)
RESUME_FROM_STEP_12=false
if [ "${1:-}" == "--resume-from-step12" ] || [ "${1:-}" == "--resume" ]; then
    RESUME_FROM_STEP_12=true
    echo "=========================================="
    echo "RESUMING FROM STEP 12 (POD READY CHECK)"
    echo "=========================================="
    echo ""
else
    echo "=========================================="
    echo "COMPLETE CLEANUP AND DEPLOY"
    echo "=========================================="
    echo ""
fi

IMAGE_NAME="crossplane-provider-proxmox"
NEW_TAG="v1.0.1-cookie-fix-final"
NEW_IMAGE_TAR="/tmp/crossplane-v1.0.1-fixed.tar"

# Detect if running in kind cluster
KIND_NODE=""
if docker ps --format '{{.Names}}' 2>/dev/null | grep -qE '^(sankofa-control-plane|.*-control-plane)$'; then
    KIND_NODE=$(docker ps --format '{{.Names}}' 2>/dev/null | grep -E '^(sankofa-control-plane|.*-control-plane)$' | head -1)
    echo "🔍 Detected kind cluster node: $KIND_NODE"
    echo ""
fi

# Function to run ctr commands - either in kind node or on host
ctr_cmd() {
    if [ -n "$KIND_NODE" ]; then
        docker exec "$KIND_NODE" ctr "$@"
    else
        sudo ctr "$@"
    fi
}

# Skip to step 12 if resuming
if [ "$RESUME_FROM_STEP_12" = true ]; then
    echo "Skipping steps 1-11, resuming from pod ready check..."
    echo ""
    SKIP_TO_STEP_12=true
else
    SKIP_TO_STEP_12=false
fi

# Find kubectl - check both current user and original user
KUBECTL=$(which kubectl 2>/dev/null || echo "")
if [ -z "$KUBECTL" ]; then
    ORIG_USER="${SUDO_USER:-${USER}}"
    # Try original user's PATH
    if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ]; then
        KUBECTL=$(sudo -u "$ORIG_USER" which kubectl 2>/dev/null || echo "")
    fi
    # Try common locations
    if [ -z "$KUBECTL" ]; then
        for path in /usr/local/bin/kubectl /usr/bin/kubectl ~/.local/bin/kubectl /home/$ORIG_USER/.local/bin/kubectl; do
            if [ -x "$path" ]; then
                KUBECTL="$path"
                break
            fi
        done
    fi
fi

if [ "$SKIP_TO_STEP_12" = false ]; then
    echo "Step 1: Deleting Kubernetes deployment and pods..."
    ORIG_USER="${SUDO_USER:-${USER}}"
    if [ -n "$KUBECTL" ]; then
    # Use original user's kubeconfig when running with sudo
    if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ] && [ "$EUID" -eq 0 ]; then
        KUBECONFIG="${KUBECONFIG:-/home/$ORIG_USER/.kube/config}"
        if [ -f "$KUBECONFIG" ]; then
            export KUBECONFIG
            sudo -u "$ORIG_USER" env KUBECONFIG="$KUBECONFIG" $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo "  (Deployment not found)"
            sudo -u "$ORIG_USER" env KUBECONFIG="$KUBECONFIG" $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo "  (Pods not found)"
        else
            sudo -u "$ORIG_USER" $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo "  (Deployment not found)"
            sudo -u "$ORIG_USER" $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo "  (Pods not found)"
        fi
    else
        $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo "  (Deployment not found)"
        $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo "  (Pods not found)"
    fi
    sleep 3
    echo "  ✅ Kubernetes resources deleted"
else
    echo "  ⚠️  kubectl not found, skipping Kubernetes cleanup"
    fi
    echo ""

    echo "Step 2: Removing ALL crossplane-provider-proxmox images from containerd..."
ALL_IMAGES=$(ctr_cmd -n k8s.io images ls -q | grep "${IMAGE_NAME}" || true)
if [ -n "$ALL_IMAGES" ]; then
    echo "$ALL_IMAGES" | while read -r img; do
        echo "  Removing: $img"
        ctr_cmd -n k8s.io images rm "$img" 2>/dev/null || true
    done
    echo "  ✅ All crossplane images removed from containerd"
else
    echo "  (No crossplane images found)"
fi
echo ""

echo "Step 3: Pruning unused containerd images (aggressive)..."
ctr_cmd -n k8s.io images prune --all 2>&1 | head -10 || echo "  (Prune completed)"
echo "  ✅ Containerd pruned"
echo ""

echo "Step 4: Removing old Docker images..."
docker rmi crossplane-provider-proxmox:latest crossplane-provider-proxmox:fixed crossplane-provider-proxmox:final-fix crossplane-provider-proxmox:v1.0.1-cookie-fix 2>/dev/null || echo "  (Some images not found, continuing...)"
docker rmi crossplane-provider-proxmox:${NEW_TAG} 2>/dev/null || echo "  (New image not in Docker yet, will import)"
echo "  ✅ Old Docker images removed"
echo ""

echo "Step 5: Cleaning Docker build cache (aggressive)..."
# Try with user's Docker (rootless)
ORIG_USER="${SUDO_USER:-${USER}}"
if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ]; then
    DOCKER_HOST_VAL="${DOCKER_HOST:-unix:///run/user/$(id -u $ORIG_USER)/docker.sock}"
    sudo -u "$ORIG_USER" env DOCKER_HOST=$DOCKER_HOST_VAL docker builder prune -af 2>&1 | tail -5 || echo "  (Cache clean completed)"
else
    docker builder prune -af 2>&1 | tail -5 || echo "  (Cache clean completed)"
fi
echo "  ✅ Docker build cache cleaned"
echo ""

echo "Step 6: Checking if new image tar exists, rebuilding if needed..."
if [ ! -f "$NEW_IMAGE_TAR" ]; then
    echo "  ⚠️  Image tar not found, rebuilding..."
    PROJECT_DIR="/home/intlc/projects/Sankofa/crossplane-provider-proxmox"
    if [ -d "$PROJECT_DIR" ]; then
        if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ]; then
            DOCKER_HOST_VAL="${DOCKER_HOST:-unix:///run/user/$(id -u $ORIG_USER)/docker.sock}"
            sudo -u "$ORIG_USER" env DOCKER_HOST=$DOCKER_HOST_VAL docker build --no-cache -t crossplane-provider-proxmox:${NEW_TAG} "$PROJECT_DIR" 2>&1 | tail -10
            sudo -u "$ORIG_USER" env DOCKER_HOST=$DOCKER_HOST_VAL docker save crossplane-provider-proxmox:${NEW_TAG} -o "$NEW_IMAGE_TAR"
        else
            docker build --no-cache -t crossplane-provider-proxmox:${NEW_TAG} "$PROJECT_DIR" 2>&1 | tail -10
            docker save crossplane-provider-proxmox:${NEW_TAG} -o "$NEW_IMAGE_TAR"
        fi
        echo "  ✅ Image rebuilt and saved"
    else
        echo "  ❌ ERROR: Project directory not found at $PROJECT_DIR"
        exit 1
    fi
else
    echo "  ✅ Image tar already exists"
    # Remove other old tar files but keep the new one
    find /tmp -name "crossplane-*.tar" ! -name "crossplane-v1.0.1-fixed.tar" -delete 2>/dev/null || true
fi
echo ""

echo "Step 7: Verifying new image tar exists..."
if [ ! -f "$NEW_IMAGE_TAR" ]; then
    echo "  ❌ ERROR: New image tar not found at $NEW_IMAGE_TAR"
    exit 1
fi
echo "  ✅ New image tar found: $(ls -lh $NEW_IMAGE_TAR | awk '{print $5}')"
echo ""

echo "Step 8: Importing NEW image to containerd..."
if [ -n "$KIND_NODE" ]; then
    echo "  Copying tar into kind node container (using stdin method)..."
    docker exec -i "$KIND_NODE" sh -c "cat > /tmp/$(basename $NEW_IMAGE_TAR)" < "$NEW_IMAGE_TAR"
    echo "  Importing into kind node's containerd..."
    ctr_cmd -n k8s.io images import "/tmp/$(basename $NEW_IMAGE_TAR)"
else
    ctr_cmd -n k8s.io images import "$NEW_IMAGE_TAR"
fi
echo "  ✅ New image imported"
echo ""

echo "Step 9: Verifying import and tagging correctly..."
# Get the imported image (it might have a different name from docker save)
IMPORTED_RAW=$(ctr_cmd -n k8s.io images ls -q | grep crossplane | head -1 || echo "")
if [ -z "$IMPORTED_RAW" ]; then
    echo "  ❌ ERROR: No crossplane image found after import!"
    echo "  All images in containerd:"
    ctr_cmd -n k8s.io images ls -q | head -5
    exit 1
fi

# Tag it with the exact name Kubernetes expects (fully qualified)
EXPECTED_IMAGE="docker.io/library/${IMAGE_NAME}:${NEW_TAG}"
echo "  Tagging imported image ($IMPORTED_RAW) as ${EXPECTED_IMAGE}..."
ctr_cmd -n k8s.io images tag "$IMPORTED_RAW" "${EXPECTED_IMAGE}" 2>/dev/null || {
    echo "  ⚠️  Tagging failed, checking if tag already exists..."
    # Check if the expected image already exists
    EXISTING=$(ctr_cmd -n k8s.io images ls -q | grep -E "(^|/)${EXPECTED_IMAGE}$" || echo "")
    if [ -n "$EXISTING" ]; then
        echo "  ✅ Expected image already exists: $EXISTING"
    else
        echo "  ❌ ERROR: Could not tag image. Trying to remove old tag and retag..."
        ctr_cmd -n k8s.io images rm "${EXPECTED_IMAGE}" 2>/dev/null || true
        ctr_cmd -n k8s.io images tag "$IMPORTED_RAW" "${EXPECTED_IMAGE}" || {
            echo "  ❌ ERROR: Failed to tag image as ${EXPECTED_IMAGE}"
            exit 1
        }
    fi
}

# Verify the expected image exists
IMPORTED=$(ctr_cmd -n k8s.io images ls -q | grep -E "(^|/)${EXPECTED_IMAGE}$" || true)
if [ -z "$IMPORTED" ]; then
    echo "  ❌ ERROR: Image ${EXPECTED_IMAGE} not found after tagging!"
    echo "  Available crossplane images:"
    ctr_cmd -n k8s.io images ls -q | grep crossplane || echo "  (none)"
    exit 1
fi
echo "  ✅ Image verified: $IMPORTED"
echo ""

echo "Step 10: Final containerd image list..."
ctr_cmd -n k8s.io images ls | grep crossplane-provider-proxmox || echo "  (No images found - this is wrong!)"
echo ""

fi  # Close the if [ "$SKIP_TO_STEP_12" = false ] block

if [ -n "$KUBECTL" ]; then
    # Set up kubectl command with proper user context using a function
    ORIG_USER="${SUDO_USER:-${USER}}"
    if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ] && [ "$EUID" -eq 0 ]; then
        KUBECONFIG="${KUBECONFIG:-/home/$ORIG_USER/.kube/config}"
        if [ -f "$KUBECONFIG" ]; then
            # Use a function to properly handle env vars and command
            kubectl_cmd() {
                sudo -u "$ORIG_USER" env KUBECONFIG="$KUBECONFIG" "$KUBECTL" "$@"
            }
        else
            kubectl_cmd() {
                sudo -u "$ORIG_USER" "$KUBECTL" "$@"
            }
        fi
    else
        kubectl_cmd() {
            "$KUBECTL" "$@"
        }
    fi
    # kubectl_cmd function is now defined and ready to use

    if [ "$SKIP_TO_STEP_12" = false ]; then
        echo "Step 11: Applying deployment with new image tag..."
        kubectl_cmd apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml
        echo "  ✅ Deployment applied"
        echo ""
    fi

    # Step 12: Wait for pod with retry loop and verbose feedback (always runs)
    echo "Step 12: Waiting for pod to be ready (with retry loop)..."
    
    # First, check if deployment exists, create it if not
    echo "  Checking if deployment exists..."
    DEPLOYMENT_EXISTS=$(kubectl_cmd get deployment crossplane-provider-proxmox -n crossplane-system 2>/dev/null && echo "yes" || echo "no")
    if [ "$DEPLOYMENT_EXISTS" = "no" ]; then
        echo "  ⚠️  Deployment not found! Creating it now..."
        kubectl_cmd apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml
        echo "  ✅ Deployment created, waiting 5s for pod to start..."
        sleep 5
    else
        echo "  ✅ Deployment exists"
    fi
    
    MAX_RETRIES=30
    RETRY_DELAY=5
    RETRY_COUNT=0
    POD_READY=false

    while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
        RETRY_COUNT=$((RETRY_COUNT + 1))
        echo ""
        echo "  Attempt $RETRY_COUNT/$MAX_RETRIES: Checking pod status..."
        
        # Get pod status - handle case where no pods exist yet
        POD_COUNT=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox --no-headers 2>/dev/null | wc -l || echo "0")
        if [ "$POD_COUNT" = "0" ] || [ -z "$POD_COUNT" ]; then
            POD_STATUS="NOT_FOUND"
            POD_READY_STATUS="Unknown"
            POD_CONTAINER_STATUS=""
            POD_IMAGE="Unknown"
        else
            POD_STATUS=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "NOT_FOUND")
            POD_READY_STATUS=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown")
            POD_CONTAINER_STATUS=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || echo "")
            POD_IMAGE=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].spec.containers[0].image}' 2>/dev/null || echo "Unknown")
        fi
        
        echo "    Pod Phase: $POD_STATUS"
        echo "    Ready Status: $POD_READY_STATUS"
        echo "    Container Image: $POD_IMAGE"
        
        if [ -n "$POD_CONTAINER_STATUS" ] && [ "$POD_CONTAINER_STATUS" != "null" ]; then
            echo "    Container Waiting Reason: $POD_CONTAINER_STATUS"
            
            # Check for specific errors
            if [ "$POD_CONTAINER_STATUS" = "ErrImageNeverPull" ]; then
                echo "    ⚠️  ERROR: Image not found in containerd!"
                echo "    Checking if image exists in containerd..."
                # Check for image with exact name Kubernetes expects (fully qualified)
                EXPECTED_IMAGE="docker.io/library/${IMAGE_NAME}:${NEW_TAG}"
                CONTAINERD_IMAGE=$(ctr_cmd -n k8s.io images ls -q | grep -E "(^|/)${EXPECTED_IMAGE}$" || echo "")
                
                if [ -z "$CONTAINERD_IMAGE" ]; then
                    echo "    ❌ Image ${EXPECTED_IMAGE} NOT found in containerd."
                    echo "    Checking for any crossplane images..."
                    ALL_CROSSPLANE=$(ctr_cmd -n k8s.io images ls -q | grep crossplane || echo "")
                    if [ -n "$ALL_CROSSPLANE" ]; then
                        echo "    Found other crossplane images:"
                        echo "$ALL_CROSSPLANE" | head -3 | sed 's/^/      /'
                        echo "    Re-importing and tagging correctly..."
                    fi
                    
                    if [ -f "$NEW_IMAGE_TAR" ]; then
                        echo "    Importing image from $NEW_IMAGE_TAR..."
                        if [ -n "$KIND_NODE" ]; then
                            echo "    Copying tar into kind node (using stdin method)..."
                            docker exec -i "$KIND_NODE" sh -c "cat > /tmp/$(basename $NEW_IMAGE_TAR)" < "$NEW_IMAGE_TAR"
                            ctr_cmd -n k8s.io images import "/tmp/$(basename $NEW_IMAGE_TAR)"
                        else
                            ctr_cmd -n k8s.io images import "$NEW_IMAGE_TAR"
                        fi
                        
                        # Get the imported image ID and tag it correctly
                        IMPORTED_ID=$(ctr_cmd -n k8s.io images ls -q | grep crossplane | head -1)
                        if [ -n "$IMPORTED_ID" ]; then
                            echo "    Tagging imported image as ${EXPECTED_IMAGE}..."
                            ctr_cmd -n k8s.io images tag "$IMPORTED_ID" "${EXPECTED_IMAGE}" 2>/dev/null || true
                            echo "    ✅ Image imported and tagged. Restarting pod..."
                        else
                            echo "    ⚠️  Could not find imported image, but continuing..."
                        fi
                        echo "    Restarting deployment once (no pod churn)..."
                        kubectl_cmd rollout restart deployment/crossplane-provider-proxmox -n crossplane-system
                        sleep 5
                    else
                        echo "    ❌ Image tar not found at $NEW_IMAGE_TAR"
                        echo "    Run script without --resume to rebuild image"
                        exit 1
                    fi
                else
                    echo "    ⚠️  Image exists in containerd but pod can't find it"
                    echo "    Image in containerd: $CONTAINERD_IMAGE"
                    echo "    Expected by pod: ${EXPECTED_IMAGE}"
                    echo "    Attempting to tag existing image with correct name..."
                    # Try to tag the existing image with the exact name Kubernetes expects
                    ctr_cmd -n k8s.io images tag "$CONTAINERD_IMAGE" "${EXPECTED_IMAGE}" 2>/dev/null || {
                        echo "    ⚠️  Tagging failed, trying to re-import..."
                        if [ -f "$NEW_IMAGE_TAR" ]; then
                            if [ -n "$KIND_NODE" ]; then
                                docker exec -i "$KIND_NODE" sh -c "cat > /tmp/$(basename $NEW_IMAGE_TAR)" < "$NEW_IMAGE_TAR"
                                ctr_cmd -n k8s.io images import "/tmp/$(basename $NEW_IMAGE_TAR)"
                            else
                                ctr_cmd -n k8s.io images import "$NEW_IMAGE_TAR"
                            fi
                            IMPORTED_ID=$(ctr_cmd -n k8s.io images ls -q | grep crossplane | head -1)
                            if [ -n "$IMPORTED_ID" ]; then
                                ctr_cmd -n k8s.io images tag "$IMPORTED_ID" "${EXPECTED_IMAGE}" 2>/dev/null || true
                            fi
                        fi
                    }
                    echo "    Restarting deployment once to refresh..."
                    kubectl_cmd rollout restart deployment/crossplane-provider-proxmox -n crossplane-system
                    sleep 5
                fi
            fi
        fi
        
        # Check if pod is ready
        if [ "$POD_READY_STATUS" = "True" ]; then
            echo "    ✅ Pod is READY!"
            POD_READY=true
            break
        fi
        
        # Show detailed pod info
        if [ "$POD_COUNT" != "0" ] && [ -n "$POD_COUNT" ]; then
            echo "    Pod details:"
            kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o wide 2>/dev/null | tail -1
        else
            echo "    No pods found yet - deployment may still be creating pod..."
        fi
        
        if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
            echo "    Waiting ${RETRY_DELAY}s before next check..."
            sleep $RETRY_DELAY
        fi
    done

    if [ "$POD_READY" = false ]; then
        echo ""
        echo "  ❌ Pod did not become ready after $MAX_RETRIES attempts"
        echo "  Final pod status:"
        if [ "$POD_COUNT" != "0" ] && [ -n "$POD_COUNT" ]; then
            kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o yaml 2>/dev/null | grep -A 10 "status:" | head -15 || echo "  (Could not get pod details)"
        else
            echo "  No pods found. Checking deployment status:"
            kubectl_cmd get deployment crossplane-provider-proxmox -n crossplane-system 2>/dev/null || echo "  Deployment also not found!"
        fi
        echo ""
        echo "  To retry from this point, run:"
        echo "    sudo $0 --resume-from-step12"
        exit 1
    fi

    echo ""
    echo "  ✅ Pod is ready!"
    echo ""

    echo "Step 13: Checking pod image..."
    POD_IMAGE=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].spec.containers[0].image}' || echo "")
    POD_IMAGE_ID=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].imageID}' || echo "")
    echo "  Pod image: $POD_IMAGE"
    echo "  Pod image ID: $POD_IMAGE_ID"
    if echo "$POD_IMAGE" | grep -q "${NEW_TAG}"; then
        echo "  ✅ Pod is using NEW image tag!"
    else
        echo "  ⚠️  Pod is NOT using new tag!"
    fi
    echo ""

    echo "Step 14: Waiting for logs to generate..."
    sleep 5
    echo ""

    echo "Step 15: Checking for [FIXED CODE] message..."
    if kubectl_cmd logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 2>&1 | grep -q '\[FIXED CODE\]'; then
        echo "  ✅ [FIXED CODE] message found! Fix is ACTIVE!"
        kubectl_cmd logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 | grep '\[FIXED CODE\]' | head -3
    else
        echo "  ❌ [FIXED CODE] message NOT found!"
        echo ""
        echo "  Recent logs:"
        kubectl_cmd logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=20 | grep -E "Cookie|Authorization|FIXED|DEBUG" | head -5 || echo "  (No relevant logs)"
    fi
fi

if [ -z "$KUBECTL" ]; then
    echo "Step 11: Skipping Kubernetes operations (kubectl not found)"
    echo ""
    echo "  Please manually run:"
    echo "    kubectl apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml"
    echo "    kubectl wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s"
    echo "    kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox | grep '\[FIXED CODE\]'"
fi

echo ""
echo "=========================================="
echo "CLEANUP AND DEPLOY COMPLETE"
echo "=========================================="