#!/bin/bash set -e # Check if resuming from step 12 (pod ready check) RESUME_FROM_STEP_12=false if [ "${1:-}" == "--resume-from-step12" ] || [ "${1:-}" == "--resume" ]; then RESUME_FROM_STEP_12=true echo "==========================================" echo "RESUMING FROM STEP 12 (POD READY CHECK)" echo "==========================================" echo "" else echo "==========================================" echo "COMPLETE CLEANUP AND DEPLOY" echo "==========================================" echo "" fi IMAGE_NAME="crossplane-provider-proxmox" NEW_TAG="v1.0.1-cookie-fix-final" NEW_IMAGE_TAR="/tmp/crossplane-v1.0.1-fixed.tar" # Detect if running in kind cluster KIND_NODE="" if docker ps --format '{{.Names}}' 2>/dev/null | grep -qE '^(sankofa-control-plane|.*-control-plane)$'; then KIND_NODE=$(docker ps --format '{{.Names}}' 2>/dev/null | grep -E '^(sankofa-control-plane|.*-control-plane)$' | head -1) echo "🔍 Detected kind cluster node: $KIND_NODE" echo "" fi # Function to run ctr commands - either in kind node or on host ctr_cmd() { if [ -n "$KIND_NODE" ]; then docker exec "$KIND_NODE" ctr "$@" else sudo ctr "$@" fi } # Skip to step 12 if resuming if [ "$RESUME_FROM_STEP_12" = true ]; then echo "Skipping steps 1-11, resuming from pod ready check..." echo "" SKIP_TO_STEP_12=true else SKIP_TO_STEP_12=false fi # Find kubectl - check both current user and original user KUBECTL=$(which kubectl 2>/dev/null || echo "") if [ -z "$KUBECTL" ]; then ORIG_USER="${SUDO_USER:-${USER}}" # Try original user's PATH if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ]; then KUBECTL=$(sudo -u "$ORIG_USER" which kubectl 2>/dev/null || echo "") fi # Try common locations if [ -z "$KUBECTL" ]; then for path in /usr/local/bin/kubectl /usr/bin/kubectl ~/.local/bin/kubectl /home/$ORIG_USER/.local/bin/kubectl; do if [ -x "$path" ]; then KUBECTL="$path" break fi done fi fi if [ "$SKIP_TO_STEP_12" = false ]; then echo "Step 1: Deleting Kubernetes deployment and pods..." ORIG_USER="${SUDO_USER:-${USER}}" if [ -n "$KUBECTL" ]; then # Use original user's kubeconfig when running with sudo if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ] && [ "$EUID" -eq 0 ]; then KUBECONFIG="${KUBECONFIG:-/home/$ORIG_USER/.kube/config}" if [ -f "$KUBECONFIG" ]; then export KUBECONFIG sudo -u "$ORIG_USER" env KUBECONFIG="$KUBECONFIG" $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo " (Deployment not found)" sudo -u "$ORIG_USER" env KUBECONFIG="$KUBECONFIG" $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo " (Pods not found)" else sudo -u "$ORIG_USER" $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo " (Deployment not found)" sudo -u "$ORIG_USER" $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo " (Pods not found)" fi else $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo " (Deployment not found)" $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo " (Pods not found)" fi sleep 3 echo " ✅ Kubernetes resources deleted" else echo " ⚠️ kubectl not found, skipping Kubernetes cleanup" fi echo "" echo "Step 2: Removing ALL crossplane-provider-proxmox images from containerd..." ALL_IMAGES=$(ctr_cmd -n k8s.io images ls -q | grep "${IMAGE_NAME}" || true) if [ -n "$ALL_IMAGES" ]; then echo "$ALL_IMAGES" | while read -r img; do echo " Removing: $img" ctr_cmd -n k8s.io images rm "$img" 2>/dev/null || true done echo " ✅ All crossplane images removed from containerd" else echo " (No crossplane images found)" fi echo "" echo "Step 3: Pruning unused containerd images (aggressive)..." ctr_cmd -n k8s.io images prune --all 2>&1 | head -10 || echo " (Prune completed)" echo " ✅ Containerd pruned" echo "" echo "Step 4: Removing old Docker images..." docker rmi crossplane-provider-proxmox:latest crossplane-provider-proxmox:fixed crossplane-provider-proxmox:final-fix crossplane-provider-proxmox:v1.0.1-cookie-fix 2>/dev/null || echo " (Some images not found, continuing...)" docker rmi crossplane-provider-proxmox:${NEW_TAG} 2>/dev/null || echo " (New image not in Docker yet, will import)" echo " ✅ Old Docker images removed" echo "" echo "Step 5: Cleaning Docker build cache (aggressive)..." # Try with user's Docker (rootless) ORIG_USER="${SUDO_USER:-${USER}}" if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ]; then DOCKER_HOST_VAL="${DOCKER_HOST:-unix:///run/user/$(id -u $ORIG_USER)/docker.sock}" sudo -u "$ORIG_USER" env DOCKER_HOST=$DOCKER_HOST_VAL docker builder prune -af 2>&1 | tail -5 || echo " (Cache clean completed)" else docker builder prune -af 2>&1 | tail -5 || echo " (Cache clean completed)" fi echo " ✅ Docker build cache cleaned" echo "" echo "Step 6: Checking if new image tar exists, rebuilding if needed..." if [ ! -f "$NEW_IMAGE_TAR" ]; then echo " ⚠️ Image tar not found, rebuilding..." PROJECT_DIR="/home/intlc/projects/Sankofa/crossplane-provider-proxmox" if [ -d "$PROJECT_DIR" ]; then if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ]; then DOCKER_HOST_VAL="${DOCKER_HOST:-unix:///run/user/$(id -u $ORIG_USER)/docker.sock}" sudo -u "$ORIG_USER" env DOCKER_HOST=$DOCKER_HOST_VAL docker build --no-cache -t crossplane-provider-proxmox:${NEW_TAG} "$PROJECT_DIR" 2>&1 | tail -10 sudo -u "$ORIG_USER" env DOCKER_HOST=$DOCKER_HOST_VAL docker save crossplane-provider-proxmox:${NEW_TAG} -o "$NEW_IMAGE_TAR" else docker build --no-cache -t crossplane-provider-proxmox:${NEW_TAG} "$PROJECT_DIR" 2>&1 | tail -10 docker save crossplane-provider-proxmox:${NEW_TAG} -o "$NEW_IMAGE_TAR" fi echo " ✅ Image rebuilt and saved" else echo " ❌ ERROR: Project directory not found at $PROJECT_DIR" exit 1 fi else echo " ✅ Image tar already exists" # Remove other old tar files but keep the new one find /tmp -name "crossplane-*.tar" ! -name "crossplane-v1.0.1-fixed.tar" -delete 2>/dev/null || true fi echo "" echo "Step 7: Verifying new image tar exists..." if [ ! -f "$NEW_IMAGE_TAR" ]; then echo " ❌ ERROR: New image tar not found at $NEW_IMAGE_TAR" exit 1 fi echo " ✅ New image tar found: $(ls -lh $NEW_IMAGE_TAR | awk '{print $5}')" echo "" echo "Step 8: Importing NEW image to containerd..." if [ -n "$KIND_NODE" ]; then echo " Copying tar into kind node container (using stdin method)..." docker exec -i "$KIND_NODE" sh -c "cat > /tmp/$(basename $NEW_IMAGE_TAR)" < "$NEW_IMAGE_TAR" echo " Importing into kind node's containerd..." ctr_cmd -n k8s.io images import "/tmp/$(basename $NEW_IMAGE_TAR)" else ctr_cmd -n k8s.io images import "$NEW_IMAGE_TAR" fi echo " ✅ New image imported" echo "" echo "Step 9: Verifying import and tagging correctly..." # Get the imported image (it might have a different name from docker save) IMPORTED_RAW=$(ctr_cmd -n k8s.io images ls -q | grep crossplane | head -1 || echo "") if [ -z "$IMPORTED_RAW" ]; then echo " ❌ ERROR: No crossplane image found after import!" echo " All images in containerd:" ctr_cmd -n k8s.io images ls -q | head -5 exit 1 fi # Tag it with the exact name Kubernetes expects (fully qualified) EXPECTED_IMAGE="docker.io/library/${IMAGE_NAME}:${NEW_TAG}" echo " Tagging imported image ($IMPORTED_RAW) as ${EXPECTED_IMAGE}..." ctr_cmd -n k8s.io images tag "$IMPORTED_RAW" "${EXPECTED_IMAGE}" 2>/dev/null || { echo " ⚠️ Tagging failed, checking if tag already exists..." # Check if the expected image already exists EXISTING=$(ctr_cmd -n k8s.io images ls -q | grep -E "(^|/)${EXPECTED_IMAGE}$" || echo "") if [ -n "$EXISTING" ]; then echo " ✅ Expected image already exists: $EXISTING" else echo " ❌ ERROR: Could not tag image. Trying to remove old tag and retag..." ctr_cmd -n k8s.io images rm "${EXPECTED_IMAGE}" 2>/dev/null || true ctr_cmd -n k8s.io images tag "$IMPORTED_RAW" "${EXPECTED_IMAGE}" || { echo " ❌ ERROR: Failed to tag image as ${EXPECTED_IMAGE}" exit 1 } fi } # Verify the expected image exists IMPORTED=$(ctr_cmd -n k8s.io images ls -q | grep -E "(^|/)${EXPECTED_IMAGE}$" || true) if [ -z "$IMPORTED" ]; then echo " ❌ ERROR: Image ${EXPECTED_IMAGE} not found after tagging!" echo " Available crossplane images:" ctr_cmd -n k8s.io images ls -q | grep crossplane || echo " (none)" exit 1 fi echo " ✅ Image verified: $IMPORTED" echo "" echo "Step 10: Final containerd image list..." ctr_cmd -n k8s.io images ls | grep crossplane-provider-proxmox || echo " (No images found - this is wrong!)" echo "" fi # Close the if [ "$SKIP_TO_STEP_12" = false ] block if [ -n "$KUBECTL" ]; then # Set up kubectl command with proper user context using a function ORIG_USER="${SUDO_USER:-${USER}}" if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ] && [ "$EUID" -eq 0 ]; then KUBECONFIG="${KUBECONFIG:-/home/$ORIG_USER/.kube/config}" if [ -f "$KUBECONFIG" ]; then # Use a function to properly handle env vars and command kubectl_cmd() { sudo -u "$ORIG_USER" env KUBECONFIG="$KUBECONFIG" "$KUBECTL" "$@" } else kubectl_cmd() { sudo -u "$ORIG_USER" "$KUBECTL" "$@" } fi else kubectl_cmd() { "$KUBECTL" "$@" } fi # kubectl_cmd function is now defined and ready to use if [ "$SKIP_TO_STEP_12" = false ]; then echo "Step 11: Applying deployment with new image tag..." kubectl_cmd apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml echo " ✅ Deployment applied" echo "" fi # Step 12: Wait for pod with retry loop and verbose feedback (always runs) echo "Step 12: Waiting for pod to be ready (with retry loop)..." # First, check if deployment exists, create it if not echo " Checking if deployment exists..." DEPLOYMENT_EXISTS=$(kubectl_cmd get deployment crossplane-provider-proxmox -n crossplane-system 2>/dev/null && echo "yes" || echo "no") if [ "$DEPLOYMENT_EXISTS" = "no" ]; then echo " ⚠️ Deployment not found! Creating it now..." kubectl_cmd apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml echo " ✅ Deployment created, waiting 5s for pod to start..." sleep 5 else echo " ✅ Deployment exists" fi MAX_RETRIES=30 RETRY_DELAY=5 RETRY_COUNT=0 POD_READY=false while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do RETRY_COUNT=$((RETRY_COUNT + 1)) echo "" echo " Attempt $RETRY_COUNT/$MAX_RETRIES: Checking pod status..." # Get pod status - handle case where no pods exist yet POD_COUNT=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox --no-headers 2>/dev/null | wc -l || echo "0") if [ "$POD_COUNT" = "0" ] || [ -z "$POD_COUNT" ]; then POD_STATUS="NOT_FOUND" POD_READY_STATUS="Unknown" POD_CONTAINER_STATUS="" POD_IMAGE="Unknown" else POD_STATUS=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "NOT_FOUND") POD_READY_STATUS=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown") POD_CONTAINER_STATUS=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || echo "") POD_IMAGE=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].spec.containers[0].image}' 2>/dev/null || echo "Unknown") fi echo " Pod Phase: $POD_STATUS" echo " Ready Status: $POD_READY_STATUS" echo " Container Image: $POD_IMAGE" if [ -n "$POD_CONTAINER_STATUS" ] && [ "$POD_CONTAINER_STATUS" != "null" ]; then echo " Container Waiting Reason: $POD_CONTAINER_STATUS" # Check for specific errors if [ "$POD_CONTAINER_STATUS" = "ErrImageNeverPull" ]; then echo " ⚠️ ERROR: Image not found in containerd!" echo " Checking if image exists in containerd..." # Check for image with exact name Kubernetes expects (fully qualified) EXPECTED_IMAGE="docker.io/library/${IMAGE_NAME}:${NEW_TAG}" CONTAINERD_IMAGE=$(ctr_cmd -n k8s.io images ls -q | grep -E "(^|/)${EXPECTED_IMAGE}$" || echo "") if [ -z "$CONTAINERD_IMAGE" ]; then echo " ❌ Image ${EXPECTED_IMAGE} NOT found in containerd." echo " Checking for any crossplane images..." ALL_CROSSPLANE=$(ctr_cmd -n k8s.io images ls -q | grep crossplane || echo "") if [ -n "$ALL_CROSSPLANE" ]; then echo " Found other crossplane images:" echo "$ALL_CROSSPLANE" | head -3 | sed 's/^/ /' echo " Re-importing and tagging correctly..." fi if [ -f "$NEW_IMAGE_TAR" ]; then echo " Importing image from $NEW_IMAGE_TAR..." if [ -n "$KIND_NODE" ]; then echo " Copying tar into kind node (using stdin method)..." docker exec -i "$KIND_NODE" sh -c "cat > /tmp/$(basename $NEW_IMAGE_TAR)" < "$NEW_IMAGE_TAR" ctr_cmd -n k8s.io images import "/tmp/$(basename $NEW_IMAGE_TAR)" else ctr_cmd -n k8s.io images import "$NEW_IMAGE_TAR" fi # Get the imported image ID and tag it correctly IMPORTED_ID=$(ctr_cmd -n k8s.io images ls -q | grep crossplane | head -1) if [ -n "$IMPORTED_ID" ]; then echo " Tagging imported image as ${EXPECTED_IMAGE}..." ctr_cmd -n k8s.io images tag "$IMPORTED_ID" "${EXPECTED_IMAGE}" 2>/dev/null || true echo " ✅ Image imported and tagged. Restarting pod..." else echo " ⚠️ Could not find imported image, but continuing..." fi echo " Restarting deployment once (no pod churn)..." kubectl_cmd rollout restart deployment/crossplane-provider-proxmox -n crossplane-system sleep 5 else echo " ❌ Image tar not found at $NEW_IMAGE_TAR" echo " Run script without --resume to rebuild image" exit 1 fi else echo " ⚠️ Image exists in containerd but pod can't find it" echo " Image in containerd: $CONTAINERD_IMAGE" echo " Expected by pod: ${EXPECTED_IMAGE}" echo " Attempting to tag existing image with correct name..." # Try to tag the existing image with the exact name Kubernetes expects ctr_cmd -n k8s.io images tag "$CONTAINERD_IMAGE" "${EXPECTED_IMAGE}" 2>/dev/null || { echo " ⚠️ Tagging failed, trying to re-import..." if [ -f "$NEW_IMAGE_TAR" ]; then if [ -n "$KIND_NODE" ]; then docker exec -i "$KIND_NODE" sh -c "cat > /tmp/$(basename $NEW_IMAGE_TAR)" < "$NEW_IMAGE_TAR" ctr_cmd -n k8s.io images import "/tmp/$(basename $NEW_IMAGE_TAR)" else ctr_cmd -n k8s.io images import "$NEW_IMAGE_TAR" fi IMPORTED_ID=$(ctr_cmd -n k8s.io images ls -q | grep crossplane | head -1) if [ -n "$IMPORTED_ID" ]; then ctr_cmd -n k8s.io images tag "$IMPORTED_ID" "${EXPECTED_IMAGE}" 2>/dev/null || true fi fi } echo " Restarting deployment once to refresh..." kubectl_cmd rollout restart deployment/crossplane-provider-proxmox -n crossplane-system sleep 5 fi fi fi # Check if pod is ready if [ "$POD_READY_STATUS" = "True" ]; then echo " ✅ Pod is READY!" POD_READY=true break fi # Show detailed pod info if [ "$POD_COUNT" != "0" ] && [ -n "$POD_COUNT" ]; then echo " Pod details:" kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o wide 2>/dev/null | tail -1 else echo " No pods found yet - deployment may still be creating pod..." fi if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then echo " Waiting ${RETRY_DELAY}s before next check..." sleep $RETRY_DELAY fi done if [ "$POD_READY" = false ]; then echo "" echo " ❌ Pod did not become ready after $MAX_RETRIES attempts" echo " Final pod status:" if [ "$POD_COUNT" != "0" ] && [ -n "$POD_COUNT" ]; then kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o yaml 2>/dev/null | grep -A 10 "status:" | head -15 || echo " (Could not get pod details)" else echo " No pods found. Checking deployment status:" kubectl_cmd get deployment crossplane-provider-proxmox -n crossplane-system 2>/dev/null || echo " Deployment also not found!" fi echo "" echo " To retry from this point, run:" echo " sudo $0 --resume-from-step12" exit 1 fi echo "" echo " ✅ Pod is ready!" echo "" echo "Step 13: Checking pod image..." POD_IMAGE=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].spec.containers[0].image}' || echo "") POD_IMAGE_ID=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].imageID}' || echo "") echo " Pod image: $POD_IMAGE" echo " Pod image ID: $POD_IMAGE_ID" if echo "$POD_IMAGE" | grep -q "${NEW_TAG}"; then echo " ✅ Pod is using NEW image tag!" else echo " ⚠️ Pod is NOT using new tag!" fi echo "" echo "Step 14: Waiting for logs to generate..." sleep 5 echo "" echo "Step 15: Checking for [FIXED CODE] message..." if kubectl_cmd logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 2>&1 | grep -q '\[FIXED CODE\]'; then echo " ✅ [FIXED CODE] message found! Fix is ACTIVE!" kubectl_cmd logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 | grep '\[FIXED CODE\]' | head -3 else echo " ❌ [FIXED CODE] message NOT found!" echo "" echo " Recent logs:" kubectl_cmd logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=20 | grep -E "Cookie|Authorization|FIXED|DEBUG" | head -5 || echo " (No relevant logs)" fi fi if [ -z "$KUBECTL" ]; then echo "Step 11: Skipping Kubernetes operations (kubectl not found)" echo "" echo " Please manually run:" echo " kubectl apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml" echo " kubectl wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s" echo " kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox | grep '\[FIXED CODE\]'" fi echo "" echo "==========================================" echo "CLEANUP AND DEPLOY COMPLETE" echo "=========================================="