Sankofa/scripts/force-new-image.sh

#!/bin/bash
set -e

# Find kubectl
KUBECTL=$(which kubectl 2>/dev/null || echo "")
if [ -z "$KUBECTL" ]; then
    for path in /usr/local/bin/kubectl /usr/bin/kubectl ~/.local/bin/kubectl; do
        if [ -x "$path" ]; then
            KUBECTL="$path"
            break
        fi
    done
fi

IMAGE_NAME="docker.io/library/crossplane-provider-proxmox"
NEW_DOCKER_ID="sha256:02dfb597dcb17ee74eab0bfe3483d5bbc5e0db66642bdbfd74bf9fcd2af40aaa"
NEW_MANIFEST_DIGEST="sha256:219e9651ca232b6320614f2f2289ba4c9bf8d29809a90319038a6390871a4c26"

echo "=========================================="
echo "FORCE NEW IMAGE - Complete Cleanup"
echo "=========================================="
echo ""

if [ -n "$KUBECTL" ]; then
    echo "Using kubectl: $KUBECTL"
    echo ""
    echo "Step 1: Deleting deployment and pods..."
    $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo "  (Deployment not found)"
    $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo "  (Pods not found)"
else
    echo "⚠️  kubectl not found - skipping Kubernetes operations"
    echo "  You'll need to manually delete/recreate the deployment"
fi
sleep 3

echo ""
echo "Step 2: Listing ALL crossplane images in containerd..."
sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox || echo "  (No images found)"
echo ""

echo "Step 3: Removing ALL crossplane-provider-proxmox images (by name and digest)..."
# Remove by name
ALL_IMAGES=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}" || true)
if [ -n "$ALL_IMAGES" ]; then
    echo "$ALL_IMAGES" | while read -r img; do
        echo "  Removing: $img"
        sudo ctr -n k8s.io images rm "$img" 2>/dev/null || true
    done
fi

# Also try to remove by the old image ID if it exists
echo "  Attempting to remove old image by ID: 5d83df9f..."
sudo ctr -n k8s.io images rm "${IMAGE_NAME}@sha256:5d83df9f8d127aa8988ace01254f4adbb6336f0ad59be8193349b3eacd972cfc" 2>/dev/null || true

echo "  ✅ All images removed"
echo ""

echo "Step 4: Pruning containerd images..."
sudo ctr -n k8s.io images prune 2>&1 | head -5 || echo "  (Prune completed)"
echo ""

echo "Step 5: Verifying image tar exists..."
if [ ! -f /tmp/crossplane-fresh-nuclear.tar ]; then
    echo "  ❌ ERROR: Image tar not found at /tmp/crossplane-fresh-nuclear.tar"
    echo "  Please rebuild: sudo /home/intlc/projects/Sankofa/scripts/nuclear-cleanup-rebuild.sh"
    exit 1
fi
echo "  ✅ Image tar found: $(ls -lh /tmp/crossplane-fresh-nuclear.tar | awk '{print $5}')"
echo ""

echo "Step 6: Re-importing fresh image..."
sudo ctr -n k8s.io images import /tmp/crossplane-fresh-nuclear.tar
echo "  ✅ Image imported"
echo ""

echo "Step 7: Verifying import and checking digest..."
IMPORTED=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}:latest" || true)
if [ -z "$IMPORTED" ]; then
    echo "  ❌ ERROR: Image ${IMAGE_NAME}:latest not found after import!"
    exit 1
fi
echo "  ✅ Image found: $IMPORTED"

# Check the actual digest from the image list output
IMAGE_DIGEST=$(sudo ctr -n k8s.io images ls | grep "${IMAGE_NAME}:latest" | grep -o "sha256:[a-f0-9]\{64\}" | head -1 || true)
if [ -n "$IMAGE_DIGEST" ]; then
    echo "  Image manifest digest: $IMAGE_DIGEST"
    if echo "$IMAGE_DIGEST" | grep -q "219e9651"; then
        echo "  ✅ Correct digest (new image)"
    else
        echo "  ⚠️  WARNING: Digest doesn't match expected new image!"
        echo "     Expected: ...219e9651..."
        echo "     Got: $IMAGE_DIGEST"
    fi
fi
echo ""

echo "Step 8: Final image list in containerd:"
sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox
echo ""

if [ -n "$KUBECTL" ]; then
    echo "Step 9: Recreating deployment..."
    $KUBECTL apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml
    echo "  ✅ Deployment created"
    echo ""

    echo "Step 10: Waiting for pod to be ready..."
    $KUBECTL wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s || {
        echo "  ⚠️  Pod not ready, checking status..."
        $KUBECTL get pod -n crossplane-system -l app=crossplane-provider-proxmox
        exit 1
    }
    echo "  ✅ Pod is ready"
    echo ""

    echo "Step 11: Checking pod image ID..."
    POD_IMAGE_ID=$($KUBECTL get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].imageID}')
    echo "  Pod image ID: $POD_IMAGE_ID"
    if echo "$POD_IMAGE_ID" | grep -q "5d83df9f"; then
        echo "  ⚠️  WARNING: Pod is still using OLD image ID!"
        echo "     This means containerd is still resolving 'latest' to old image."
    elif echo "$POD_IMAGE_ID" | grep -q "219e9651\|02dfb597"; then
        echo "  ✅ Pod is using NEW image!"
    else
        echo "  ⚠️  Unknown image ID"
    fi
    echo ""

    echo "Step 12: Waiting for logs to generate..."
    sleep 5

    echo ""
    echo "Step 13: Checking for [FIXED CODE] message..."
    if $KUBECTL logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 2>&1 | grep -q '\[FIXED CODE\]'; then
        echo "  ✅ [FIXED CODE] message found! Fix is active."
        $KUBECTL logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 | grep '\[FIXED CODE\]' | head -3
    else
        echo "  ❌ [FIXED CODE] message NOT found!"
        echo ""
        echo "  Checking recent logs for Cookie header..."
        $KUBECTL logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=20 | grep -E "Cookie|Authorization" | head -5 || echo "  (No relevant logs found)"
        echo ""
        echo "  This indicates the old code is still running."
        echo "  Possible causes:"
        echo "    1. Containerd is caching the old image"
        echo "    2. The image ID shown is a layer digest, not manifest digest"
        echo "    3. Need to restart containerd (disruptive)"
    fi
else
    echo "Step 9: Skipping Kubernetes operations (kubectl not found)"
    echo ""
    echo "  Please manually run:"
    echo "    kubectl apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml"
    echo "    kubectl wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s"
    echo "    kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox | grep '\[FIXED CODE\]'"
fi

echo ""
echo "=========================================="
echo "FORCE NEW IMAGE COMPLETE"
echo "=========================================="