Sankofa/scripts/kill-and-fix.sh

#!/bin/bash
set -e

# Find kubectl in common locations
KUBECTL=$(which kubectl 2>/dev/null || echo "")
if [ -z "$KUBECTL" ]; then
    # Try common locations
    for path in /usr/local/bin/kubectl /usr/bin/kubectl ~/.local/bin/kubectl; do
        if [ -x "$path" ]; then
            KUBECTL="$path"
            break
        fi
    done
fi

if [ -z "$KUBECTL" ]; then
    echo "ERROR: kubectl not found. Please ensure kubectl is in PATH or update this script."
    exit 1
fi

echo "=========================================="
echo "KILL AND FIX - Aggressive Cleanup"
echo "=========================================="
echo "Using kubectl: $KUBECTL"
echo ""

# Step 1: Force kill pod
echo "Step 1: Force killing pod..."
$KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo "  (Pod not found)"
sleep 2

# Step 2: Force delete deployment
echo ""
echo "Step 2: Force deleting deployment..."
$KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo "  (Deployment not found)"
sleep 2

# Step 3: Remove finalizers if stuck
echo ""
echo "Step 3: Removing finalizers..."
$KUBECTL patch deployment crossplane-provider-proxmox -n crossplane-system -p '{"metadata":{"finalizers":[]}}' --type=merge 2>&1 || echo "  (No finalizers to remove)"
sleep 2

# Step 4: Verify cleanup
echo ""
echo "Step 4: Verifying cleanup..."
REMAINING=$($KUBECTL get all -n crossplane-system -l app=crossplane-provider-proxmox 2>&1 | grep -v "No resources" || true)
if [ -n "$REMAINING" ]; then
    echo "  ⚠️  Some resources still exist:"
    echo "$REMAINING"
else
    echo "  ✅ All resources cleaned up"
fi

# Step 5: Remove ALL crossplane-provider-proxmox images from containerd
echo ""
echo "Step 5: Removing ALL crossplane-provider-proxmox images from containerd..."
ALL_IMAGES=$(sudo ctr -n k8s.io images ls -q | grep crossplane-provider-proxmox || true)
if [ -n "$ALL_IMAGES" ]; then
    echo "$ALL_IMAGES" | while read -r img; do
        echo "  Removing: $img"
        sudo ctr -n k8s.io images rm "$img" 2>/dev/null || true
    done
    echo "  ✅ All images removed"
else
    echo "  (No images found)"
fi

# Step 6: Verify image tar exists
echo ""
echo "Step 6: Verifying image tar exists..."
if [ ! -f /tmp/crossplane-fresh-nuclear.tar ]; then
    echo "  ❌ ERROR: Image tar not found at /tmp/crossplane-fresh-nuclear.tar"
    echo "  Please rebuild the image first using:"
    echo "    sudo /home/intlc/projects/Sankofa/scripts/nuclear-cleanup-rebuild.sh"
    exit 1
fi
echo "  ✅ Image tar found"

# Step 7: Re-import fresh image
echo ""
echo "Step 7: Re-importing fresh image..."
sudo ctr -n k8s.io images import /tmp/crossplane-fresh-nuclear.tar
echo "  ✅ Image imported"

# Step 8: Verify import
echo ""
echo "Step 8: Verifying import..."
IMPORTED=$(sudo ctr -n k8s.io images ls -q | grep "crossplane-provider-proxmox:latest" || true)
if [ -z "$IMPORTED" ]; then
    echo "  ❌ ERROR: Image not found after import!"
    exit 1
fi
echo "  ✅ Image verified: $IMPORTED"

# Step 9: Show image details
echo ""
echo "Step 9: Image details:"
sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox

# Step 10: Recreate deployment
echo ""
echo "Step 10: Recreating deployment..."
$KUBECTL apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml

# Step 11: Wait for pod
echo ""
echo "Step 11: Waiting for pod to be ready..."
$KUBECTL wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s || {
    echo "  ⚠️  Pod not ready within timeout, checking status..."
    $KUBECTL get pod -n crossplane-system -l app=crossplane-provider-proxmox
    exit 1
}
echo "  ✅ Pod is ready"

# Step 12: Check image ID
echo ""
echo "Step 12: Checking pod image ID..."
POD_IMAGE_ID=$($KUBECTL get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].imageID}')
echo "  Pod image ID: $POD_IMAGE_ID"

# Step 13: Wait for logs
echo ""
echo "Step 13: Waiting for logs to generate..."
sleep 5

# Step 14: Check for [FIXED CODE] message
echo ""
echo "Step 14: Checking for [FIXED CODE] message in logs..."
if $KUBECTL logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 | grep -q '\[FIXED CODE\]'; then
    echo "  ✅ [FIXED CODE] message found! Fix is active."
else
    echo "  ⚠️  [FIXED CODE] message NOT found. Checking logs..."
    $KUBECTL logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=20 | grep -E "Cookie|Authorization|FIXED" || echo "  (No relevant log lines found)"
fi

echo ""
echo "=========================================="
echo "KILL AND FIX COMPLETE"
echo "=========================================="