chore: consolidate local WIP (repo cleanup 20260707)

Co-authored-by: Cursor <cursoragent@cursor.com>

scripts/__tests__/scan-projects.test.ts

@@ -0,0 +1,312 @@
+/**
+ * Unit tests for scan-projects.ts
+ */
+
+import { describe, it, expect, vi, beforeEach } from 'vitest'
+import { ProjectScanner, ProjectInfo } from '../scan-projects'
+import { z } from 'zod'
+
+// Mock dependencies
+vi.mock('fs/promises')
+vi.mock('fs')
+vi.mock('child_process')
+vi.mock('cli-progress')
+vi.mock('dotenv')
+
+describe('ProjectScanner', () => {
+  const mockOptions = {
+    projectsDir: '/test/projects',
+    dryRun: false,
+    skipExisting: false,
+    apiUrl: 'http://localhost:4000/graphql',
+    email: 'test@example.com',
+    password: 'password123',
+    outputFormat: 'text' as const,
+    verbose: false,
+    batchSize: 1,
+  }
+
+  describe('validateProjectName', () => {
+    it('should validate and sanitize valid project names', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const testCases = [
+        { input: 'my-project', expected: 'my-project' },
+        { input: 'my_project', expected: 'my_project' },
+        { input: 'myProject123', expected: 'myProject123' },
+        { input: 'my-project_123', expected: 'my-project_123' },
+      ]
+
+      testCases.forEach(({ input, expected }) => {
+        const result = (scanner as any).validateProjectName(input)
+        expect(result).toBe(expected)
+      })
+    })
+
+    it('should sanitize invalid characters', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const testCases = [
+        { input: 'my project', expected: 'my-project' },
+        { input: 'my@project', expected: 'my-project' },
+        { input: 'my.project', expected: 'my-project' },
+        { input: 'my---project', expected: 'my-project' },
+        { input: '-my-project-', expected: 'my-project' },
+      ]
+
+      testCases.forEach(({ input, expected }) => {
+        const result = (scanner as any).validateProjectName(input)
+        expect(result).toBe(expected)
+      })
+    })
+
+    it('should truncate long names', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const longName = 'a'.repeat(300)
+      const result = (scanner as any).validateProjectName(longName)
+      expect(result.length).toBeLessThanOrEqual(255)
+    })
+
+    it('should handle empty names', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const result = (scanner as any).validateProjectName('')
+      expect(result).toBe('project')
+    })
+  })
+
+  describe('sanitizeMetadata', () => {
+    it('should sanitize metadata and remove invalid types', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const metadata = {
+        string: 'value',
+        number: 123,
+        boolean: true,
+        null: null,
+        array: [1, 2, 3],
+        object: { nested: 'value' },
+        function: () => {}, // Should be removed
+        undefined: undefined, // Should be removed
+      }
+
+      const result = (scanner as any).sanitizeMetadata(metadata)
+      expect(result.string).toBe('value')
+      expect(result.number).toBe(123)
+      expect(result.boolean).toBe(true)
+      expect(result.null).toBeNull()
+      expect(Array.isArray(result.array)).toBe(true)
+      expect(typeof result.object).toBe('object')
+      expect(result.function).toBeUndefined()
+      expect(result.undefined).toBeUndefined()
+    })
+
+    it('should enforce size limits', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const largeMetadata = {
+        description: 'x'.repeat(15000), // Too large
+        version: '1.0.0',
+      }
+
+      expect(() => {
+        (scanner as any).sanitizeMetadata(largeMetadata)
+      }).toThrow('Metadata too large')
+    })
+  })
+
+  describe('extractDomainFromGitRemote', () => {
+    it('should extract domain from HTTPS URLs', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const testCases = [
+        {
+          input: 'https://github.com/user/repo.git',
+          expected: 'github-com.dev',
+        },
+        {
+          input: 'https://gitlab.com/user/repo.git',
+          expected: 'gitlab-com.dev',
+        },
+        {
+          input: 'https://bitbucket.org/user/repo.git',
+          expected: 'bitbucket-org.dev',
+        },
+      ]
+
+      testCases.forEach(({ input, expected }) => {
+        const result = (scanner as any).extractDomainFromGitRemote(input)
+        expect(result).toBe(expected)
+      })
+    })
+
+    it('should extract domain from SSH URLs', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const testCases = [
+        {
+          input: 'git@github.com:user/repo.git',
+          expected: 'github-com.dev',
+        },
+        {
+          input: 'ssh://git@gitlab.com:user/repo.git',
+          expected: 'gitlab-com.dev',
+        },
+      ]
+
+      testCases.forEach(({ input, expected }) => {
+        const result = (scanner as any).extractDomainFromGitRemote(input)
+        expect(result).toBe(expected)
+      })
+    })
+
+    it('should return undefined for invalid URLs', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const result = (scanner as any).extractDomainFromGitRemote('invalid-url')
+      expect(result).toBeUndefined()
+    })
+
+    it('should return undefined for empty input', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const result = (scanner as any).extractDomainFromGitRemote(undefined)
+      expect(result).toBeUndefined()
+    })
+  })
+
+  describe('detectPackageManager', () => {
+    it('should detect pnpm from pnpm-lock.yaml', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const { existsSync } = require('fs')
+      vi.mocked(existsSync).mockImplementation((path: string) => {
+        return path.includes('pnpm-lock.yaml')
+      })
+
+      const result = (scanner as any).detectPackageManager('/test/project')
+      expect(result).toBe('pnpm')
+    })
+
+    it('should detect yarn from yarn.lock', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const { existsSync } = require('fs')
+      vi.mocked(existsSync).mockImplementation((path: string) => {
+        return path.includes('yarn.lock')
+      })
+
+      const result = (scanner as any).detectPackageManager('/test/project')
+      expect(result).toBe('yarn')
+    })
+
+    it('should detect npm from package-lock.json', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const { existsSync } = require('fs')
+      vi.mocked(existsSync).mockImplementation((path: string) => {
+        return path.includes('package-lock.json')
+      })
+
+      const result = (scanner as any).detectPackageManager('/test/project')
+      expect(result).toBe('npm')
+    })
+
+    it('should return unknown if no lock file found', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      const { existsSync } = require('fs')
+      vi.mocked(existsSync).mockReturnValue(false)
+
+      const result = (scanner as any).detectPackageManager('/test/project')
+      expect(result).toBe('unknown')
+    })
+  })
+
+  describe('isDuplicate', () => {
+    it('should detect duplicates by name', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      ;(scanner as any).existingTenants = new Map([
+        ['my-project', { id: '123', name: 'my-project' }],
+      ])
+
+      const project: ProjectInfo = {
+        name: 'my-project',
+        path: '/test',
+        type: 'repo',
+        metadata: {},
+      }
+
+      const result = (scanner as any).isDuplicate(project)
+      expect(result).not.toBeNull()
+      expect(result.name).toBe('my-project')
+    })
+
+    it('should detect duplicates by domain', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      ;(scanner as any).existingTenants = new Map([
+        ['github-com.dev', { id: '123', name: 'existing', domain: 'github-com.dev' }],
+      ])
+
+      const project: ProjectInfo = {
+        name: 'new-project',
+        path: '/test',
+        type: 'repo',
+        gitRemote: 'https://github.com/user/repo.git',
+        metadata: {},
+      }
+
+      const result = (scanner as any).isDuplicate(project)
+      expect(result).not.toBeNull()
+    })
+
+    it('should return null for non-duplicates', () => {
+      const scanner = new ProjectScanner(mockOptions)
+      ;(scanner as any).existingTenants = new Map()
+
+      const project: ProjectInfo = {
+        name: 'new-project',
+        path: '/test',
+        type: 'repo',
+        metadata: {},
+      }
+
+      const result = (scanner as any).isDuplicate(project)
+      expect(result).toBeNull()
+    })
+  })
+})
+
+describe('Validation Schemas', () => {
+  describe('projectNameSchema', () => {
+    it('should accept valid project names', () => {
+      const validNames = ['my-project', 'my_project', 'myProject123', 'my-project_123']
+      validNames.forEach((name) => {
+        expect(() => {
+          z.string().min(1).max(255).regex(/^[a-zA-Z0-9_-]+$/).parse(name)
+        }).not.toThrow()
+      })
+    })
+
+    it('should reject invalid project names', () => {
+      const invalidNames = ['my project', 'my@project', 'my.project', 'my-project!']
+      invalidNames.forEach((name) => {
+        expect(() => {
+          z.string().min(1).max(255).regex(/^[a-zA-Z0-9_-]+$/).parse(name)
+        }).toThrow()
+      })
+    })
+  })
+
+  describe('gitUrlSchema', () => {
+    it('should accept valid Git URLs', () => {
+      const validUrls = [
+        'https://github.com/user/repo.git',
+        'git@github.com:user/repo.git',
+        'ssh://git@gitlab.com:user/repo.git',
+      ]
+      validUrls.forEach((url) => {
+        expect(() => {
+          z.string().url().parse(url)
+        }).not.toThrow()
+      })
+    })
+
+    it('should reject invalid URLs', () => {
+      const invalidUrls = ['not-a-url', 'http://', 'git@']
+      invalidUrls.forEach((url) => {
+        expect(() => {
+          z.string().url().parse(url)
+        }).toThrow()
+      })
+    })
+  })
+})
+

scripts/add-cephfs-storage-via-webui.md

@@ -0,0 +1,37 @@
+# Adding CephFS Storage via Proxmox Web UI
+
+Since the CLI and API methods are having issues with content types, use the Web UI:
+
+## Steps:
+
+1. **Open Proxmox Web UI**
+   - Navigate to: `https://192.168.11.11:8006`
+   - Login with your credentials
+
+2. **Add CephFS Storage**
+   - Go to: **Datacenter** → **Storage** → **Add** → **CephFS**
+   - Fill in the form:
+     - **ID**: `ceph-fs`
+     - **FS Name**: `ceph-fs`
+     - **Monitors**: `192.168.11.10:6789,192.168.11.11:6789`
+     - **Username**: `admin`
+     - **Nodes**: Select `r630-01`
+     - **Content**: Leave default or select available options
+   - Click **Add**
+
+3. **Verify Storage**
+   - After adding, verify in: **Datacenter** → **Storage**
+   - You should see `ceph-fs` listed
+
+## Important Note:
+
+**CephFS may not support VM disk images**. If your VMs fail to create with CephFS storage, you may need to:
+
+1. **Create RBD storage instead** (for VM disk images):
+   - **Datacenter** → **Storage** → **Add** → **RBD**
+   - **ID**: `ceph-rbd`
+   - **Pool**: `rbd`
+   - **Nodes**: `r630-01`
+
+2. **Update VM configurations** to use `ceph-rbd` or `local-lvm` instead of `ceph-fs`
+

scripts/analyze-r630-disk-layout.sh

@@ -0,0 +1,221 @@
+#!/bin/bash
+# Comprehensive disk layout analysis for R630-01
+# Analyzes the current disk configuration and provides recommendations
+
+# Don't exit on error - we want to continue even if some commands fail
+set +e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+echo "=========================================="
+echo -e "${CYAN}R630-01 Disk Layout Analysis${NC}"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${YELLOW}Warning: Some information may be limited without root access${NC}"
+    echo ""
+fi
+
+echo -e "${BLUE}=== System Overview ===${NC}"
+echo "-----------------------------------"
+echo "Hostname: $(hostname 2>/dev/null || echo 'unknown')"
+echo "Date: $(date)"
+echo ""
+
+echo -e "${BLUE}=== Complete Block Device Layout ===${NC}"
+echo "-----------------------------------"
+lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL,SERIAL,UUID
+echo ""
+
+echo -e "${BLUE}=== Disk Categorization ===${NC}"
+echo "-----------------------------------"
+
+# System disk (sda)
+echo -e "${GREEN}System Disk (sda):${NC}"
+if [ -b /dev/sda ]; then
+    sda_size=$(lsblk -d -n -o SIZE /dev/sda 2>/dev/null || echo "unknown")
+    echo "  Size: $sda_size"
+    echo "  Partitions:"
+    lsblk -n /dev/sda | grep -v "^sda" | sed 's/^/    /'
+    echo "  LVM Volumes:"
+    if command -v pvs &>/dev/null; then
+        pvs 2>/dev/null | grep sda | sed 's/^/    /' || echo "    (No LVM volumes found)"
+    fi
+    echo "  Status: ${GREEN}Active system disk${NC}"
+else
+    echo "  ${RED}Not found${NC}"
+fi
+echo ""
+
+# Secondary disk (sdb)
+echo -e "${YELLOW}Secondary Disk (sdb):${NC}"
+if [ -b /dev/sdb ]; then
+    sdb_size=$(lsblk -d -n -o SIZE /dev/sdb 2>/dev/null || echo "unknown")
+    echo "  Size: $sdb_size"
+    partitions=$(lsblk -n /dev/sdb 2>/dev/null | grep -c "part" || echo "0")
+    if [ "$partitions" -eq 0 ]; then
+        echo "  Status: ${YELLOW}Unused - No partitions${NC}"
+        echo "  Recommendation: Available for Ceph OSD or Proxmox storage"
+    else
+        echo "  Partitions:"
+        lsblk -n /dev/sdb | grep -v "^sdb" | sed 's/^/    /'
+        echo "  Status: ${YELLOW}Has partitions${NC}"
+    fi
+else
+    echo "  ${RED}Not found${NC}"
+fi
+echo ""
+
+# Ceph candidate disks (sdc-sdh)
+echo -e "${CYAN}Ceph Candidate Disks (sdc-sdh):${NC}"
+ceph_disks=0
+ceph_available=0
+ceph_used=0
+
+for disk in sdc sdd sde sdf sdg sdh; do
+    if [ -b "/dev/$disk" ]; then
+        ceph_disks=$((ceph_disks + 1))
+        size=$(lsblk -d -n -o SIZE /dev/$disk 2>/dev/null || echo "unknown")
+        partitions=$(lsblk -n /dev/$disk 2>/dev/null | grep -c "part" || echo "0")
+        partitions=$(echo "$partitions" | tr -d '\n' | head -1)
+        mountpoint=$(lsblk -d -n -o MOUNTPOINT /dev/$disk 2>/dev/null | tr -d '\n' || echo "")
+        fstype=$(lsblk -d -n -o FSTYPE /dev/$disk 2>/dev/null | tr -d '\n' || echo "")
+        
+        echo "  /dev/$disk:"
+        echo "    Size: $size"
+        
+        if [ "${partitions:-0}" -eq 0 ] && [ -z "$mountpoint" ] && [ -z "$fstype" ]; then
+            echo -e "    Status: ${GREEN}Available (no partitions, unmounted)${NC}"
+            ceph_available=$((ceph_available + 1))
+        elif [ "${partitions:-0}" -gt 0 ]; then
+            echo "    Partitions: $partitions"
+            lsblk -n /dev/$disk | grep -v "^$disk" | sed 's/^/      /'
+            echo -e "    Status: ${YELLOW}Has partitions${NC}"
+            ceph_used=$((ceph_used + 1))
+        else
+            echo -e "    Status: ${YELLOW}In use (mounted or has filesystem)${NC}"
+            ceph_used=$((ceph_used + 1))
+        fi
+        
+        # Check if in LVM
+        if command -v pvs &>/dev/null; then
+            in_lvm=$(pvs 2>/dev/null | grep "/dev/$disk" | wc -l)
+            if [ "$in_lvm" -gt 0 ]; then
+                vg=$(pvs 2>/dev/null | grep "/dev/$disk" | awk '{print $2}')
+                echo -e "    LVM: ${YELLOW}In volume group: $vg${NC}"
+            fi
+        fi
+        
+        # Check if has Ceph OSD
+        if command -v ceph-volume &>/dev/null; then
+            has_osd=$(ceph-volume lvm list /dev/$disk 2>/dev/null | grep -c "osd\." || echo "0")
+            if [ "$has_osd" -gt 0 ]; then
+                osd_id=$(ceph-volume lvm list /dev/$disk 2>/dev/null | grep -oP "osd\.\K\d+" | head -1)
+                echo -e "    Ceph: ${GREEN}OSD $osd_id${NC}"
+            fi
+        fi
+    else
+        echo "  /dev/$disk: ${RED}Not found${NC}"
+    fi
+done
+echo ""
+
+echo -e "${BLUE}=== Storage Summary ===${NC}"
+echo "-----------------------------------"
+echo "Total block devices: $(lsblk -d -n | grep -c '^sd' || echo '0')"
+echo "System disk (sda): 1"
+echo "Secondary disk (sdb): $([ -b /dev/sdb ] && echo '1' || echo '0')"
+echo "Ceph candidate disks (sdc-sdh): $ceph_disks"
+echo "  - Available: $ceph_available"
+echo "  - In use: $ceph_used"
+echo ""
+
+echo -e "${BLUE}=== Proxmox Storage Status ===${NC}"
+echo "-----------------------------------"
+if command -v pvesm &>/dev/null; then
+    pvesm status 2>/dev/null || echo -e "${YELLOW}pvesm not accessible${NC}"
+else
+    echo -e "${YELLOW}pvesm command not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Ceph Status ===${NC}"
+echo "-----------------------------------"
+if command -v ceph &>/dev/null; then
+    echo "Ceph health:"
+    ceph health 2>/dev/null || echo -e "${YELLOW}Ceph not accessible${NC}"
+    echo ""
+    echo "OSD tree:"
+    ceph osd tree 2>/dev/null || echo -e "${YELLOW}Ceph not accessible${NC}"
+    echo ""
+    echo "OSD details:"
+    ceph osd df 2>/dev/null || echo -e "${YELLOW}Ceph not accessible${NC}"
+else
+    echo -e "${YELLOW}Ceph not installed${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== LVM Status ===${NC}"
+echo "-----------------------------------"
+if command -v vgs &>/dev/null; then
+    echo "Volume Groups:"
+    vgs 2>/dev/null | sed 's/^/  /' || echo -e "  ${YELLOW}No volume groups found${NC}"
+    echo ""
+    echo "Physical Volumes:"
+    pvs 2>/dev/null | sed 's/^/  /' || echo -e "  ${YELLOW}No physical volumes found${NC}"
+    echo ""
+    echo "Logical Volumes:"
+    lvs 2>/dev/null | sed 's/^/  /' || echo -e "  ${YELLOW}No logical volumes found${NC}"
+else
+    echo -e "${YELLOW}LVM not available${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Recommendations ===${NC}"
+echo "-----------------------------------"
+
+if [ "$ceph_available" -gt 0 ]; then
+    echo -e "${GREEN}Available disks for Ceph OSD:${NC}"
+    for disk in sdc sdd sde sdf sdg sdh; do
+        if [ -b "/dev/$disk" ]; then
+            partitions=$(lsblk -n /dev/$disk 2>/dev/null | grep -c "part" || echo "0")
+            mountpoint=$(lsblk -d -n -o MOUNTPOINT /dev/$disk 2>/dev/null || echo "")
+            if [ "$partitions" -eq 0 ] && [ -z "$mountpoint" ]; then
+                echo "  - /dev/$disk: Ready for Ceph OSD"
+                echo "    Command: ceph-volume lvm create --data /dev/$disk"
+            fi
+        fi
+    done
+    echo ""
+fi
+
+if [ -b /dev/sdb ]; then
+    partitions=$(lsblk -n /dev/sdb 2>/dev/null | grep -c "part" || echo "0")
+    if [ "$partitions" -eq 0 ]; then
+        echo -e "${GREEN}Secondary disk (sdb) available:${NC}"
+        echo "  - Can be used for Ceph OSD"
+        echo "  - Can be added as Proxmox storage"
+        echo "  - Can be used for backup storage"
+        echo ""
+    fi
+fi
+
+if [ "$ceph_used" -gt 0 ]; then
+    echo -e "${YELLOW}Disks currently in use:${NC}"
+    echo "  - Review partitions and usage before repurposing"
+    echo "  - Check if data needs to be migrated"
+    echo ""
+fi
+
+echo "=========================================="
+echo -e "${CYAN}Analysis Complete${NC}"
+echo "=========================================="
+

scripts/check-proxmox-storage.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Quick script to check Proxmox storage and block devices
+# Run on R630-01
+
+echo "=========================================="
+echo "Proxmox Storage and Block Device Check"
+echo "=========================================="
+echo ""
+
+echo "=== Proxmox Storage Status ==="
+pvesm status
+echo ""
+
+echo "=== Block Devices ==="
+lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL
+echo ""
+
+echo "=== 250GB Drives (sdc-sdh) ==="
+for drive in sdc sdd sde sdf sdg sdh; do
+    if [ -b "/dev/$drive" ]; then
+        echo ""
+        echo "/dev/$drive:"
+        lsblk /dev/$drive
+        fdisk -l /dev/$drive 2>/dev/null | head -10
+    fi
+done
+echo ""
+
+echo "=== Summary ==="
+echo "Total drives visible: $(lsblk -d -n | grep -c '^sd')"
+echo "250GB drives (sdc-sdh): $(lsblk -d -n | grep -E '^sd[c-h]' | wc -l)"
+

scripts/check-storage-sshpass.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Run pvesm status and lsblk on R630-01 using sshpass
+# Usage: ./check-storage-sshpass.sh [password]
+# Or: PROXMOX_PASSWORD=yourpass ./check-storage-sshpass.sh
+
+PROXMOX_HOST="192.168.11.11"
+PROXMOX_USER="root"
+
+# Get password from argument or environment variable
+if [ -n "$1" ]; then
+    PASSWORD="$1"
+elif [ -n "$PROXMOX_PASSWORD" ]; then
+    PASSWORD="$PROXMOX_PASSWORD"
+else
+    echo "Usage: $0 <password>"
+    echo "Or: PROXMOX_PASSWORD=yourpass $0"
+    exit 1
+fi
+
+echo "=========================================="
+echo "Checking Proxmox Storage and Block Devices"
+echo "=========================================="
+echo ""
+
+echo "=== Proxmox Storage Status ==="
+sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$PROXMOX_USER@$PROXMOX_HOST" "pvesm status"
+echo ""
+
+echo "=== Block Devices ==="
+sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$PROXMOX_USER@$PROXMOX_HOST" "lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL"
+echo ""
+
+echo "=== 250GB Drives Detail ==="
+sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$PROXMOX_USER@$PROXMOX_HOST" "
+for drive in sdc sdd sde sdf sdg sdh; do
+    if [ -b \"/dev/\$drive\" ]; then
+        echo \"\"
+        echo \"/dev/\$drive:\"
+        lsblk /dev/\$drive
+        fdisk -l /dev/\$drive 2>/dev/null | head -5
+    fi
+done
+"
+

scripts/check-ubuntu-vg-and-prepare-osd.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+# Script to check ubuntu-vg and prepare a drive for Ceph OSD
+# Run on R630-01
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+echo "=========================================="
+echo "Checking ubuntu-vg and Preparing for Ceph OSD"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+echo -e "${BLUE}=== Step 1: Current ubuntu-vg Status ===${NC}"
+echo "-----------------------------------"
+echo "Volume Group Information:"
+vgs ubuntu-vg 2>/dev/null || echo -e "${YELLOW}ubuntu-vg not found${NC}"
+echo ""
+
+echo "Logical Volumes in ubuntu-vg:"
+lvs ubuntu-vg 2>/dev/null || echo -e "${YELLOW}No logical volumes found${NC}"
+echo ""
+
+echo "Physical Volumes in ubuntu-vg:"
+pvs | grep ubuntu-vg || echo -e "${YELLOW}No physical volumes found${NC}"
+echo ""
+
+echo -e "${BLUE}=== Step 2: All Block Devices ===${NC}"
+echo "-----------------------------------"
+lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL | grep -E "NAME|sd"
+echo ""
+
+echo -e "${BLUE}=== Step 3: Identifying 250GB Drives ===${NC}"
+echo "-----------------------------------"
+echo "Drives that are ~250GB (247-248 GB):"
+for disk in /dev/sd[a-z]; do
+    if [ -b "$disk" ]; then
+        size_info=$(fdisk -l "$disk" 2>/dev/null | grep "^Disk $disk" | awk '{print $3, $4}')
+        if [ ! -z "$size_info" ]; then
+            size_num=$(echo "$size_info" | grep -oE '[0-9]+' | head -1)
+            if [ "$size_num" -ge 247 ] && [ "$size_num" -le 248 ]; then
+                echo -e "${GREEN}  $disk: ${size_info}${NC}"
+                # Check if it's in ubuntu-vg
+                in_vg=$(pvs 2>/dev/null | grep "$disk" | grep ubuntu-vg | wc -l)
+                if [ "$in_vg" -gt 0 ]; then
+                    echo -e "    ${YELLOW}  → In ubuntu-vg volume group${NC}"
+                fi
+            fi
+        fi
+    fi
+done
+echo ""
+
+echo -e "${BLUE}=== Step 4: Checking What's Using ubuntu-vg ===${NC}"
+echo "-----------------------------------"
+if vgs ubuntu-vg &>/dev/null; then
+    echo "Logical volumes in ubuntu-vg:"
+    lvs ubuntu-vg -o lv_name,lv_size,lv_attr,mountpoint 2>/dev/null
+    echo ""
+    echo "Checking if LVs are mounted:"
+    for lv in $(lvs ubuntu-vg -o lv_path --no-headings 2>/dev/null | awk '{print $1}'); do
+        if [ ! -z "$lv" ]; then
+            mount_point=$(mount | grep "$lv" | awk '{print $3}')
+            if [ ! -z "$mount_point" ]; then
+                echo -e "  ${YELLOW}$lv is mounted at $mount_point${NC}"
+            else
+                echo -e "  ${GREEN}$lv is not mounted${NC}"
+            fi
+        fi
+    done
+else
+    echo -e "${YELLOW}ubuntu-vg not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 5: Current Ceph OSD Status ===${NC}"
+echo "-----------------------------------"
+ceph osd tree 2>/dev/null || echo -e "${YELLOW}Ceph not accessible${NC}"
+echo ""
+
+echo "=========================================="
+echo -e "${GREEN}Summary and Recommendations${NC}"
+echo "=========================================="
+echo ""
+echo "If ubuntu-vg is not needed:"
+echo "1. Unmount any mounted logical volumes"
+echo "2. Remove logical volumes from ubuntu-vg"
+echo "3. Remove physical volumes from ubuntu-vg"
+echo "4. Remove ubuntu-vg volume group"
+echo "5. Wipe the drive(s) to prepare for Ceph OSD"
+echo ""
+echo "If you need to free up a drive for Ceph OSD:"
+echo "1. Identify which drive to use (e.g., /dev/sdc)"
+echo "2. Remove it from ubuntu-vg (if safe to do so)"
+echo "3. Wipe the drive: wipefs -a /dev/sdX"
+echo "4. Create Ceph OSD: ceph-volume lvm create --data /dev/sdX"
+echo ""
+echo -e "${YELLOW}WARNING: Removing drives from ubuntu-vg will destroy data!${NC}"
+echo "Make sure ubuntu-vg is not needed before proceeding."
+echo ""
+

scripts/complete-ceph-reinstall.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+# Complete Ceph uninstall, cleanup, and reinstall
+set -e
+
+echo "=== CEPH COMPLETE REINSTALL SCRIPT ==="
+echo "WARNING: This will destroy all Ceph data and configuration!"
+echo ""
+
+# Backup critical configs first
+echo "=== Step 1: Backing up critical configurations ==="
+BACKUP_DIR="/root/ceph-backup-$(date +%Y%m%d-%H%M%S)"
+mkdir -p "$BACKUP_DIR"
+cp -a /etc/ceph/ "$BACKUP_DIR/ceph-config/" 2>/dev/null || true
+cp -a /etc/pve/ceph.conf "$BACKUP_DIR/" 2>/dev/null || true
+cp -a /var/lib/ceph/bootstrap-osd/ "$BACKUP_DIR/bootstrap-osd/" 2>/dev/null || true
+echo "Backup created at: $BACKUP_DIR"
+
+echo ""
+echo "=== Step 2: Stopping all Ceph services ==="
+systemctl stop ceph-osd@*.service 2>/dev/null || true
+systemctl stop ceph-mon@*.service 2>/dev/null || true
+systemctl stop ceph-mgr@*.service 2>/dev/null || true
+systemctl stop ceph-mds@*.service 2>/dev/null || true
+systemctl stop ceph.target 2>/dev/null || true
+sleep 3
+
+echo ""
+echo "=== Step 3: Disabling all Ceph services ==="
+systemctl disable ceph-osd@*.service 2>/dev/null || true
+systemctl disable ceph-mon@*.service 2>/dev/null || true
+systemctl disable ceph-mgr@*.service 2>/dev/null || true
+systemctl disable ceph-mds@*.service 2>/dev/null || true
+systemctl disable ceph.target 2>/dev/null || true
+
+echo ""
+echo "=== Step 4: Unmounting OSD directories ==="
+# Unmount all OSD tmpfs mounts
+for osd_dir in /var/lib/ceph/osd/ceph-*; do
+    if mountpoint -q "$osd_dir" 2>/dev/null; then
+        echo "Unmounting $osd_dir..."
+        umount -f "$osd_dir" 2>/dev/null || true
+    fi
+done
+sleep 2
+
+echo ""
+echo "=== Step 5: Removing Ceph packages (Ceph only, not Proxmox) ==="
+# Only remove Ceph packages, not Proxmox dependencies
+apt-get remove --purge -y ceph ceph-base ceph-common ceph-mgr ceph-mon ceph-osd ceph-mds ceph-fuse ceph-volume 2>/dev/null || true
+apt-get remove --purge -y libcephfs2 librados2 librbd1 python3-cephfs python3-rados python3-rbd 2>/dev/null || true
+apt-get autoremove -y 2>/dev/null || true
+
+echo ""
+echo "=== Step 6: Cleaning up Ceph directories ==="
+# Force remove OSD directories
+for osd_dir in /var/lib/ceph/osd/ceph-*; do
+    if [ -d "$osd_dir" ]; then
+        umount -f "$osd_dir" 2>/dev/null || true
+        rm -rf "$osd_dir" 2>/dev/null || true
+    fi
+done
+rm -rf /var/lib/ceph/mon/*
+rm -rf /var/lib/ceph/mgr/*
+rm -rf /var/lib/ceph/mds/*
+rm -rf /var/lib/ceph/bootstrap-osd/*
+rm -rf /var/log/ceph/*
+rm -rf /etc/ceph/*
+rm -rf /etc/pve/ceph.conf
+rm -rf /run/ceph/*
+
+echo ""
+echo "=== Step 7: Removing Ceph LVM volumes ==="
+# Find and remove all Ceph LVs
+for vg in $(vgs --noheadings -o vg_name | grep ceph); do
+    echo "Removing volume group: $vg"
+    vgremove -f "$vg" 2>/dev/null || true
+done
+
+# Remove Ceph physical volumes
+for pv in $(pvs --noheadings -o pv_name | grep -E "(sdc|sdd|sde|sdf|sdg|sdh)"); do
+    echo "Removing physical volume: $pv"
+    pvremove -f "$pv" 2>/dev/null || true
+done
+
+echo ""
+echo "=== Step 8: Wiping Ceph drives ==="
+for disk in sdc sdd sde sdf sdg sdh; do
+    if [ -e "/dev/$disk" ]; then
+        echo "Wiping /dev/$disk..."
+        wipefs -a /dev/$disk 2>/dev/null || true
+        dd if=/dev/zero of=/dev/$disk bs=1M count=100 2>/dev/null || true
+    fi
+done
+
+echo ""
+echo "=== Step 9: Flushing kernel caches ==="
+sync
+echo 3 > /proc/sys/vm/drop_caches
+
+echo ""
+echo "=== Step 10: Reinstalling Ceph ==="
+apt-get update
+apt-get install -y ceph ceph-common ceph-base
+
+echo ""
+echo "=== Step 11: Restoring bootstrap keyring if available ==="
+if [ -d "$BACKUP_DIR/bootstrap-osd" ]; then
+    mkdir -p /var/lib/ceph/bootstrap-osd/ceph/
+    cp -a "$BACKUP_DIR/bootstrap-osd/"* /var/lib/ceph/bootstrap-osd/ceph/ 2>/dev/null || true
+fi
+
+echo ""
+echo "=== Step 12: Restoring Ceph configuration if available ==="
+if [ -d "$BACKUP_DIR/ceph-config" ]; then
+    mkdir -p /etc/ceph/
+    cp -a "$BACKUP_DIR/ceph-config/"* /etc/ceph/ 2>/dev/null || true
+fi
+
+if [ -f "$BACKUP_DIR/ceph.conf" ]; then
+    cp "$BACKUP_DIR/ceph.conf" /etc/pve/ceph.conf 2>/dev/null || true
+fi
+
+echo ""
+echo "=== COMPLETE ==="
+echo "Ceph has been uninstalled, cleaned, and reinstalled."
+echo "Backup location: $BACKUP_DIR"
+echo ""
+echo "Next steps:"
+echo "1. Review /etc/ceph/ceph.conf and /etc/pve/ceph.conf"
+echo "2. Reinitialize Ceph cluster if needed: pveceph init"
+echo "3. Create monitors: pveceph mon create <node>"
+echo "4. Create OSDs: ceph-volume lvm create --data /dev/<disk>"
+

scripts/complete-cleanup-and-deploy.sh

@@ -0,0 +1,452 @@
+#!/bin/bash
+set -e
+
+# Check if resuming from step 12 (pod ready check)
+RESUME_FROM_STEP_12=false
+if [ "${1:-}" == "--resume-from-step12" ] || [ "${1:-}" == "--resume" ]; then
+    RESUME_FROM_STEP_12=true
+    echo "=========================================="
+    echo "RESUMING FROM STEP 12 (POD READY CHECK)"
+    echo "=========================================="
+    echo ""
+else
+    echo "=========================================="
+    echo "COMPLETE CLEANUP AND DEPLOY"
+    echo "=========================================="
+    echo ""
+fi
+
+IMAGE_NAME="crossplane-provider-proxmox"
+NEW_TAG="v1.0.1-cookie-fix-final"
+NEW_IMAGE_TAR="/tmp/crossplane-v1.0.1-fixed.tar"
+
+# Detect if running in kind cluster
+KIND_NODE=""
+if docker ps --format '{{.Names}}' 2>/dev/null | grep -qE '^(sankofa-control-plane|.*-control-plane)$'; then
+    KIND_NODE=$(docker ps --format '{{.Names}}' 2>/dev/null | grep -E '^(sankofa-control-plane|.*-control-plane)$' | head -1)
+    echo "🔍 Detected kind cluster node: $KIND_NODE"
+    echo ""
+fi
+
+# Function to run ctr commands - either in kind node or on host
+ctr_cmd() {
+    if [ -n "$KIND_NODE" ]; then
+        docker exec "$KIND_NODE" ctr "$@"
+    else
+        sudo ctr "$@"
+    fi
+}
+
+# Skip to step 12 if resuming
+if [ "$RESUME_FROM_STEP_12" = true ]; then
+    echo "Skipping steps 1-11, resuming from pod ready check..."
+    echo ""
+    SKIP_TO_STEP_12=true
+else
+    SKIP_TO_STEP_12=false
+fi
+
+# Find kubectl - check both current user and original user
+KUBECTL=$(which kubectl 2>/dev/null || echo "")
+if [ -z "$KUBECTL" ]; then
+    ORIG_USER="${SUDO_USER:-${USER}}"
+    # Try original user's PATH
+    if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ]; then
+        KUBECTL=$(sudo -u "$ORIG_USER" which kubectl 2>/dev/null || echo "")
+    fi
+    # Try common locations
+    if [ -z "$KUBECTL" ]; then
+        for path in /usr/local/bin/kubectl /usr/bin/kubectl ~/.local/bin/kubectl /home/$ORIG_USER/.local/bin/kubectl; do
+            if [ -x "$path" ]; then
+                KUBECTL="$path"
+                break
+            fi
+        done
+    fi
+fi
+
+if [ "$SKIP_TO_STEP_12" = false ]; then
+    echo "Step 1: Deleting Kubernetes deployment and pods..."
+    ORIG_USER="${SUDO_USER:-${USER}}"
+    if [ -n "$KUBECTL" ]; then
+    # Use original user's kubeconfig when running with sudo
+    if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ] && [ "$EUID" -eq 0 ]; then
+        KUBECONFIG="${KUBECONFIG:-/home/$ORIG_USER/.kube/config}"
+        if [ -f "$KUBECONFIG" ]; then
+            export KUBECONFIG
+            sudo -u "$ORIG_USER" env KUBECONFIG="$KUBECONFIG" $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo "  (Deployment not found)"
+            sudo -u "$ORIG_USER" env KUBECONFIG="$KUBECONFIG" $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo "  (Pods not found)"
+        else
+            sudo -u "$ORIG_USER" $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo "  (Deployment not found)"
+            sudo -u "$ORIG_USER" $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo "  (Pods not found)"
+        fi
+    else
+        $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo "  (Deployment not found)"
+        $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo "  (Pods not found)"
+    fi
+    sleep 3
+    echo "  ✅ Kubernetes resources deleted"
+else
+    echo "  ⚠️  kubectl not found, skipping Kubernetes cleanup"
+    fi
+    echo ""
+
+    echo "Step 2: Removing ALL crossplane-provider-proxmox images from containerd..."
+ALL_IMAGES=$(ctr_cmd -n k8s.io images ls -q | grep "${IMAGE_NAME}" || true)
+if [ -n "$ALL_IMAGES" ]; then
+    echo "$ALL_IMAGES" | while read -r img; do
+        echo "  Removing: $img"
+        ctr_cmd -n k8s.io images rm "$img" 2>/dev/null || true
+    done
+    echo "  ✅ All crossplane images removed from containerd"
+else
+    echo "  (No crossplane images found)"
+fi
+echo ""
+
+echo "Step 3: Pruning unused containerd images (aggressive)..."
+ctr_cmd -n k8s.io images prune --all 2>&1 | head -10 || echo "  (Prune completed)"
+echo "  ✅ Containerd pruned"
+echo ""
+
+echo "Step 4: Removing old Docker images..."
+docker rmi crossplane-provider-proxmox:latest crossplane-provider-proxmox:fixed crossplane-provider-proxmox:final-fix crossplane-provider-proxmox:v1.0.1-cookie-fix 2>/dev/null || echo "  (Some images not found, continuing...)"
+docker rmi crossplane-provider-proxmox:${NEW_TAG} 2>/dev/null || echo "  (New image not in Docker yet, will import)"
+echo "  ✅ Old Docker images removed"
+echo ""
+
+echo "Step 5: Cleaning Docker build cache (aggressive)..."
+# Try with user's Docker (rootless)
+ORIG_USER="${SUDO_USER:-${USER}}"
+if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ]; then
+    DOCKER_HOST_VAL="${DOCKER_HOST:-unix:///run/user/$(id -u $ORIG_USER)/docker.sock}"
+    sudo -u "$ORIG_USER" env DOCKER_HOST=$DOCKER_HOST_VAL docker builder prune -af 2>&1 | tail -5 || echo "  (Cache clean completed)"
+else
+    docker builder prune -af 2>&1 | tail -5 || echo "  (Cache clean completed)"
+fi
+echo "  ✅ Docker build cache cleaned"
+echo ""
+
+echo "Step 6: Checking if new image tar exists, rebuilding if needed..."
+if [ ! -f "$NEW_IMAGE_TAR" ]; then
+    echo "  ⚠️  Image tar not found, rebuilding..."
+    PROJECT_DIR="/home/intlc/projects/Sankofa/crossplane-provider-proxmox"
+    if [ -d "$PROJECT_DIR" ]; then
+        if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ]; then
+            DOCKER_HOST_VAL="${DOCKER_HOST:-unix:///run/user/$(id -u $ORIG_USER)/docker.sock}"
+            sudo -u "$ORIG_USER" env DOCKER_HOST=$DOCKER_HOST_VAL docker build --no-cache -t crossplane-provider-proxmox:${NEW_TAG} "$PROJECT_DIR" 2>&1 | tail -10
+            sudo -u "$ORIG_USER" env DOCKER_HOST=$DOCKER_HOST_VAL docker save crossplane-provider-proxmox:${NEW_TAG} -o "$NEW_IMAGE_TAR"
+        else
+            docker build --no-cache -t crossplane-provider-proxmox:${NEW_TAG} "$PROJECT_DIR" 2>&1 | tail -10
+            docker save crossplane-provider-proxmox:${NEW_TAG} -o "$NEW_IMAGE_TAR"
+        fi
+        echo "  ✅ Image rebuilt and saved"
+    else
+        echo "  ❌ ERROR: Project directory not found at $PROJECT_DIR"
+        exit 1
+    fi
+else
+    echo "  ✅ Image tar already exists"
+    # Remove other old tar files but keep the new one
+    find /tmp -name "crossplane-*.tar" ! -name "crossplane-v1.0.1-fixed.tar" -delete 2>/dev/null || true
+fi
+echo ""
+
+echo "Step 7: Verifying new image tar exists..."
+if [ ! -f "$NEW_IMAGE_TAR" ]; then
+    echo "  ❌ ERROR: New image tar not found at $NEW_IMAGE_TAR"
+    exit 1
+fi
+echo "  ✅ New image tar found: $(ls -lh $NEW_IMAGE_TAR | awk '{print $5}')"
+echo ""
+
+echo "Step 8: Importing NEW image to containerd..."
+if [ -n "$KIND_NODE" ]; then
+    echo "  Copying tar into kind node container (using stdin method)..."
+    docker exec -i "$KIND_NODE" sh -c "cat > /tmp/$(basename $NEW_IMAGE_TAR)" < "$NEW_IMAGE_TAR"
+    echo "  Importing into kind node's containerd..."
+    ctr_cmd -n k8s.io images import "/tmp/$(basename $NEW_IMAGE_TAR)"
+else
+    ctr_cmd -n k8s.io images import "$NEW_IMAGE_TAR"
+fi
+echo "  ✅ New image imported"
+echo ""
+
+echo "Step 9: Verifying import and tagging correctly..."
+# Get the imported image (it might have a different name from docker save)
+IMPORTED_RAW=$(ctr_cmd -n k8s.io images ls -q | grep crossplane | head -1 || echo "")
+if [ -z "$IMPORTED_RAW" ]; then
+    echo "  ❌ ERROR: No crossplane image found after import!"
+    echo "  All images in containerd:"
+    ctr_cmd -n k8s.io images ls -q | head -5
+    exit 1
+fi
+
+# Tag it with the exact name Kubernetes expects (fully qualified)
+EXPECTED_IMAGE="docker.io/library/${IMAGE_NAME}:${NEW_TAG}"
+echo "  Tagging imported image ($IMPORTED_RAW) as ${EXPECTED_IMAGE}..."
+ctr_cmd -n k8s.io images tag "$IMPORTED_RAW" "${EXPECTED_IMAGE}" 2>/dev/null || {
+    echo "  ⚠️  Tagging failed, checking if tag already exists..."
+    # Check if the expected image already exists
+    EXISTING=$(ctr_cmd -n k8s.io images ls -q | grep -E "(^|/)${EXPECTED_IMAGE}$" || echo "")
+    if [ -n "$EXISTING" ]; then
+        echo "  ✅ Expected image already exists: $EXISTING"
+    else
+        echo "  ❌ ERROR: Could not tag image. Trying to remove old tag and retag..."
+        ctr_cmd -n k8s.io images rm "${EXPECTED_IMAGE}" 2>/dev/null || true
+        ctr_cmd -n k8s.io images tag "$IMPORTED_RAW" "${EXPECTED_IMAGE}" || {
+            echo "  ❌ ERROR: Failed to tag image as ${EXPECTED_IMAGE}"
+            exit 1
+        }
+    fi
+}
+
+# Verify the expected image exists
+IMPORTED=$(ctr_cmd -n k8s.io images ls -q | grep -E "(^|/)${EXPECTED_IMAGE}$" || true)
+if [ -z "$IMPORTED" ]; then
+    echo "  ❌ ERROR: Image ${EXPECTED_IMAGE} not found after tagging!"
+    echo "  Available crossplane images:"
+    ctr_cmd -n k8s.io images ls -q | grep crossplane || echo "  (none)"
+    exit 1
+fi
+echo "  ✅ Image verified: $IMPORTED"
+echo ""
+
+echo "Step 10: Final containerd image list..."
+ctr_cmd -n k8s.io images ls | grep crossplane-provider-proxmox || echo "  (No images found - this is wrong!)"
+echo ""
+
+fi  # Close the if [ "$SKIP_TO_STEP_12" = false ] block
+
+if [ -n "$KUBECTL" ]; then
+    # Set up kubectl command with proper user context using a function
+    ORIG_USER="${SUDO_USER:-${USER}}"
+    if [ -n "$ORIG_USER" ] && [ "$ORIG_USER" != "root" ] && [ "$EUID" -eq 0 ]; then
+        KUBECONFIG="${KUBECONFIG:-/home/$ORIG_USER/.kube/config}"
+        if [ -f "$KUBECONFIG" ]; then
+            # Use a function to properly handle env vars and command
+            kubectl_cmd() {
+                sudo -u "$ORIG_USER" env KUBECONFIG="$KUBECONFIG" "$KUBECTL" "$@"
+            }
+        else
+            kubectl_cmd() {
+                sudo -u "$ORIG_USER" "$KUBECTL" "$@"
+            }
+        fi
+    else
+        kubectl_cmd() {
+            "$KUBECTL" "$@"
+        }
+    fi
+    # kubectl_cmd function is now defined and ready to use
+
+    if [ "$SKIP_TO_STEP_12" = false ]; then
+        echo "Step 11: Applying deployment with new image tag..."
+        kubectl_cmd apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml
+        echo "  ✅ Deployment applied"
+        echo ""
+    fi
+
+    # Step 12: Wait for pod with retry loop and verbose feedback (always runs)
+    echo "Step 12: Waiting for pod to be ready (with retry loop)..."
+    
+    # First, check if deployment exists, create it if not
+    echo "  Checking if deployment exists..."
+    DEPLOYMENT_EXISTS=$(kubectl_cmd get deployment crossplane-provider-proxmox -n crossplane-system 2>/dev/null && echo "yes" || echo "no")
+    if [ "$DEPLOYMENT_EXISTS" = "no" ]; then
+        echo "  ⚠️  Deployment not found! Creating it now..."
+        kubectl_cmd apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml
+        echo "  ✅ Deployment created, waiting 5s for pod to start..."
+        sleep 5
+    else
+        echo "  ✅ Deployment exists"
+    fi
+    
+    MAX_RETRIES=30
+    RETRY_DELAY=5
+    RETRY_COUNT=0
+    POD_READY=false
+
+    while [ $RETRY_COUNT -lt $MAX_RETRIES ]; do
+        RETRY_COUNT=$((RETRY_COUNT + 1))
+        echo ""
+        echo "  Attempt $RETRY_COUNT/$MAX_RETRIES: Checking pod status..."
+        
+        # Get pod status - handle case where no pods exist yet
+        POD_COUNT=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox --no-headers 2>/dev/null | wc -l || echo "0")
+        if [ "$POD_COUNT" = "0" ] || [ -z "$POD_COUNT" ]; then
+            POD_STATUS="NOT_FOUND"
+            POD_READY_STATUS="Unknown"
+            POD_CONTAINER_STATUS=""
+            POD_IMAGE="Unknown"
+        else
+            POD_STATUS=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "NOT_FOUND")
+            POD_READY_STATUS=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown")
+            POD_CONTAINER_STATUS=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].state.waiting.reason}' 2>/dev/null || echo "")
+            POD_IMAGE=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].spec.containers[0].image}' 2>/dev/null || echo "Unknown")
+        fi
+        
+        echo "    Pod Phase: $POD_STATUS"
+        echo "    Ready Status: $POD_READY_STATUS"
+        echo "    Container Image: $POD_IMAGE"
+        
+        if [ -n "$POD_CONTAINER_STATUS" ] && [ "$POD_CONTAINER_STATUS" != "null" ]; then
+            echo "    Container Waiting Reason: $POD_CONTAINER_STATUS"
+            
+            # Check for specific errors
+            if [ "$POD_CONTAINER_STATUS" = "ErrImageNeverPull" ]; then
+                echo "    ⚠️  ERROR: Image not found in containerd!"
+                echo "    Checking if image exists in containerd..."
+                # Check for image with exact name Kubernetes expects (fully qualified)
+                EXPECTED_IMAGE="docker.io/library/${IMAGE_NAME}:${NEW_TAG}"
+                CONTAINERD_IMAGE=$(ctr_cmd -n k8s.io images ls -q | grep -E "(^|/)${EXPECTED_IMAGE}$" || echo "")
+                
+                if [ -z "$CONTAINERD_IMAGE" ]; then
+                    echo "    ❌ Image ${EXPECTED_IMAGE} NOT found in containerd."
+                    echo "    Checking for any crossplane images..."
+                    ALL_CROSSPLANE=$(ctr_cmd -n k8s.io images ls -q | grep crossplane || echo "")
+                    if [ -n "$ALL_CROSSPLANE" ]; then
+                        echo "    Found other crossplane images:"
+                        echo "$ALL_CROSSPLANE" | head -3 | sed 's/^/      /'
+                        echo "    Re-importing and tagging correctly..."
+                    fi
+                    
+                    if [ -f "$NEW_IMAGE_TAR" ]; then
+                        echo "    Importing image from $NEW_IMAGE_TAR..."
+                        if [ -n "$KIND_NODE" ]; then
+                            echo "    Copying tar into kind node (using stdin method)..."
+                            docker exec -i "$KIND_NODE" sh -c "cat > /tmp/$(basename $NEW_IMAGE_TAR)" < "$NEW_IMAGE_TAR"
+                            ctr_cmd -n k8s.io images import "/tmp/$(basename $NEW_IMAGE_TAR)"
+                        else
+                            ctr_cmd -n k8s.io images import "$NEW_IMAGE_TAR"
+                        fi
+                        
+                        # Get the imported image ID and tag it correctly
+                        IMPORTED_ID=$(ctr_cmd -n k8s.io images ls -q | grep crossplane | head -1)
+                        if [ -n "$IMPORTED_ID" ]; then
+                            echo "    Tagging imported image as ${EXPECTED_IMAGE}..."
+                            ctr_cmd -n k8s.io images tag "$IMPORTED_ID" "${EXPECTED_IMAGE}" 2>/dev/null || true
+                            echo "    ✅ Image imported and tagged. Restarting pod..."
+                        else
+                            echo "    ⚠️  Could not find imported image, but continuing..."
+                        fi
+                        echo "    Restarting deployment once (no pod churn)..."
+                        kubectl_cmd rollout restart deployment/crossplane-provider-proxmox -n crossplane-system
+                        sleep 5
+                    else
+                        echo "    ❌ Image tar not found at $NEW_IMAGE_TAR"
+                        echo "    Run script without --resume to rebuild image"
+                        exit 1
+                    fi
+                else
+                    echo "    ⚠️  Image exists in containerd but pod can't find it"
+                    echo "    Image in containerd: $CONTAINERD_IMAGE"
+                    echo "    Expected by pod: ${EXPECTED_IMAGE}"
+                    echo "    Attempting to tag existing image with correct name..."
+                    # Try to tag the existing image with the exact name Kubernetes expects
+                    ctr_cmd -n k8s.io images tag "$CONTAINERD_IMAGE" "${EXPECTED_IMAGE}" 2>/dev/null || {
+                        echo "    ⚠️  Tagging failed, trying to re-import..."
+                        if [ -f "$NEW_IMAGE_TAR" ]; then
+                            if [ -n "$KIND_NODE" ]; then
+                                docker exec -i "$KIND_NODE" sh -c "cat > /tmp/$(basename $NEW_IMAGE_TAR)" < "$NEW_IMAGE_TAR"
+                                ctr_cmd -n k8s.io images import "/tmp/$(basename $NEW_IMAGE_TAR)"
+                            else
+                                ctr_cmd -n k8s.io images import "$NEW_IMAGE_TAR"
+                            fi
+                            IMPORTED_ID=$(ctr_cmd -n k8s.io images ls -q | grep crossplane | head -1)
+                            if [ -n "$IMPORTED_ID" ]; then
+                                ctr_cmd -n k8s.io images tag "$IMPORTED_ID" "${EXPECTED_IMAGE}" 2>/dev/null || true
+                            fi
+                        fi
+                    }
+                    echo "    Restarting deployment once to refresh..."
+                    kubectl_cmd rollout restart deployment/crossplane-provider-proxmox -n crossplane-system
+                    sleep 5
+                fi
+            fi
+        fi
+        
+        # Check if pod is ready
+        if [ "$POD_READY_STATUS" = "True" ]; then
+            echo "    ✅ Pod is READY!"
+            POD_READY=true
+            break
+        fi
+        
+        # Show detailed pod info
+        if [ "$POD_COUNT" != "0" ] && [ -n "$POD_COUNT" ]; then
+            echo "    Pod details:"
+            kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o wide 2>/dev/null | tail -1
+        else
+            echo "    No pods found yet - deployment may still be creating pod..."
+        fi
+        
+        if [ $RETRY_COUNT -lt $MAX_RETRIES ]; then
+            echo "    Waiting ${RETRY_DELAY}s before next check..."
+            sleep $RETRY_DELAY
+        fi
+    done
+
+    if [ "$POD_READY" = false ]; then
+        echo ""
+        echo "  ❌ Pod did not become ready after $MAX_RETRIES attempts"
+        echo "  Final pod status:"
+        if [ "$POD_COUNT" != "0" ] && [ -n "$POD_COUNT" ]; then
+            kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o yaml 2>/dev/null | grep -A 10 "status:" | head -15 || echo "  (Could not get pod details)"
+        else
+            echo "  No pods found. Checking deployment status:"
+            kubectl_cmd get deployment crossplane-provider-proxmox -n crossplane-system 2>/dev/null || echo "  Deployment also not found!"
+        fi
+        echo ""
+        echo "  To retry from this point, run:"
+        echo "    sudo $0 --resume-from-step12"
+        exit 1
+    fi
+
+    echo ""
+    echo "  ✅ Pod is ready!"
+    echo ""
+
+    echo "Step 13: Checking pod image..."
+    POD_IMAGE=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].spec.containers[0].image}' || echo "")
+    POD_IMAGE_ID=$(kubectl_cmd get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].imageID}' || echo "")
+    echo "  Pod image: $POD_IMAGE"
+    echo "  Pod image ID: $POD_IMAGE_ID"
+    if echo "$POD_IMAGE" | grep -q "${NEW_TAG}"; then
+        echo "  ✅ Pod is using NEW image tag!"
+    else
+        echo "  ⚠️  Pod is NOT using new tag!"
+    fi
+    echo ""
+
+    echo "Step 14: Waiting for logs to generate..."
+    sleep 5
+    echo ""
+
+    echo "Step 15: Checking for [FIXED CODE] message..."
+    if kubectl_cmd logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 2>&1 | grep -q '\[FIXED CODE\]'; then
+        echo "  ✅ [FIXED CODE] message found! Fix is ACTIVE!"
+        kubectl_cmd logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 | grep '\[FIXED CODE\]' | head -3
+    else
+        echo "  ❌ [FIXED CODE] message NOT found!"
+        echo ""
+        echo "  Recent logs:"
+        kubectl_cmd logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=20 | grep -E "Cookie|Authorization|FIXED|DEBUG" | head -5 || echo "  (No relevant logs)"
+    fi
+fi
+
+if [ -z "$KUBECTL" ]; then
+    echo "Step 11: Skipping Kubernetes operations (kubectl not found)"
+    echo ""
+    echo "  Please manually run:"
+    echo "    kubectl apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml"
+    echo "    kubectl wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s"
+    echo "    kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox | grep '\[FIXED CODE\]'"
+fi
+
+echo ""
+echo "=========================================="
+echo "CLEANUP AND DEPLOY COMPLETE"
+echo "=========================================="
+
+

scripts/complete-image-fix.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+set -e
+
+echo "=========================================="
+echo "COMPLETE IMAGE FIX - Full Cleanup & Rebuild"
+echo "=========================================="
+echo ""
+
+# Step 1: Delete deployment
+echo "Step 1: Deleting deployment..."
+kubectl delete deployment crossplane-provider-proxmox -n crossplane-system 2>/dev/null || echo "  (Deployment not found)"
+sleep 3
+
+# Step 2: Remove ALL crossplane-provider-proxmox images from containerd
+echo ""
+echo "Step 2: Removing ALL crossplane-provider-proxmox images from containerd..."
+ALL_IMAGES=$(sudo ctr -n k8s.io images ls -q | grep crossplane-provider-proxmox || true)
+if [ -n "$ALL_IMAGES" ]; then
+    echo "$ALL_IMAGES" | while read -r img; do
+        echo "  Removing: $img"
+        sudo ctr -n k8s.io images rm "$img" 2>/dev/null || true
+    done
+    echo "  ✅ All images removed"
+else
+    echo "  (No images found)"
+fi
+
+# Step 3: Verify image tar exists
+echo ""
+echo "Step 3: Verifying image tar exists..."
+if [ ! -f /tmp/crossplane-fresh-nuclear.tar ]; then
+    echo "  ❌ ERROR: Image tar not found at /tmp/crossplane-fresh-nuclear.tar"
+    echo "  Please rebuild the image first using:"
+    echo "    sudo /home/intlc/projects/Sankofa/scripts/nuclear-cleanup-rebuild.sh"
+    exit 1
+fi
+echo "  ✅ Image tar found"
+
+# Step 4: Re-import fresh image
+echo ""
+echo "Step 4: Re-importing fresh image..."
+sudo ctr -n k8s.io images import /tmp/crossplane-fresh-nuclear.tar
+echo "  ✅ Image imported"
+
+# Step 5: Verify import
+echo ""
+echo "Step 5: Verifying import..."
+IMPORTED=$(sudo ctr -n k8s.io images ls -q | grep "crossplane-provider-proxmox:latest" || true)
+if [ -z "$IMPORTED" ]; then
+    echo "  ❌ ERROR: Image not found after import!"
+    exit 1
+fi
+echo "  ✅ Image verified: $IMPORTED"
+
+# Step 6: Show image details
+echo ""
+echo "Step 6: Image details:"
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox
+
+# Step 7: Recreate deployment
+echo ""
+echo "Step 7: Recreating deployment..."
+kubectl apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml
+
+# Step 8: Wait for pod
+echo ""
+echo "Step 8: Waiting for pod to be ready..."
+kubectl wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s || {
+    echo "  ⚠️  Pod not ready within timeout, checking status..."
+    kubectl get pod -n crossplane-system -l app=crossplane-provider-proxmox
+    exit 1
+}
+echo "  ✅ Pod is ready"
+
+# Step 9: Check image ID
+echo ""
+echo "Step 9: Checking pod image ID..."
+POD_IMAGE_ID=$(kubectl get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].imageID}')
+echo "  Pod image ID: $POD_IMAGE_ID"
+
+# Step 10: Wait a moment for logs
+echo ""
+echo "Step 10: Waiting for logs to generate..."
+sleep 5
+
+# Step 11: Check for [FIXED CODE] message
+echo ""
+echo "Step 11: Checking for [FIXED CODE] message in logs..."
+if kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 | grep -q '\[FIXED CODE\]'; then
+    echo "  ✅ [FIXED CODE] message found! Fix is active."
+else
+    echo "  ⚠️  [FIXED CODE] message NOT found. Checking logs..."
+    kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=20 | grep -E "Cookie|Authorization|FIXED" || echo "  (No relevant log lines found)"
+fi
+
+echo ""
+echo "=========================================="
+echo "COMPLETE IMAGE FIX FINISHED"
+echo "=========================================="
+echo ""
+echo "To verify the fix is working, check logs:"
+echo "  kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox | grep '\[FIXED CODE\]'"
+echo ""
+echo "The Cookie header should be removed from requests when using token auth."
+

scripts/comprehensive-status-check.sh

@@ -0,0 +1,181 @@
+#!/bin/bash
+# Comprehensive Status Check
+# Verifies all systems, checks for issues, and reports status
+
+set -e
+
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+PASSED=0
+FAILED=0
+WARNINGS=0
+
+echo "=========================================="
+echo "Comprehensive Status Check"
+echo "=========================================="
+echo ""
+
+# 1. Provider Status
+echo -e "${BLUE}1. Provider Status${NC}"
+POD_STATUS=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "NOT_FOUND")
+if [[ "$POD_STATUS" == "Running" ]]; then
+    READY=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].ready}' 2>/dev/null)
+    if [[ "$READY" == "true" ]]; then
+        echo -e "${GREEN}✓${NC} Provider pod running and ready"
+        ((PASSED++))
+    else
+        echo -e "${YELLOW}⚠${NC} Provider pod running but not ready"
+        ((WARNINGS++))
+    fi
+else
+    echo -e "${RED}✗${NC} Provider pod not running (status: $POD_STATUS)"
+    ((FAILED++))
+fi
+echo ""
+
+# 2. Provider Logs - Critical Errors
+echo -e "${BLUE}2. Provider Logs - Critical Errors${NC}"
+CRITICAL_ERRORS=$(kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=100 2>&1 | grep -i -E "error.*timeout|error.*authentication|error.*connection" | grep -v "CRD" | wc -l)
+if [[ $CRITICAL_ERRORS -eq 0 ]]; then
+    echo -e "${GREEN}✓${NC} No critical errors in logs"
+    ((PASSED++))
+else
+    echo -e "${YELLOW}⚠${NC} $CRITICAL_ERRORS critical errors found (may be transient)"
+    ((WARNINGS++))
+fi
+echo ""
+
+# 3. VM Status
+echo -e "${BLUE}3. VM Status${NC}"
+TOTAL_VMS=$(kubectl get proxmoxvm -A --no-headers 2>&1 | wc -l)
+FAILED_VMS=$(kubectl get proxmoxvm -A -o json 2>&1 | jq -r '.items[] | select(.status.conditions[]?.type=="Failed" or .status.conditions[]?.type=="ValidationFailed") | .metadata.name' 2>/dev/null | wc -l)
+VMS_WITH_ID=$(kubectl get proxmoxvm -A -o jsonpath='{range .items[*]}{.status.vmId}{"\n"}{end}' 2>&1 | grep -v "^$" | wc -l)
+
+echo "  Total VMs: $TOTAL_VMS"
+echo "  VMs with VMID: $VMS_WITH_ID"
+echo "  Failed VMs: $FAILED_VMS"
+
+if [[ $FAILED_VMS -eq 0 ]]; then
+    echo -e "${GREEN}✓${NC} No failed VMs"
+    ((PASSED++))
+else
+    echo -e "${RED}✗${NC} $FAILED_VMS VMs have failed"
+    ((FAILED++))
+fi
+
+if [[ $VMS_WITH_ID -gt 0 ]]; then
+    echo -e "${GREEN}✓${NC} VMs are being created on Proxmox"
+    ((PASSED++))
+else
+    echo -e "${YELLOW}⚠${NC} VMs pending creation (provider processing)"
+    ((WARNINGS++))
+fi
+echo ""
+
+# 4. Connectivity
+echo -e "${BLUE}4. Network Connectivity${NC}"
+if ping -c 2 192.168.11.10 &>/dev/null; then
+    echo -e "${GREEN}✓${NC} ML110-01 reachable"
+    ((PASSED++))
+else
+    echo -e "${RED}✗${NC} ML110-01 not reachable"
+    ((FAILED++))
+fi
+
+if ping -c 2 192.168.11.11 &>/dev/null; then
+    echo -e "${GREEN}✓${NC} R630-01 reachable"
+    ((PASSED++))
+else
+    echo -e "${RED}✗${NC} R630-01 not reachable"
+    ((FAILED++))
+fi
+echo ""
+
+# 5. Credentials
+echo -e "${BLUE}5. Credentials${NC}"
+if kubectl get secret proxmox-credentials -n crossplane-system &>/dev/null; then
+    KEYS=$(kubectl get secret proxmox-credentials -n crossplane-system -o jsonpath='{.data}' 2>&1 | jq -r 'keys[]' 2>/dev/null || echo "")
+    if [[ "$KEYS" == *"token"* && "$KEYS" == *"tokenid"* ]]; then
+        echo -e "${GREEN}✓${NC} Credentials secret exists with token format"
+        ((PASSED++))
+    else
+        echo -e "${YELLOW}⚠${NC} Credentials format may be incorrect"
+        ((WARNINGS++))
+    fi
+else
+    echo -e "${RED}✗${NC} Credentials secret not found"
+    ((FAILED++))
+fi
+echo ""
+
+# 6. Provider Config
+echo -e "${BLUE}6. Provider Configuration${NC}"
+if kubectl get providerconfig proxmox-provider-config -n crossplane-system &>/dev/null; then
+    SITES=$(kubectl get providerconfig proxmox-provider-config -n crossplane-system -o jsonpath='{.spec.sites[*].name}' 2>/dev/null)
+    if [[ "$SITES" == *"site-1"* && "$SITES" == *"site-2"* ]]; then
+        echo -e "${GREEN}✓${NC} ProviderConfig exists with both sites"
+        ((PASSED++))
+    else
+        echo -e "${RED}✗${NC} Sites not correctly configured: $SITES"
+        ((FAILED++))
+    fi
+else
+    echo -e "${RED}✗${NC} ProviderConfig not found"
+    ((FAILED++))
+fi
+echo ""
+
+# 7. YAML Syntax
+echo -e "${BLUE}7. YAML Syntax Validation${NC}"
+YAML_ERRORS=0
+for file in examples/production/phoenix/*.yaml; do
+    if ! kubectl apply --dry-run=client -f "$file" &>/dev/null; then
+        ((YAML_ERRORS++))
+    fi
+done
+
+if [[ $YAML_ERRORS -eq 0 ]]; then
+    echo -e "${GREEN}✓${NC} All YAML files valid"
+    ((PASSED++))
+else
+    echo -e "${RED}✗${NC} $YAML_ERRORS YAML files have syntax errors"
+    ((FAILED++))
+fi
+echo ""
+
+# 8. CRD Warnings
+echo -e "${BLUE}8. CRD Warnings (Non-Critical)${NC}"
+CRD_WARNINGS=$(kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 2>&1 | grep -c "CRD.*should be installed" || echo "0")
+if [[ $CRD_WARNINGS -gt 0 ]]; then
+    echo -e "${YELLOW}⚠${NC} $CRD_WARNINGS CRD warnings (non-critical, log noise only)"
+    ((WARNINGS++))
+else
+    echo -e "${GREEN}✓${NC} No CRD warnings"
+    ((PASSED++))
+fi
+echo ""
+
+# Summary
+echo "=========================================="
+echo "Summary"
+echo "=========================================="
+echo -e "${GREEN}Passed:${NC} $PASSED"
+echo -e "${YELLOW}Warnings:${NC} $WARNINGS"
+echo -e "${RED}Failed:${NC} $FAILED"
+echo ""
+
+if [[ $FAILED -eq 0 && $WARNINGS -eq 0 ]]; then
+    echo -e "${GREEN}✓ All checks passed!${NC}"
+    exit 0
+elif [[ $FAILED -eq 0 ]]; then
+    echo -e "${YELLOW}⚠ All critical checks passed with minor warnings${NC}"
+    exit 0
+else
+    echo -e "${RED}✗ Critical issues found${NC}"
+    exit 1
+fi
+

scripts/continuous-monitor.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+# Continuous VM Deployment Monitor
+# Monitors VM deployment progress and provides real-time updates
+
+set -e
+
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+RED='\033[0;31m'
+NC='\033[0m'
+
+INTERVAL=${1:-30}  # Default 30 seconds
+
+echo "=========================================="
+echo "VM Deployment Continuous Monitor"
+echo "=========================================="
+echo "Update interval: ${INTERVAL} seconds"
+echo "Press Ctrl+C to stop"
+echo ""
+
+PREV_COUNT=0
+ITERATION=0
+
+while true; do
+    ITERATION=$((ITERATION + 1))
+    TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
+    
+    # Get counts
+    TOTAL=$(kubectl get proxmoxvm -A --no-headers 2>&1 | wc -l)
+    WITH_VMID=$(kubectl get proxmoxvm -A -o jsonpath='{range .items[*]}{.status.vmId}{"\n"}{end}' 2>&1 | grep -v "^$" | wc -l)
+    PENDING=$((TOTAL - WITH_VMID))
+    NEW_VMS=$((WITH_VMID - PREV_COUNT))
+    
+    # Provider status
+    PROVIDER_STATUS=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.phase}' 2>&1 || echo "Unknown")
+    
+    # Check for errors
+    AUTH_ERRORS=$(kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --since=1m 2>&1 | grep -i "invalid PVE ticket\|401.*authentication" | wc -l)
+    
+    echo -e "${BLUE}[$TIMESTAMP] Iteration $ITERATION${NC}"
+    echo "  Provider: $PROVIDER_STATUS"
+    echo "  Total VMs: $TOTAL"
+    echo -e "  ${GREEN}Created (with VMID): $WITH_VMID${NC}"
+    echo -e "  ${YELLOW}Pending: $PENDING${NC}"
+    
+    if [ $NEW_VMS -gt 0 ]; then
+        echo -e "  ${GREEN}✨ New VMs created: +$NEW_VMS${NC}"
+    fi
+    
+    if [ $AUTH_ERRORS -gt 0 ]; then
+        echo -e "  ${RED}⚠️  Authentication errors: $AUTH_ERRORS${NC}"
+    else
+        echo -e "  ${GREEN}✅ No authentication errors${NC}"
+    fi
+    
+    # Progress percentage
+    if [ $TOTAL -gt 0 ]; then
+        PERCENT=$((WITH_VMID * 100 / TOTAL))
+        echo "  Progress: $PERCENT% ($WITH_VMID/$TOTAL)"
+    fi
+    
+    # Show some VMs with VMID
+    if [ $WITH_VMID -gt 0 ]; then
+        echo ""
+        echo "  Recent VMs with VMID:"
+        kubectl get proxmoxvm -A -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.vmId}{"\n"}{end}' 2>&1 | grep -v "\t$" | tail -5 | sed 's/^/    /'
+    fi
+    
+    PREV_COUNT=$WITH_VMID
+    
+    echo ""
+    echo "----------------------------------------"
+    sleep $INTERVAL
+done
+

scripts/copy-image-to-local-lvm.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+# Copy cloud image from ISO storage to local-lvm storage
+
+set -euo pipefail
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  COPY IMAGE TO local-lvm STORAGE"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+SOURCE_IMAGE="/var/lib/vz/template/iso/ubuntu-22.04-cloud.img"
+TARGET_STORAGE="local-lvm"
+IMAGE_NAME="ubuntu-22.04-cloud"
+
+# Check if source exists
+echo "Step 1: Checking if source image exists..."
+if [ ! -f "$SOURCE_IMAGE" ]; then
+    echo "❌ Source image not found: $SOURCE_IMAGE"
+    exit 1
+fi
+
+IMAGE_SIZE=$(du -h "$SOURCE_IMAGE" | cut -f1)
+echo "✅ Source image found: $SOURCE_IMAGE ($IMAGE_SIZE)"
+echo ""
+
+# Check if target storage exists
+echo "Step 2: Checking if target storage exists..."
+if ! pvesm status | grep -q "^${TARGET_STORAGE}"; then
+    echo "❌ Target storage '$TARGET_STORAGE' not found"
+    exit 1
+fi
+echo "✅ Target storage '$TARGET_STORAGE' exists"
+echo ""
+
+# Check if image already exists in local-lvm
+echo "Step 3: Checking if image already exists in local-lvm..."
+if pvesm list "$TARGET_STORAGE" 2>/dev/null | grep -q "${IMAGE_NAME}"; then
+    echo "⚠️  Image already exists in local-lvm"
+    echo "   Skipping copy (image is ready to use)"
+    exit 0
+fi
+echo "✅ Image not found in local-lvm (will copy)"
+echo ""
+
+# Copy image using pvesm copy
+echo "Step 4: Copying image to local-lvm storage..."
+echo "   This may take a few minutes for a 660MB image..."
+echo ""
+
+# Use pvesm copy to copy from local:iso/ to local-lvm
+SOURCE_VOLID="local:iso/ubuntu-22.04-cloud.img"
+TARGET_VOLID="${TARGET_STORAGE}:${IMAGE_NAME}.img"
+
+echo "   Copying from $SOURCE_VOLID to $TARGET_VOLID..."
+if pvesm copy "$SOURCE_VOLID" "$TARGET_VOLID" 2>&1; then
+    echo "✅ Copy successful"
+else
+    echo "⚠️  pvesm copy failed, trying alternative method..."
+    
+    # Alternative: Use qm disk import with a temporary VM
+    TEMP_VMID=9999
+    
+    # Clean up any existing temp VM
+    qm destroy "$TEMP_VMID" 2>/dev/null || true
+    
+    # Create a temporary VM for import
+    echo "   Creating temporary VM for import..."
+    qm create "$TEMP_VMID" \
+        --name "temp-import" \
+        --memory 512 \
+        --net0 virtio,bridge=vmbr0 \
+        --agent enabled=1 2>/dev/null || true
+    
+    # Import the disk
+    echo "   Importing disk..."
+    if qm disk import "$TEMP_VMID" "$SOURCE_IMAGE" "$TARGET_STORAGE" --format qcow2 2>&1; then
+        echo "✅ Import successful"
+        # Clean up temp VM
+        qm destroy "$TEMP_VMID" 2>/dev/null || true
+    else
+        echo "❌ Import failed"
+        # Clean up temp VM
+        qm destroy "$TEMP_VMID" 2>/dev/null || true
+        exit 1
+    fi
+fi
+
+# Verify import
+echo ""
+echo "Step 5: Verifying import..."
+sleep 2  # Give Proxmox a moment to refresh
+if pvesm list "$TARGET_STORAGE" 2>/dev/null | grep -q "${IMAGE_NAME}"; then
+    echo "✅ Image found in local-lvm storage"
+else
+    echo "⚠️  Image copied but not visible in pvesm list yet"
+    echo "   This is normal - Proxmox may need a moment to refresh"
+    echo "   The image should be available shortly"
+fi
+
+echo ""
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  VERIFICATION"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+echo "Checking local-lvm storage contents..."
+pvesm list "$TARGET_STORAGE" 2>/dev/null | grep -E "ubuntu-22.04|IMAGE_NAME" | head -5 || echo "Image should be available soon"
+echo ""
+echo "✅ Setup complete! Image should now be available in local-lvm storage."
+echo ""
+echo "📝 Next Steps:"
+echo "   The provider will automatically discover the image when creating VMs"
+

scripts/copy-image-to-storage.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+# Copy cloud image from ISO storage to image-capable storage
+# This fixes the issue where images in local:iso/ cannot be used as VM disks
+
+set -e
+
+NODE="r630-01"
+SOURCE_IMAGE="local:iso/ubuntu-22.04-cloud.img"
+TARGET_STORAGE="${1:-ceph-rbd}"  # Default to ceph-rbd, or pass as first argument
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  COPY CLOUD IMAGE TO IMAGE STORAGE"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+# Check if running on r630-01
+HOSTNAME=$(hostname)
+if [ "$HOSTNAME" != "r630-01" ]; then
+    echo "⚠️  WARNING: This script should be run on r630-01"
+    echo "   Current hostname: $HOSTNAME"
+    echo ""
+    echo "To run remotely via SSH:"
+    echo "  ssh root@192.168.11.11 'bash -s' < $0"
+    echo ""
+    read -p "Continue anyway? (y/N) " -n 1 -r
+    echo
+    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+        exit 1
+    fi
+fi
+
+echo "Step 1: Checking if image exists in ISO storage..."
+if ! pvesm list local 2>/dev/null | grep -q "ubuntu-22.04-cloud"; then
+    echo "⚠️  Image not found: $SOURCE_IMAGE"
+    echo "   Listing available images in local storage:"
+    pvesm list local 2>/dev/null | grep -E "\.img$|\.qcow2$" | head -10 || echo "   No images found in local storage"
+    exit 1
+fi
+
+echo "✅ Image found: $SOURCE_IMAGE"
+echo ""
+
+echo "Step 2: Extracting filename..."
+FILENAME=$(echo "$SOURCE_IMAGE" | sed 's/.*\///')
+TARGET_PATH="template/cache/$FILENAME"
+TARGET_VOLID="$TARGET_STORAGE:$TARGET_PATH"
+
+echo "   Source: $SOURCE_IMAGE"
+echo "   Target: $TARGET_VOLID"
+echo ""
+
+echo "Step 3: Copying image to $TARGET_STORAGE storage..."
+# Method 1: Use qm disk import (recommended)
+if command -v qm &> /dev/null; then
+    echo "   Using qm disk import method..."
+    # Create a temporary VM to import the disk
+    TEMP_VMID=9999
+    echo "   Creating temporary VM (ID: $TEMP_VMID) for import..."
+    qm create $TEMP_VMID --name temp-import --memory 128 --net0 virtio,bridge=vmbr0 2>/dev/null || {
+        # VM might already exist, try to destroy it first
+        qm destroy $TEMP_VMID 2>/dev/null || true
+        sleep 2
+        qm create $TEMP_VMID --name temp-import --memory 128 --net0 virtio,bridge=vmbr0 2>/dev/null || {
+            echo "❌ Failed to create temporary VM"
+            exit 1
+        }
+    }
+    
+    # Import the disk from ISO storage to target storage
+    echo "   Importing disk from $SOURCE_IMAGE to $TARGET_STORAGE..."
+    if qm disk import $TEMP_VMID $SOURCE_IMAGE $TARGET_STORAGE --format qcow2 2>&1; then
+        echo "   ✅ Disk imported successfully"
+        
+        # Get the imported disk volid
+        IMPORTED_DISK=$(qm config $TEMP_VMID 2>/dev/null | grep -E "^scsi[0-9]+:" | head -1 | cut -d: -f2 | cut -d, -f1 | tr -d ' ')
+        if [ -n "$IMPORTED_DISK" ]; then
+            echo "   Imported disk: $IMPORTED_DISK"
+        fi
+        
+        # Clean up temp VM
+        echo "   Cleaning up temporary VM..."
+        qm destroy $TEMP_VMID 2>/dev/null || true
+        echo "✅ Image copied successfully to $TARGET_STORAGE"
+    else
+        echo "⚠️  qm disk import failed, trying alternative method..."
+        # Clean up temp VM first
+        qm destroy $TEMP_VMID 2>/dev/null || true
+        
+        # Method 2: Use rbd import directly (for RBD storage)
+        if [ "$TARGET_STORAGE" = "ceph-rbd" ] || [ "$TARGET_STORAGE" = "rbd" ]; then
+            echo "   Using rbd import method for RBD storage..."
+            SOURCE_PATH="/var/lib/vz/template/iso/$(basename $SOURCE_IMAGE | sed 's/local:iso\///')"
+            RBD_IMAGE_NAME="template_cache_$(basename $SOURCE_IMAGE | sed 's/\.img$//' | tr '[:upper:]' '[:lower:]' | tr -cd '[:alnum:]_')"
+            
+            if [ -f "$SOURCE_PATH" ]; then
+                echo "   Importing image to RBD pool 'rbd' as '$RBD_IMAGE_NAME'..."
+                if rbd import "$SOURCE_PATH" "rbd/$RBD_IMAGE_NAME" --image-format 2 2>&1; then
+                    echo "   ✅ Image imported to RBD pool"
+                    echo "   Note: Proxmox will need to discover this image"
+                    echo "   The image is now in RBD pool 'rbd' as '$RBD_IMAGE_NAME'"
+                else
+                    echo "❌ RBD import failed"
+                    exit 1
+                fi
+            else
+                echo "❌ Source file not found at $SOURCE_PATH"
+                exit 1
+            fi
+        else
+            # Method 3: Use direct file copy (for local storage types)
+            echo "   Trying direct file copy method..."
+            SOURCE_PATH="/var/lib/vz/template/iso/$(basename $SOURCE_IMAGE | sed 's/local:iso\///')"
+            TARGET_DIR="/var/lib/vz/images"
+            
+            if [ -f "$SOURCE_PATH" ]; then
+                echo "   Copying file: $SOURCE_PATH -> $TARGET_DIR/"
+                mkdir -p "$TARGET_DIR" 2>/dev/null || true
+                cp "$SOURCE_PATH" "$TARGET_DIR/" 2>&1 && echo "✅ File copied" || {
+                    echo "❌ File copy failed"
+                    exit 1
+                }
+            else
+                echo "❌ Source file not found at $SOURCE_PATH"
+                echo "   Please copy the image manually or use Proxmox web UI"
+                exit 1
+            fi
+        fi
+    fi
+else
+    echo "❌ qm command not found"
+    exit 1
+fi
+
+echo ""
+echo "Step 4: Verifying copied image..."
+if pvesm list "$TARGET_STORAGE" | grep -q "$FILENAME"; then
+    echo "✅ Image copied to: $TARGET_VOLID"
+    echo ""
+    echo "The provider can now use this image directly for VM creation."
+else
+    echo "⚠️  Image not found in target storage, but copy may have succeeded"
+    echo "   Check manually: pvesm list $TARGET_STORAGE"
+fi
+
+echo ""
+echo "✅ Setup complete!"
+

scripts/copy-image-via-api.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+# Copy cloud image from ISO storage to image-capable storage using Proxmox API
+# This script uses the Proxmox API to copy the image
+
+set -e
+
+PROXMOX_HOST="192.168.11.11"
+PROXMOX_PORT="8006"
+NODE="r630-01"
+SOURCE_IMAGE="local:iso/ubuntu-22.04-cloud.img"
+TARGET_STORAGE="${1:-ceph-rbd}"  # Default to ceph-rbd
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  COPY IMAGE VIA PROXMOX API"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+# Get credentials from Kubernetes secret
+echo "Step 1: Getting Proxmox credentials from Kubernetes..."
+TOKEN=$(kubectl -n crossplane-system get secret proxmox-credentials -o jsonpath='{.data.token}' 2>/dev/null | base64 -d 2>/dev/null)
+TOKENID=$(kubectl -n crossplane-system get secret proxmox-credentials -o jsonpath='{.data.tokenid}' 2>/dev/null | base64 -d 2>/dev/null)
+
+if [ -z "$TOKEN" ] || [ -z "$TOKENID" ]; then
+    echo "❌ ERROR: Could not get Proxmox credentials from Kubernetes secret"
+    echo "   Make sure you're connected to the Kubernetes cluster"
+    exit 1
+fi
+
+AUTH_HEADER="PVEAPIToken=${TOKENID}=${TOKEN}"
+
+echo "✅ Credentials retrieved"
+echo ""
+
+# Extract filename
+FILENAME=$(echo "$SOURCE_IMAGE" | sed 's/.*\///')
+TARGET_PATH="template/cache/$FILENAME"
+TARGET_VOLID="$TARGET_STORAGE:$TARGET_PATH"
+
+echo "Step 2: Image details..."
+echo "   Source: $SOURCE_IMAGE"
+echo "   Target: $TARGET_VOLID"
+echo ""
+
+# Method 1: Try using storage upload with download-url
+echo "Step 3: Attempting to copy via API..."
+echo "   Method 1: Using storage download-url and upload..."
+
+# Get download URL for source image
+DOWNLOAD_URL_PATH="/nodes/$NODE/storage/local/download-url"
+DOWNLOAD_CONFIG="{\"volid\":\"$SOURCE_IMAGE\"}"
+
+echo "   Getting download URL..."
+DOWNLOAD_RESPONSE=$(curl -k -s -X POST \
+    -H "Authorization: $AUTH_HEADER" \
+    -H "Content-Type: application/json" \
+    -d "$DOWNLOAD_CONFIG" \
+    "https://$PROXMOX_HOST:$PROXMOX_PORT/api2/json$DOWNLOAD_URL_PATH" 2>&1)
+
+DOWNLOAD_URL=$(echo "$DOWNLOAD_RESPONSE" | jq -r '.data // empty' 2>/dev/null)
+
+if [ -n "$DOWNLOAD_URL" ] && [ "$DOWNLOAD_URL" != "null" ]; then
+    echo "   ✅ Download URL obtained: $DOWNLOAD_URL"
+    
+    # Upload to target storage
+    UPLOAD_PATH="/nodes/$NODE/storage/$TARGET_STORAGE/upload"
+    UPLOAD_CONFIG="{\"content\":\"images\",\"filename\":\"$FILENAME\",\"url\":\"$DOWNLOAD_URL\",\"format\":\"qcow2\"}"
+    
+    echo "   Uploading to $TARGET_STORAGE..."
+    UPLOAD_RESPONSE=$(curl -k -s -X POST \
+        -H "Authorization: $AUTH_HEADER" \
+        -H "Content-Type: application/json" \
+        -d "$UPLOAD_CONFIG" \
+        "https://$PROXMOX_HOST:$PROXMOX_PORT/api2/json$UPLOAD_PATH" 2>&1)
+    
+    if echo "$UPLOAD_RESPONSE" | jq -e '.data' >/dev/null 2>&1; then
+        echo "   ✅ Upload initiated successfully"
+        echo ""
+        echo "Step 4: Waiting for upload to complete..."
+        echo "   (This may take a few minutes for large images)"
+        echo "   Monitor progress in Proxmox Web UI or check storage content"
+        echo ""
+        echo "✅ Image copy initiated!"
+        echo "   The image will be available at: $TARGET_VOLID"
+        exit 0
+    else
+        echo "   ⚠️  Upload failed: $UPLOAD_RESPONSE"
+    fi
+else
+    echo "   ⚠️  Could not get download URL: $DOWNLOAD_RESPONSE"
+fi
+
+echo ""
+echo "❌ API copy methods failed"
+echo ""
+echo "📋 Alternative: Use manual copy via SSH"
+echo "   ssh root@$PROXMOX_HOST"
+echo "   bash < scripts/copy-image-to-storage.sh $TARGET_STORAGE"
+echo ""
+echo "   OR use Proxmox Web UI to copy the image"
+exit 1
+

scripts/create-all-osds-efficient.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Efficient script to create all OSDs on available drives
+set -e
+
+DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")
+
+echo "=== Checking cluster connectivity ==="
+if ! ceph quorum_status &>/dev/null; then
+    echo "WARNING: Cluster quorum may not be established. Continuing anyway..."
+fi
+
+echo "=== Creating OSDs on all drives ==="
+for disk in "${DRIVES[@]}"; do
+    echo "Processing /dev/$disk..."
+    
+    # Check if already an OSD
+    if ceph-volume lvm list /dev/$disk 2>/dev/null | grep -q "osd\."; then
+        echo "  /dev/$disk already has an OSD, skipping"
+        continue
+    fi
+    
+    # Create OSD
+    echo "  Creating OSD on /dev/$disk..."
+    if ceph-volume lvm create --data /dev/$disk --no-systemd 2>&1 | tee /tmp/osd-create-$disk.log; then
+        # Get OSD ID
+        OSD_ID=$(grep -oP "osd\.\K\d+" /tmp/osd-create-$disk.log | head -1)
+        if [ -n "$OSD_ID" ]; then
+            echo "  OSD $OSD_ID created on /dev/$disk"
+            # Enable and start service
+            systemctl enable ceph-osd@$OSD_ID
+            systemctl start ceph-osd@$OSD_ID
+            echo "  OSD $OSD_ID service started"
+        fi
+    else
+        echo "  WARNING: Failed to create OSD on /dev/$disk"
+    fi
+    echo ""
+done
+
+echo "=== Final OSD status ==="
+ceph osd tree 2>&1 || echo "Could not get OSD tree"
+
+echo "=== All OSD services ==="
+systemctl list-units | grep ceph-osd | grep active || echo "No active OSD services found"
+

scripts/create-ceph-osds-all-drives.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+# Create Ceph OSDs on all 6x 250GB drives
+# Run on R630-01 as root
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")
+
+echo "=========================================="
+echo "Create Ceph OSDs on All 6x 250GB Drives"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+echo -e "${BLUE}=== Step 1: Remove Partitions from Drives ===${NC}"
+echo "-----------------------------------"
+
+for drive in "${DRIVES[@]}"; do
+    if [ ! -b "/dev/$drive" ]; then
+        echo -e "${YELLOW}  /dev/$drive not found, skipping...${NC}"
+        continue
+    fi
+    
+    echo -e "${BLUE}  Preparing /dev/$drive...${NC}"
+    
+    # Unmount any partitions
+    umount /dev/${drive}* 2>/dev/null || true
+    
+    # Remove partitions/filesystem signatures
+    echo "    Removing partitions and filesystem signatures..."
+    wipefs -a "/dev/$drive" 2>/dev/null || true
+    
+    echo -e "${GREEN}    ✓ /dev/$drive prepared${NC}"
+done
+echo ""
+
+echo -e "${BLUE}=== Step 2: Create Ceph OSDs ===${NC}"
+echo "-----------------------------------"
+
+osd_count=0
+failed_drives=()
+
+for drive in "${DRIVES[@]}"; do
+    if [ ! -b "/dev/$drive" ]; then
+        continue
+    fi
+    
+    echo -e "${BLUE}  Creating OSD on /dev/$drive...${NC}"
+    
+    # Try to create OSD
+    if ceph-volume lvm create --data "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
+        echo -e "${GREEN}    ✓ OSD created on /dev/$drive${NC}"
+        osd_count=$((osd_count + 1))
+    else
+        echo -e "${YELLOW}    ⚠ OSD creation had issues${NC}"
+        failed_drives+=("$drive")
+        echo "    Check /tmp/ceph-osd-${drive}.log for details"
+    fi
+    echo ""
+done
+
+echo "=========================================="
+echo -e "${GREEN}OSD Creation Summary${NC}"
+echo "=========================================="
+echo ""
+echo "Successfully created: $osd_count OSD(s) out of ${#DRIVES[@]} drives"
+
+if [ ${#failed_drives[@]} -gt 0 ]; then
+    echo -e "${YELLOW}Failed drives: ${failed_drives[@]}${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 3: Verify Ceph Status ===${NC}"
+echo "-----------------------------------"
+
+echo "OSD Tree:"
+ceph osd tree 2>/dev/null || echo -e "${YELLOW}Cannot get OSD tree${NC}"
+echo ""
+
+echo "Ceph Health:"
+ceph health 2>/dev/null || echo -e "${YELLOW}Cannot get health${NC}"
+echo ""
+
+echo "OSD Details:"
+ceph osd df 2>/dev/null || echo -e "${YELLOW}Cannot get OSD details${NC}"
+echo ""
+
+OSD_COUNT=$(ceph osd tree 2>/dev/null | grep -c "osd\." || echo "0")
+echo "Current OSD count: $OSD_COUNT"
+
+if [ "$OSD_COUNT" -ge 3 ]; then
+    echo -e "${GREEN}  ✓ SUCCESS: Have 3+ OSDs (fixes TOO_FEW_OSDS error)${NC}"
+else
+    echo -e "${YELLOW}  ⚠ Still need more OSDs (currently have $OSD_COUNT, need 3+)${NC}"
+fi
+echo ""
+

scripts/create-ceph-rbd-storage-r630.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+# Create ceph-rbd storage pool on r630-01
+# RBD (Ceph Block Device) supports VM disk images, unlike CephFS
+
+set -e
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  CREATE CEPH RBD STORAGE POOL ON r630-01"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+# Check if running on r630-01
+HOSTNAME=$(hostname)
+if [ "$HOSTNAME" != "r630-01" ]; then
+    echo "⚠️  WARNING: This script should be run on r630-01"
+    echo "   Current hostname: $HOSTNAME"
+    echo ""
+    echo "To run remotely via SSH:"
+    echo "  ssh root@192.168.11.11 'bash -s' < $0"
+    echo ""
+    read -p "Continue anyway? (y/N) " -n 1 -r
+    echo
+    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+        exit 1
+    fi
+fi
+
+echo "Step 1: Checking Ceph cluster status..."
+if ! command -v ceph &> /dev/null; then
+    echo "❌ ERROR: ceph command not found. Ceph packages may not be installed."
+    exit 1
+fi
+
+CEPH_STATUS=$(ceph -s 2>&1 | head -5)
+echo "$CEPH_STATUS"
+echo ""
+
+echo "Step 2: Checking if RBD pool exists..."
+RBD_POOL_EXISTS=$(ceph osd pool ls 2>/dev/null | grep -c "^rbd$" || echo "0")
+if [ "$RBD_POOL_EXISTS" = "0" ]; then
+    echo "⚠️  RBD pool 'rbd' does not exist"
+    echo ""
+    echo "Step 3: Creating RBD pool..."
+    # Create RBD pool with appropriate PG count
+    # For small clusters, use 32 PGs
+    ceph osd pool create rbd 32 32 2>&1 || {
+        echo "❌ Failed to create RBD pool"
+        exit 1
+    }
+    echo "✅ RBD pool created"
+    
+    # Initialize the pool for RBD
+    echo "Step 4: Initializing RBD pool..."
+    rbd pool init rbd 2>&1 || {
+        echo "⚠️  Pool initialization failed or already initialized"
+    }
+else
+    echo "✅ RBD pool 'rbd' already exists"
+fi
+
+echo ""
+echo "Step 5: Checking if RBD storage is already added to Proxmox..."
+if pvesm status 2>/dev/null | grep -q "ceph-rbd"; then
+    echo "✅ Storage 'ceph-rbd' already exists in Proxmox"
+    pvesm status | grep "ceph-rbd"
+else
+    echo "Adding ceph-rbd storage to Proxmox..."
+    # Add RBD storage - RBD supports 'images' content type for VM disks
+    pvesm add rbd ceph-rbd \
+        --pool rbd \
+        --nodes r630-01 \
+        --content images \
+        --monhost 192.168.11.10:6789,192.168.11.11:6789 \
+        --username admin 2>&1 || {
+        echo "⚠️  Failed to add via pvesm. Trying alternative method..."
+        echo ""
+        echo "Alternative: Add via Web UI:"
+        echo "   Datacenter → Storage → Add → RBD"
+        echo "   ID: ceph-rbd"
+        echo "   Pool: rbd"
+        echo "   Nodes: r630-01"
+        echo "   Content: images"
+        echo ""
+        echo "Or via API (from your workstation):"
+        echo "   curl -k -X POST -H \"Authorization: PVEAPIToken=...\" \\"
+        echo "     \"https://192.168.11.11:8006/api2/json/storage\" \\"
+        echo "     -d \"storage=ceph-rbd&type=rbd&pool=rbd&nodes=r630-01&content=images&monhost=192.168.11.10:6789,192.168.11.11:6789&username=admin\""
+        exit 1
+    }
+    echo "✅ RBD storage added successfully"
+fi
+
+echo ""
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  VERIFICATION"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+echo "Checking storage status..."
+pvesm status | grep -E "ceph-rbd|NAME" || echo "Storage not found in pvesm status"
+
+echo ""
+echo "Checking RBD pool status..."
+ceph osd pool ls | grep "^rbd$" && echo "✅ RBD pool exists" || echo "❌ RBD pool not found"
+
+echo ""
+echo "✅ Setup complete! Storage 'ceph-rbd' should now be available for VM deployment."
+echo ""
+echo "📝 Next Steps:"
+echo "   Update your VM configurations to use 'ceph-rbd' instead of 'ceph-fs'"
+echo "   Example: storage: \"ceph-rbd\""
+

scripts/create-cephfs-storage-r630.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+# Create ceph-fs storage pool on r630-01
+# This script must be run on r630-01 or via SSH
+
+set -e
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  CREATE CEPHFS STORAGE POOL ON r630-01"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+# Check if running on r630-01
+HOSTNAME=$(hostname)
+if [ "$HOSTNAME" != "r630-01" ]; then
+    echo "⚠️  WARNING: This script should be run on r630-01"
+    echo "   Current hostname: $HOSTNAME"
+    echo ""
+    echo "To run remotely via SSH:"
+    echo "  ssh root@r630-01.sankofa.nexus 'bash -s' < $0"
+    echo ""
+    read -p "Continue anyway? (y/N) " -n 1 -r
+    echo
+    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+        exit 1
+    fi
+fi
+
+echo "Step 1: Checking Ceph cluster status..."
+if ! command -v ceph &> /dev/null; then
+    echo "❌ ERROR: ceph command not found. Ceph packages may not be installed."
+    exit 1
+fi
+
+CEPH_STATUS=$(ceph -s 2>&1 | head -5)
+echo "$CEPH_STATUS"
+echo ""
+
+echo "Step 2: Checking if MDS (Metadata Server) exists..."
+MDS_COUNT=$(ceph mds stat 2>/dev/null | grep -c "up:" || echo "0")
+if [ "$MDS_COUNT" = "0" ]; then
+    echo "⚠️  No MDS running. Creating MDS..."
+    pveceph mds create 2>&1 || {
+        echo "⚠️  MDS creation failed or already exists"
+        echo "Checking MDS status..."
+        ceph mds stat 2>&1 | head -3
+    }
+else
+    echo "✅ MDS is running"
+    ceph mds stat 2>&1 | head -3
+fi
+
+echo ""
+echo "Step 3: Checking if CephFS filesystem exists..."
+CEPHFS_EXISTS=$(ceph fs ls 2>/dev/null | grep -c "ceph-fs" || echo "0")
+if [ "$CEPHFS_EXISTS" = "0" ]; then
+    echo "⚠️  CephFS filesystem 'ceph-fs' does not exist"
+    echo ""
+    echo "Step 4: Creating CephFS using pveceph (recommended method)..."
+    echo "This will create the CephFS and add it to Proxmox storage automatically."
+    pveceph fs create --name ceph-fs --pg_num 128 --add-storage 2>&1 || {
+        echo "⚠️  pveceph fs create failed. Trying manual method..."
+        echo ""
+        echo "Step 4a: Checking if required pools exist..."
+        DATA_POOL_EXISTS=$(ceph osd pool ls 2>/dev/null | grep -c "cephfs_data" || echo "0")
+        META_POOL_EXISTS=$(ceph osd pool ls 2>/dev/null | grep -c "cephfs_metadata" || echo "0")
+        
+        if [ "$DATA_POOL_EXISTS" = "0" ] || [ "$META_POOL_EXISTS" = "0" ]; then
+            echo "Creating required pools..."
+            [ "$DATA_POOL_EXISTS" = "0" ] && ceph osd pool create cephfs_data 32 32
+            [ "$META_POOL_EXISTS" = "0" ] && ceph osd pool create cephfs_metadata 16 16
+        fi
+        
+        echo "Step 4b: Creating CephFS filesystem..."
+        ceph fs new ceph-fs cephfs_metadata cephfs_data 2>&1 || {
+            echo "⚠️  CephFS may already exist"
+        }
+        
+        echo "Step 4c: Adding to Proxmox storage..."
+        pvesm add cephfs ceph-fs \
+            --nodes r630-01 \
+            --content images,rootdir \
+            --fs-name ceph-fs \
+            --monhost 192.168.11.10:6789,192.168.11.11:6789 \
+            --username admin 2>&1 || {
+            echo "⚠️  Failed to add via pvesm. Try Web UI:"
+            echo "   Datacenter → Storage → Add → CephFS"
+            echo "   ID: ceph-fs, FS Name: ceph-fs, Nodes: r630-01"
+        }
+    }
+else
+    echo "✅ CephFS filesystem 'ceph-fs' already exists"
+    
+    # Check if it's added to Proxmox storage
+    if ! pvesm status 2>/dev/null | grep -q "ceph-fs"; then
+        echo ""
+        echo "Step 4: Adding existing CephFS to Proxmox storage..."
+        echo "Note: CephFS may not support 'images' content type for VM disks."
+        echo "Trying without content types first..."
+        
+        # Try without content types (Proxmox will use defaults)
+        pvesm add cephfs ceph-fs \
+            --nodes r630-01 \
+            --fs-name ceph-fs \
+            --monhost 192.168.11.10:6789,192.168.11.11:6789 \
+            --username admin 2>&1 || {
+            echo "⚠️  Failed to add via pvesm without content types."
+            echo "Trying with only rootdir (for containers)..."
+            pvesm add cephfs ceph-fs \
+                --nodes r630-01 \
+                --content rootdir \
+                --fs-name ceph-fs \
+                --monhost 192.168.11.10:6789,192.168.11.11:6789 \
+                --username admin 2>&1 || {
+                echo "⚠️  Failed to add via pvesm. Use Web UI or API:"
+                echo "   Web UI: Datacenter → Storage → Add → CephFS"
+                echo "   ID: ceph-fs, FS Name: ceph-fs, Nodes: r630-01"
+                echo ""
+                echo "   Or via API (from your workstation):"
+                echo "   curl -k -X POST -H \"Authorization: PVEAPIToken=...\" \\"
+                echo "     \"https://192.168.11.11:8006/api2/json/storage\" \\"
+                echo "     -d \"storage=ceph-fs&type=cephfs&nodes=r630-01&monhost=192.168.11.10:6789,192.168.11.11:6789&username=admin\""
+            }
+        }
+    else
+        echo "✅ Storage 'ceph-fs' already exists in Proxmox"
+    fi
+fi
+
+echo ""
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  VERIFICATION"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+echo "Checking storage status..."
+pvesm status | grep -E "ceph-fs|NAME" || echo "Storage not found in pvesm status"
+
+echo ""
+echo "✅ Setup complete! Storage 'ceph-fs' should now be available for VM deployment."
+

scripts/create-osds-r630.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Create OSDs on R630-01 250GB drives
+set -e
+
+DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")
+
+echo "=== Creating OSDs on R630-01 ==="
+
+for disk in "${DRIVES[@]}"; do
+    if [ ! -e "/dev/$disk" ]; then
+        echo "Skipping /dev/$disk (not found)"
+        continue
+    fi
+    
+    echo "Creating OSD on /dev/$disk..."
+    
+    # Check if already an OSD
+    if ceph-volume lvm list /dev/$disk 2>/dev/null | grep -q "osd\."; then
+        echo "  /dev/$disk already has an OSD, skipping"
+        continue
+    fi
+    
+    # Create OSD
+    ceph-volume lvm create --data /dev/$disk
+    
+    # Get OSD ID and start service
+    OSD_ID=$(ceph-volume lvm list /dev/$disk 2>/dev/null | grep -oP "osd\.\K\d+" | head -1)
+    if [ -n "$OSD_ID" ]; then
+        systemctl enable ceph-osd@$OSD_ID
+        systemctl start ceph-osd@$OSD_ID
+        echo "  OSD $OSD_ID created and started"
+    fi
+    
+    sleep 2
+done
+
+echo ""
+echo "=== OSD Creation Complete ==="
+echo "Run 'ceph osd tree' to verify OSDs"
+

scripts/create-template-from-image.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+# Create a Proxmox template from the cloud image in local:iso/
+
+set -euo pipefail
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  CREATE TEMPLATE FROM CLOUD IMAGE"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+SOURCE_IMAGE="/var/lib/vz/template/iso/ubuntu-22.04-cloud.img"
+TEMPLATE_VMID=9000
+TEMPLATE_NAME="ubuntu-22.04-cloud-template"
+TARGET_STORAGE="local-lvm"
+
+# Check if source exists
+echo "Step 1: Checking if source image exists..."
+if [ ! -f "$SOURCE_IMAGE" ]; then
+    echo "❌ Source image not found: $SOURCE_IMAGE"
+    exit 1
+fi
+
+IMAGE_SIZE=$(du -h "$SOURCE_IMAGE" | cut -f1)
+echo "✅ Source image found: $SOURCE_IMAGE ($IMAGE_SIZE)"
+echo ""
+
+# Check if template already exists
+echo "Step 2: Checking if template already exists..."
+if qm list | grep -q "^${TEMPLATE_VMID}"; then
+    echo "⚠️  Template VM $TEMPLATE_VMID already exists"
+    read -p "Delete and recreate? (y/N) " -n 1 -r
+    echo
+    if [[ $REPLY =~ ^[Yy]$ ]]; then
+        qm destroy "$TEMPLATE_VMID" 2>/dev/null || true
+        echo "✅ Old template removed"
+    else
+        echo "Using existing template"
+        exit 0
+    fi
+fi
+echo ""
+
+# Create template VM
+echo "Step 3: Creating template VM..."
+qm create "$TEMPLATE_VMID" \
+    --name "$TEMPLATE_NAME" \
+    --memory 512 \
+    --net0 virtio,bridge=vmbr0 \
+    --agent enabled=1 \
+    --template 1 2>/dev/null || true
+
+# Import the cloud image
+echo "Step 4: Importing cloud image to template..."
+echo "   This will convert qcow2 to raw format for local-lvm..."
+if qm disk import "$TEMPLATE_VMID" "$SOURCE_IMAGE" "$TARGET_STORAGE" --format raw 2>&1; then
+    echo "✅ Image imported successfully"
+else
+    echo "❌ Import failed"
+    qm destroy "$TEMPLATE_VMID" 2>/dev/null || true
+    exit 1
+fi
+
+# Convert to template
+echo ""
+echo "Step 5: Converting VM to template..."
+qm template "$TEMPLATE_VMID" 2>&1 || {
+    echo "⚠️  Template conversion failed, but VM is ready"
+}
+
+echo ""
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  VERIFICATION"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+echo "Template VM status:"
+qm list | grep "^${TEMPLATE_VMID}" || echo "Template not found"
+echo ""
+echo "✅ Template created! VMs can now clone from VMID $TEMPLATE_VMID"
+echo ""
+echo "📝 Note: Update VM image spec to use VMID: $TEMPLATE_VMID"
+echo "   Example: image: \"$TEMPLATE_VMID\""
+

scripts/create-third-ceph-osd.sh

@@ -0,0 +1,130 @@
+#!/bin/bash
+# Interactive script to create third Ceph OSD from ubuntu-vg drive
+# Run on R630-01 as root
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+echo "=========================================="
+echo "Create Third Ceph OSD from ubuntu-vg Drive"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+# Available 250GB drives
+AVAILABLE_DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")
+
+echo -e "${BLUE}Available 250GB drives:${NC}"
+for i in "${!AVAILABLE_DRIVES[@]}"; do
+    drive="${AVAILABLE_DRIVES[$i]}"
+    if [ -b "/dev/$drive" ]; then
+        size=$(fdisk -l "/dev/$drive" 2>/dev/null | grep "^Disk /dev/$drive" | awk '{print $3, $4}')
+        in_vg=$(pvs 2>/dev/null | grep "/dev/${drive}3" | grep ubuntu-vg | wc -l)
+        if [ "$in_vg" -gt 0 ]; then
+            echo -e "  $((i+1)). ${YELLOW}/dev/$drive: ${size} - In ubuntu-vg${NC}"
+        else
+            echo -e "  $((i+1)). ${GREEN}/dev/$drive: ${size} - Available${NC}"
+        fi
+    fi
+done
+echo ""
+
+# Get drive selection
+read -p "Enter drive number (1-6) or device name (e.g., sdc): " selection
+
+# Parse selection
+if [[ "$selection" =~ ^[1-6]$ ]]; then
+    DRIVE="${AVAILABLE_DRIVES[$((selection-1))]}"
+elif [[ "$selection" =~ ^sd[a-z]$ ]]; then
+    DRIVE="$selection"
+else
+    echo -e "${RED}Invalid selection${NC}"
+    exit 1
+fi
+
+if [ ! -b "/dev/$DRIVE" ]; then
+    echo -e "${RED}Drive /dev/$DRIVE not found${NC}"
+    exit 1
+fi
+
+echo ""
+echo -e "${YELLOW}Selected drive: /dev/$DRIVE${NC}"
+echo ""
+
+# Check if in ubuntu-vg
+in_vg=$(pvs 2>/dev/null | grep "/dev/${DRIVE}3" | grep ubuntu-vg | wc -l)
+
+if [ "$in_vg" -gt 0 ]; then
+    echo -e "${YELLOW}WARNING: /dev/${DRIVE}3 is in ubuntu-vg volume group${NC}"
+    echo -e "${RED}Removing it will destroy data on that logical volume!${NC}"
+    echo ""
+    read -p "Continue? (yes/no): " confirm
+    if [ "$confirm" != "yes" ]; then
+        echo "Aborted"
+        exit 0
+    fi
+    
+    echo ""
+    echo -e "${BLUE}Step 1: Removing from ubuntu-vg...${NC}"
+    
+    # Unmount any mounted logical volumes
+    echo "  Unmounting logical volumes..."
+    umount /dev/ubuntu-vg/* 2>/dev/null || true
+    
+    # Deactivate volume group
+    echo "  Deactivating ubuntu-vg..."
+    vgchange -a n ubuntu-vg
+    
+    # Remove physical volume
+    echo "  Removing /dev/${DRIVE}3 from ubuntu-vg..."
+    pvremove "/dev/${DRIVE}3" -y
+    
+    echo -e "${GREEN}  ✓ Removed from ubuntu-vg${NC}"
+    echo ""
+fi
+
+# Wipe the drive
+echo -e "${BLUE}Step 2: Wiping /dev/$DRIVE...${NC}"
+echo "  Wiping filesystem signatures..."
+wipefs -a "/dev/$DRIVE"
+echo "  Zeroing first 100MB..."
+dd if=/dev/zero of="/dev/$DRIVE" bs=1M count=100 status=progress
+echo -e "${GREEN}  ✓ Drive wiped${NC}"
+echo ""
+
+# Create Ceph OSD
+echo -e "${BLUE}Step 3: Creating Ceph OSD on /dev/$DRIVE...${NC}"
+ceph-volume lvm create --data "/dev/$DRIVE"
+echo ""
+
+# Verify
+echo -e "${BLUE}Step 4: Verifying OSD creation...${NC}"
+echo "OSD Tree:"
+ceph osd tree
+echo ""
+echo "Ceph Health:"
+ceph health
+echo ""
+echo "OSD Details:"
+ceph osd df
+echo ""
+
+echo "=========================================="
+echo -e "${GREEN}Third OSD Created Successfully!${NC}"
+echo "=========================================="
+echo ""
+echo "You should now have 3 OSDs, which should fix the TOO_FEW_OSDS error."
+echo "Check 'ceph health detail' for remaining issues."
+echo ""
+

scripts/deploy-all-vms.sh

@@ -0,0 +1,137 @@
+#!/bin/bash
+# Deploy All Production VMs
+# Deploys all VMs in the correct order according to the deployment plan
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+echo "=========================================="
+echo "VM Deployment Script"
+echo "=========================================="
+echo ""
+
+# Check if kubectl is available
+if ! command -v kubectl &> /dev/null; then
+    echo -e "${RED}Error: kubectl not found${NC}"
+    exit 1
+fi
+
+# Check if we can connect to cluster
+if ! kubectl cluster-info &> /dev/null; then
+    echo -e "${RED}Error: Cannot connect to Kubernetes cluster${NC}"
+    exit 1
+fi
+
+# Function to deploy and wait
+deploy_vm() {
+    local file=$1
+    local name=$(basename "$file" .yaml)
+    
+    echo -e "${BLUE}Deploying $name...${NC}"
+    if kubectl apply -f "$file" &>/dev/null; then
+        echo -e "${GREEN}✓${NC} $name deployed"
+        return 0
+    else
+        echo -e "${RED}✗${NC} $name deployment failed"
+        return 1
+    fi
+}
+
+# Phase 1: Core Infrastructure
+echo "=========================================="
+echo "Phase 1: Core Infrastructure"
+echo "=========================================="
+echo ""
+
+deploy_vm "$PROJECT_ROOT/examples/production/nginx-proxy-vm.yaml"
+deploy_vm "$PROJECT_ROOT/examples/production/phoenix/dns-primary.yaml"
+deploy_vm "$PROJECT_ROOT/examples/production/cloudflare-tunnel-vm.yaml"
+
+echo ""
+sleep 2
+
+# Phase 2: Phoenix Infrastructure
+echo "=========================================="
+echo "Phase 2: Phoenix Infrastructure"
+echo "=========================================="
+echo ""
+
+PHOENIX_VMS=(
+    "examples/production/phoenix/git-server.yaml"
+    "examples/production/phoenix/email-server.yaml"
+    "examples/production/phoenix/devops-runner.yaml"
+    "examples/production/phoenix/codespaces-ide.yaml"
+    "examples/production/phoenix/as4-gateway.yaml"
+    "examples/production/phoenix/business-integration-gateway.yaml"
+    "examples/production/phoenix/financial-messaging-gateway.yaml"
+)
+
+for vm in "${PHOENIX_VMS[@]}"; do
+    deploy_vm "$PROJECT_ROOT/$vm"
+done
+
+echo ""
+sleep 2
+
+# Phase 3: Blockchain Infrastructure
+echo "=========================================="
+echo "Phase 3: Blockchain Infrastructure"
+echo "=========================================="
+echo ""
+
+# Validators
+echo -e "${BLUE}Deploying Validators...${NC}"
+for i in {1..4}; do
+    deploy_vm "$PROJECT_ROOT/examples/production/smom-dbis-138/validator-0$i.yaml"
+done
+
+echo ""
+
+# Sentries
+echo -e "${BLUE}Deploying Sentries...${NC}"
+for i in {1..4}; do
+    deploy_vm "$PROJECT_ROOT/examples/production/smom-dbis-138/sentry-0$i.yaml"
+done
+
+echo ""
+
+# RPC Nodes
+echo -e "${BLUE}Deploying RPC Nodes...${NC}"
+for i in {1..4}; do
+    deploy_vm "$PROJECT_ROOT/examples/production/smom-dbis-138/rpc-node-0$i.yaml"
+done
+
+echo ""
+
+# Services
+echo -e "${BLUE}Deploying Services...${NC}"
+deploy_vm "$PROJECT_ROOT/examples/production/smom-dbis-138/management.yaml"
+deploy_vm "$PROJECT_ROOT/examples/production/smom-dbis-138/monitoring.yaml"
+deploy_vm "$PROJECT_ROOT/examples/production/smom-dbis-138/services.yaml"
+deploy_vm "$PROJECT_ROOT/examples/production/smom-dbis-138/blockscout.yaml"
+
+echo ""
+echo "=========================================="
+echo "Deployment Complete"
+echo "=========================================="
+echo ""
+
+# Show status
+echo "VM Status:"
+kubectl get proxmoxvm -A --sort-by=.metadata.name 2>&1 | head -35
+
+echo ""
+echo "To monitor deployment:"
+echo "  kubectl get proxmoxvm -A -w"
+echo ""
+echo "To check provider logs:"
+echo "  kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox -f"
+

scripts/diagnose-ceph-basic.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+# Basic Ceph diagnostic - checks infrastructure without hanging
+# Run on R630-01 as root
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+echo "=========================================="
+echo "Basic Ceph Infrastructure Check"
+echo "=========================================="
+echo ""
+
+echo -e "${BLUE}=== 1. Ceph Command Availability ===${NC}"
+echo "-----------------------------------"
+which ceph && ceph --version || echo -e "${RED}ceph command not found${NC}"
+which pveceph && echo "pveceph: $(pveceph --version 2>/dev/null || echo 'available')" || echo -e "${YELLOW}pveceph not found${NC}"
+echo ""
+
+echo -e "${BLUE}=== 2. Bootstrap Keyring Locations ===${NC}"
+echo "-----------------------------------"
+echo "Standard location:"
+ls -la /var/lib/ceph/bootstrap-osd/ceph.keyring 2>/dev/null && echo -e "${GREEN}  ✓ Found${NC}" || echo -e "${YELLOW}  ✗ Not found${NC}"
+echo ""
+
+echo "Proxmox location:"
+ls -la /etc/pve/priv/ceph.client.bootstrap-osd.keyring 2>/dev/null && echo -e "${GREEN}  ✓ Found${NC}" || echo -e "${YELLOW}  ✗ Not found${NC}"
+echo ""
+
+echo "All Ceph keyrings in /etc/pve/priv/:"
+ls -la /etc/pve/priv/ceph* 2>/dev/null | head -10 || echo "  No Ceph keyrings found"
+echo ""
+
+echo -e "${BLUE}=== 3. Ceph Services (Non-blocking) ===${NC}"
+echo "-----------------------------------"
+echo "Ceph target status:"
+systemctl is-active ceph.target 2>/dev/null && echo -e "${GREEN}  ✓ Active${NC}" || echo -e "${YELLOW}  ✗ Not active${NC}"
+echo ""
+
+echo "Ceph monitor services:"
+systemctl list-units --type=service --no-pager | grep ceph-mon | head -5 || echo "  No monitor services"
+echo ""
+
+echo "Ceph OSD services:"
+systemctl list-units --type=service --no-pager | grep ceph-osd | head -5 || echo "  No OSD services"
+echo ""
+
+echo -e "${BLUE}=== 4. Network Ports ===${NC}"
+echo "-----------------------------------"
+echo "Port 6789 (Ceph monitors):"
+ss -tlnp 2>/dev/null | grep 6789 || netstat -tlnp 2>/dev/null | grep 6789 || echo "  No listeners on port 6789"
+echo ""
+
+echo -e "${BLUE}=== 5. Ceph Configuration Files ===${NC}"
+echo "-----------------------------------"
+[ -f "/etc/ceph/ceph.conf" ] && echo -e "${GREEN}  ✓ /etc/ceph/ceph.conf exists${NC}" || echo -e "${YELLOW}  ✗ /etc/ceph/ceph.conf not found${NC}"
+[ -f "/etc/pve/ceph.conf" ] && echo -e "${GREEN}  ✓ /etc/pve/ceph.conf exists${NC}" || echo -e "${YELLOW}  ✗ /etc/pve/ceph.conf not found${NC}"
+echo ""
+
+echo -e "${BLUE}=== 6. Ceph Directories ===${NC}"
+echo "-----------------------------------"
+echo "Ceph directories:"
+[ -d "/var/lib/ceph" ] && echo "  /var/lib/ceph: $(ls -d /var/lib/ceph/* 2>/dev/null | wc -l) items" || echo "  /var/lib/ceph: does not exist"
+[ -d "/etc/pve/priv" ] && echo "  /etc/pve/priv: exists" || echo "  /etc/pve/priv: does not exist"
+echo ""
+
+echo -e "${BLUE}=== 7. Quick Ceph Test (5 second timeout) ===${NC}"
+echo "-----------------------------------"
+echo "Testing ceph health (5 second timeout):"
+timeout 5 ceph health 2>&1 && echo -e "${GREEN}  ✓ Cluster accessible${NC}" || echo -e "${RED}  ✗ Cluster not accessible or timeout${NC}"
+echo ""
+
+echo -e "${BLUE}=== 8. Proxmox Ceph Status ===${NC}"
+echo "-----------------------------------"
+if command -v pveceph &> /dev/null; then
+    echo "pveceph status (non-blocking):"
+    timeout 5 pveceph status 2>&1 | head -10 || echo "  pveceph status timed out or failed"
+else
+    echo -e "${YELLOW}pveceph not available${NC}"
+fi
+echo ""
+
+echo "=========================================="
+echo -e "${GREEN}Basic Diagnostic Complete${NC}"
+echo "=========================================="
+echo ""
+echo "Key Findings:"
+echo "  - Check bootstrap keyring locations above"
+echo "  - Check if Ceph services are running"
+echo "  - Check if cluster is accessible (timeout test)"
+echo "  - Check if pveceph is available"
+echo ""
+

scripts/diagnose-ceph-issues.sh

@@ -0,0 +1,185 @@
+#!/bin/bash
+# Comprehensive Ceph diagnostic script
+# Run on R630-01 as root
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+echo "=========================================="
+echo "Ceph Cluster Diagnostic"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+echo -e "${BLUE}=== 1. Ceph Cluster Health ===${NC}"
+echo "-----------------------------------"
+if command -v ceph &> /dev/null; then
+    echo "Ceph Health:"
+    ceph health 2>&1 || echo -e "${RED}Cannot get health${NC}"
+    echo ""
+    echo "Ceph Health Detail:"
+    ceph health detail 2>&1 | head -20 || echo -e "${RED}Cannot get health detail${NC}"
+else
+    echo -e "${RED}ceph command not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== 2. Ceph Monitor Status ===${NC}"
+echo "-----------------------------------"
+if command -v ceph &> /dev/null; then
+    echo "Monitor Status:"
+    ceph mon stat 2>&1 || echo -e "${RED}Cannot get monitor status${NC}"
+    echo ""
+    echo "Monitor Dump:"
+    ceph mon dump 2>&1 | head -30 || echo -e "${RED}Cannot get monitor dump${NC}"
+else
+    echo -e "${RED}ceph command not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== 3. Ceph OSD Status ===${NC}"
+echo "-----------------------------------"
+if command -v ceph &> /dev/null; then
+    echo "OSD Tree:"
+    ceph osd tree 2>&1 || echo -e "${RED}Cannot get OSD tree${NC}"
+    echo ""
+    echo "OSD Details:"
+    ceph osd df 2>&1 || echo -e "${RED}Cannot get OSD details${NC}"
+else
+    echo -e "${RED}ceph command not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== 4. Bootstrap Keyring Check ===${NC}"
+echo "-----------------------------------"
+echo "Standard location (/var/lib/ceph/bootstrap-osd/):"
+if [ -d "/var/lib/ceph/bootstrap-osd" ]; then
+    ls -la /var/lib/ceph/bootstrap-osd/ 2>/dev/null || echo "Directory exists but empty"
+else
+    echo -e "${YELLOW}Directory does not exist${NC}"
+fi
+echo ""
+
+echo "Proxmox location (/etc/pve/priv/):"
+if [ -d "/etc/pve/priv" ]; then
+    ls -la /etc/pve/priv/ceph.client.bootstrap-osd.keyring 2>/dev/null || echo -e "${YELLOW}Bootstrap keyring not found${NC}"
+    echo "All Ceph keyrings in /etc/pve/priv/:"
+    ls -la /etc/pve/priv/ceph* 2>/dev/null || echo "No Ceph keyrings found"
+else
+    echo -e "${YELLOW}/etc/pve/priv does not exist${NC}"
+fi
+echo ""
+
+echo "Bootstrap client in auth list:"
+if command -v ceph &> /dev/null; then
+    ceph auth list 2>&1 | grep -i bootstrap || echo -e "${YELLOW}No bootstrap client found${NC}"
+else
+    echo -e "${RED}ceph command not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== 5. Ceph Services Status ===${NC}"
+echo "-----------------------------------"
+echo "Ceph Target:"
+systemctl status ceph.target --no-pager 2>&1 | head -10 || echo "Service not found"
+echo ""
+
+echo "Ceph Monitor Services:"
+systemctl list-units --type=service | grep ceph-mon || echo "No monitor services found"
+for mon in $(systemctl list-units --type=service | grep ceph-mon | awk '{print $1}'); do
+    echo "  $mon:"
+    systemctl status $mon --no-pager 2>&1 | head -5
+done
+echo ""
+
+echo "Ceph OSD Services:"
+systemctl list-units --type=service | grep ceph-osd | head -5 || echo "No OSD services found"
+echo ""
+
+echo -e "${BLUE}=== 6. Network Connectivity ===${NC}"
+echo "-----------------------------------"
+echo "Ceph Monitor Ports:"
+netstat -tlnp 2>/dev/null | grep 6789 || ss -tlnp 2>/dev/null | grep 6789 || echo "No listeners on port 6789"
+echo ""
+
+echo "Monitor Addresses (from mon dump):"
+if command -v ceph &> /dev/null; then
+    MON_ADDRS=$(ceph mon dump 2>/dev/null | grep -E "addr|rank" | head -10)
+    if [ ! -z "$MON_ADDRS" ]; then
+        echo "$MON_ADDRS"
+        echo ""
+        echo "Testing connectivity to monitors:"
+        echo "$MON_ADDRS" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+:[0-9]+' | while read addr; do
+            ip=$(echo $addr | cut -d: -f1)
+            port=$(echo $addr | cut -d: -f2)
+            echo -n "  Testing $ip:$port ... "
+            if timeout 2 bash -c "echo > /dev/tcp/$ip/$port" 2>/dev/null; then
+                echo -e "${GREEN}✓ Reachable${NC}"
+            else
+                echo -e "${RED}✗ Not reachable${NC}"
+            fi
+        done
+    else
+        echo -e "${YELLOW}Cannot get monitor addresses${NC}"
+    fi
+else
+    echo -e "${RED}ceph command not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== 7. Proxmox Ceph Integration ===${NC}"
+echo "-----------------------------------"
+echo "pveceph command:"
+which pveceph || echo -e "${YELLOW}pveceph not found${NC}"
+echo ""
+
+if command -v pveceph &> /dev/null; then
+    echo "pveceph status:"
+    pveceph status 2>&1 || echo "pveceph status failed"
+fi
+echo ""
+
+echo -e "${BLUE}=== 8. Ceph Configuration ===${NC}"
+echo "-----------------------------------"
+echo "Ceph config file:"
+if [ -f "/etc/ceph/ceph.conf" ]; then
+    echo "  /etc/ceph/ceph.conf exists"
+    cat /etc/ceph/ceph.conf 2>/dev/null | head -20
+elif [ -f "/etc/pve/ceph.conf" ]; then
+    echo "  /etc/pve/ceph.conf exists"
+    cat /etc/pve/ceph.conf 2>/dev/null | head -20
+else
+    echo -e "${YELLOW}No Ceph config file found${NC}"
+fi
+echo ""
+
+echo "Ceph cluster name:"
+if command -v ceph &> /dev/null; then
+    ceph --version 2>&1 || true
+    # Try to get cluster name
+    ceph config dump 2>&1 | grep "cluster" | head -5 || true
+fi
+echo ""
+
+echo "=========================================="
+echo -e "${GREEN}Diagnostic Complete${NC}"
+echo "=========================================="
+echo ""
+echo "Summary of findings:"
+echo "  - Review Ceph health status above"
+echo "  - Check bootstrap keyring locations"
+echo "  - Verify monitor connectivity"
+echo "  - Check if pveceph is available"
+echo ""
+

scripts/dry-run-deploy-r630-01.sh

@@ -0,0 +1,345 @@
+#!/bin/bash
+# Dry Run: Deploy All VMs to r630-01
+# Validates all VM configurations targeting r630-01 without actually deploying
+
+set -o pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+RED='\033[0;31m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+echo "=========================================="
+echo "Dry Run: VM Deployment to r630-01"
+echo "=========================================="
+echo ""
+echo -e "${CYAN}This is a DRY RUN - no VMs will be created${NC}"
+echo ""
+
+# Check if kubectl is available
+if ! command -v kubectl &> /dev/null; then
+    echo -e "${RED}Error: kubectl not found${NC}"
+    exit 1
+fi
+
+# Check if we can connect to cluster
+if ! kubectl cluster-info &> /dev/null; then
+    echo -e "${RED}Error: Cannot connect to Kubernetes cluster${NC}"
+    exit 1
+fi
+
+# Function to validate VM configuration
+validate_vm() {
+    local file=$1
+    local name=$(basename "$file" .yaml)
+    
+    # Check if this VM targets r630-01
+    local node=$(kubectl apply --dry-run=client -f "$file" -o json 2>/dev/null | jq -r '.spec.forProvider.node // ""' 2>/dev/null || echo "")
+    
+    if [[ "$node" != "r630-01" ]]; then
+        return 2  # Skip this VM (not targeting r630-01)
+    fi
+    
+    echo -e "${BLUE}Validating $name...${NC}"
+    
+    # Validate the configuration
+    if kubectl apply --dry-run=client -f "$file" &>/dev/null; then
+        # Extract configuration details
+        local config=$(kubectl apply --dry-run=client -f "$file" -o json 2>/dev/null)
+        local cpu=$(echo "$config" | jq -r '.spec.forProvider.cpu // "N/A"')
+        local memory=$(echo "$config" | jq -r '.spec.forProvider.memory // "N/A"')
+        local disk=$(echo "$config" | jq -r '.spec.forProvider.disk // "N/A"')
+        local storage=$(echo "$config" | jq -r '.spec.forProvider.storage // "N/A"')
+        local image=$(echo "$config" | jq -r '.spec.forProvider.image // "N/A"')
+        local site=$(echo "$config" | jq -r '.spec.forProvider.site // "N/A"')
+        
+        echo -e "  ${GREEN}✓${NC} Configuration valid"
+        echo -e "    CPU: $cpu cores"
+        echo -e "    Memory: $memory"
+        echo -e "    Disk: $disk"
+        echo -e "    Storage: $storage"
+        echo -e "    Image: $image"
+        echo -e "    Site: $site"
+        
+        # Check if VM already exists
+        if kubectl get proxmoxvm "$name" &>/dev/null 2>&1; then
+            local existing_node=$(kubectl get proxmoxvm "$name" -o jsonpath='{.spec.forProvider.node}' 2>/dev/null || echo "")
+            if [[ "$existing_node" == "r630-01" ]]; then
+                echo -e "    ${YELLOW}⚠${NC} VM already exists (will be updated)"
+            else
+                echo -e "    ${YELLOW}⚠${NC} VM exists on different node: $existing_node"
+            fi
+        else
+            echo -e "    ${CYAN}→${NC} New VM will be created"
+        fi
+        
+        return 0
+    else
+        echo -e "  ${RED}✗${NC} Configuration validation failed"
+        kubectl apply --dry-run=client -f "$file" 2>&1 | head -5
+        return 1
+    fi
+}
+
+# Collect all VMs targeting r630-01
+echo "=========================================="
+echo "Phase 1: Core Infrastructure"
+echo "=========================================="
+echo ""
+
+CORE_VMS=(
+    "examples/production/nginx-proxy-vm.yaml"
+    "examples/production/cloudflare-tunnel-vm.yaml"
+)
+
+CORE_COUNT=0
+CORE_VALID=0
+CORE_INVALID=0
+
+for vm in "${CORE_VMS[@]}"; do
+    if [[ -f "$PROJECT_ROOT/$vm" ]]; then
+        validate_vm "$PROJECT_ROOT/$vm"; result=$?
+        if [[ $result -eq 0 ]]; then
+            ((CORE_VALID++)) || true
+            ((CORE_COUNT++)) || true
+        elif [[ $result -eq 1 ]]; then
+            ((CORE_INVALID++)) || true
+            ((CORE_COUNT++)) || true
+        fi
+    fi
+done
+
+echo ""
+sleep 1
+
+# Phase 2: Phoenix Infrastructure
+echo "=========================================="
+echo "Phase 2: Phoenix Infrastructure"
+echo "=========================================="
+echo ""
+
+PHOENIX_VMS=(
+    "examples/production/phoenix/git-server.yaml"
+    "examples/production/phoenix/email-server.yaml"
+    "examples/production/phoenix/devops-runner.yaml"
+    "examples/production/phoenix/codespaces-ide.yaml"
+    "examples/production/phoenix/as4-gateway.yaml"
+    "examples/production/phoenix/business-integration-gateway.yaml"
+    "examples/production/phoenix/financial-messaging-gateway.yaml"
+)
+
+PHOENIX_COUNT=0
+PHOENIX_VALID=0
+PHOENIX_INVALID=0
+
+for vm in "${PHOENIX_VMS[@]}"; do
+    if [[ -f "$PROJECT_ROOT/$vm" ]]; then
+        validate_vm "$PROJECT_ROOT/$vm"; result=$?
+        if [[ $result -eq 0 ]]; then
+            ((PHOENIX_VALID++)) || true
+            ((PHOENIX_COUNT++)) || true
+        elif [[ $result -eq 1 ]]; then
+            ((PHOENIX_INVALID++)) || true
+            ((PHOENIX_COUNT++)) || true
+        fi
+    fi
+done
+
+echo ""
+sleep 1
+
+# Phase 3: Blockchain Infrastructure
+echo "=========================================="
+echo "Phase 3: Blockchain Infrastructure"
+echo "=========================================="
+echo ""
+
+# Validators
+echo -e "${BLUE}Validating Validators...${NC}"
+VALIDATOR_COUNT=0
+VALIDATOR_VALID=0
+VALIDATOR_INVALID=0
+
+for i in {1..4}; do
+    vm_file="$PROJECT_ROOT/examples/production/smom-dbis-138/validator-0$i.yaml"
+    if [[ -f "$vm_file" ]]; then
+        validate_vm "$vm_file"; result=$?
+        if [[ $result -eq 0 ]]; then
+            ((VALIDATOR_VALID++)) || true
+            ((VALIDATOR_COUNT++)) || true
+        elif [[ $result -eq 1 ]]; then
+            ((VALIDATOR_INVALID++)) || true
+            ((VALIDATOR_COUNT++)) || true
+        fi
+    fi
+done
+
+echo ""
+
+# Sentries
+echo -e "${BLUE}Validating Sentries...${NC}"
+SENTRY_COUNT=0
+SENTRY_VALID=0
+SENTRY_INVALID=0
+
+for i in {1..4}; do
+    vm_file="$PROJECT_ROOT/examples/production/smom-dbis-138/sentry-0$i.yaml"
+    if [[ -f "$vm_file" ]]; then
+        validate_vm "$vm_file"; result=$?
+        if [[ $result -eq 0 ]]; then
+            ((SENTRY_VALID++)) || true
+            ((SENTRY_COUNT++)) || true
+        elif [[ $result -eq 1 ]]; then
+            ((SENTRY_INVALID++)) || true
+            ((SENTRY_COUNT++)) || true
+        fi
+    fi
+done
+
+echo ""
+
+# RPC Nodes
+echo -e "${BLUE}Validating RPC Nodes...${NC}"
+RPC_COUNT=0
+RPC_VALID=0
+RPC_INVALID=0
+
+for i in {1..4}; do
+    vm_file="$PROJECT_ROOT/examples/production/smom-dbis-138/rpc-node-0$i.yaml"
+    if [[ -f "$vm_file" ]]; then
+        validate_vm "$vm_file"; result=$?
+        if [[ $result -eq 0 ]]; then
+            ((RPC_VALID++)) || true
+            ((RPC_COUNT++)) || true
+        elif [[ $result -eq 1 ]]; then
+            ((RPC_INVALID++)) || true
+            ((RPC_COUNT++)) || true
+        fi
+    fi
+done
+
+echo ""
+
+# Services
+echo -e "${BLUE}Validating Services...${NC}"
+SERVICE_VMS=(
+    "examples/production/smom-dbis-138/management.yaml"
+    "examples/production/smom-dbis-138/monitoring.yaml"
+    "examples/production/smom-dbis-138/services.yaml"
+    "examples/production/smom-dbis-138/blockscout.yaml"
+)
+
+SERVICE_COUNT=0
+SERVICE_VALID=0
+SERVICE_INVALID=0
+
+for vm in "${SERVICE_VMS[@]}"; do
+    if [[ -f "$PROJECT_ROOT/$vm" ]]; then
+        validate_vm "$PROJECT_ROOT/$vm"; result=$?
+        if [[ $result -eq 0 ]]; then
+            ((SERVICE_VALID++)) || true
+            ((SERVICE_COUNT++)) || true
+        elif [[ $result -eq 1 ]]; then
+            ((SERVICE_INVALID++)) || true
+            ((SERVICE_COUNT++)) || true
+        fi
+    fi
+done
+
+echo ""
+
+# Additional VMs (basic, medium, large, vm-100)
+echo "=========================================="
+echo "Phase 4: Additional VMs"
+echo "=========================================="
+echo ""
+
+ADDITIONAL_VMS=(
+    "examples/production/basic-vm.yaml"
+    "examples/production/medium-vm.yaml"
+    "examples/production/large-vm.yaml"
+    "examples/production/vm-100.yaml"
+)
+
+ADDITIONAL_COUNT=0
+ADDITIONAL_VALID=0
+ADDITIONAL_INVALID=0
+
+for vm in "${ADDITIONAL_VMS[@]}"; do
+    if [[ -f "$PROJECT_ROOT/$vm" ]]; then
+        validate_vm "$PROJECT_ROOT/$vm"; result=$?
+        if [[ $result -eq 0 ]]; then
+            ((ADDITIONAL_VALID++)) || true
+            ((ADDITIONAL_COUNT++)) || true
+        elif [[ $result -eq 1 ]]; then
+            ((ADDITIONAL_INVALID++)) || true
+            ((ADDITIONAL_COUNT++)) || true
+        fi
+    fi
+done
+
+echo ""
+
+# Summary
+echo "=========================================="
+echo "Dry Run Summary"
+echo "=========================================="
+echo ""
+
+TOTAL_VALID=$((CORE_VALID + PHOENIX_VALID + VALIDATOR_VALID + SENTRY_VALID + RPC_VALID + SERVICE_VALID + ADDITIONAL_VALID))
+TOTAL_INVALID=$((CORE_INVALID + PHOENIX_INVALID + VALIDATOR_INVALID + SENTRY_INVALID + RPC_INVALID + SERVICE_INVALID + ADDITIONAL_INVALID))
+TOTAL_COUNT=$((CORE_COUNT + PHOENIX_COUNT + VALIDATOR_COUNT + SENTRY_COUNT + RPC_COUNT + SERVICE_COUNT + ADDITIONAL_COUNT))
+
+echo "Core Infrastructure:"
+echo "  Valid: $CORE_VALID"
+echo "  Invalid: $CORE_INVALID"
+echo ""
+
+echo "Phoenix Infrastructure:"
+echo "  Valid: $PHOENIX_VALID"
+echo "  Invalid: $PHOENIX_INVALID"
+echo ""
+
+echo "Blockchain Infrastructure:"
+echo "  Validators: $VALIDATOR_VALID valid, $VALIDATOR_INVALID invalid"
+echo "  Sentries: $SENTRY_VALID valid, $SENTRY_INVALID invalid"
+echo "  RPC Nodes: $RPC_VALID valid, $RPC_INVALID invalid"
+echo "  Services: $SERVICE_VALID valid, $SERVICE_INVALID invalid"
+echo ""
+
+echo "Additional VMs:"
+echo "  Valid: $ADDITIONAL_VALID"
+echo "  Invalid: $ADDITIONAL_INVALID"
+echo ""
+
+echo "=========================================="
+echo "Total VMs targeting r630-01: $TOTAL_COUNT"
+echo -e "  ${GREEN}Valid: $TOTAL_VALID${NC}"
+if [[ $TOTAL_INVALID -gt 0 ]]; then
+    echo -e "  ${RED}Invalid: $TOTAL_INVALID${NC}"
+else
+    echo -e "  ${GREEN}Invalid: $TOTAL_INVALID${NC}"
+fi
+echo "=========================================="
+echo ""
+
+if [[ $TOTAL_INVALID -eq 0 ]]; then
+    echo -e "${GREEN}✓ All VM configurations are valid!${NC}"
+    echo ""
+    echo "To deploy these VMs, run:"
+    echo "  ./scripts/deploy-all-vms.sh"
+    echo ""
+    echo "Or deploy individually:"
+    echo "  kubectl apply -f examples/production/<vm-file>.yaml"
+    exit 0
+else
+    echo -e "${RED}✗ Some VM configurations have errors${NC}"
+    echo "Please fix the invalid configurations before deploying."
+    exit 1
+fi
+

scripts/final-image-fix.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+echo "=========================================="
+echo "FINAL IMAGE FIX - Remove ALL and Re-import"
+echo "=========================================="
+echo ""
+echo "Step 1: Removing ALL crossplane-provider-proxmox images..."
+sudo ctr -n k8s.io images rm $(sudo ctr -n k8s.io images ls -q | grep crossplane-provider-proxmox || true) 2>/dev/null || echo "  (No images to remove)"
+echo ""
+echo "Step 2: Re-importing fresh image..."
+sudo ctr -n k8s.io images import /tmp/crossplane-fresh-nuclear.tar
+echo ""
+echo "Step 3: Verifying import..."
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox
+echo ""
+echo "✅ Image re-imported. Now delete and recreate the deployment:"
+echo "   kubectl delete deployment crossplane-provider-proxmox -n crossplane-system"
+echo "   kubectl apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml"

scripts/fix-and-create-all-osds.sh

@@ -0,0 +1,191 @@
+#!/bin/bash
+# Complete script to fix bootstrap keyring and create OSDs on all 6 drives
+# Run on R630-01 as root
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")
+
+echo "=========================================="
+echo "Fix Bootstrap Keyring and Create All OSDs"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+echo -e "${BLUE}=== Step 1: Check Ceph Cluster Connectivity ===${NC}"
+echo "-----------------------------------"
+
+if ! command -v ceph &> /dev/null; then
+    echo -e "${RED}Ceph command not found. Please install ceph-common.${NC}"
+    exit 1
+fi
+
+echo "Testing Ceph connectivity..."
+if ceph health &>/dev/null; then
+    echo -e "${GREEN}  ✓ Ceph cluster is accessible${NC}"
+    ceph health
+else
+    echo -e "${YELLOW}  ⚠ Ceph cluster may not be fully accessible${NC}"
+    echo "  Attempting to continue anyway..."
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 2: Fix Bootstrap Keyring ===${NC}"
+echo "-----------------------------------"
+
+BOOTSTRAP_DIR="/var/lib/ceph/bootstrap-osd"
+BOOTSTRAP_KEYRING="$BOOTSTRAP_DIR/ceph.keyring"
+
+# Create directory
+mkdir -p "$BOOTSTRAP_DIR"
+
+# Try to get bootstrap keyring from cluster
+echo "Attempting to get bootstrap keyring from cluster..."
+if ceph auth get client.bootstrap-osd -o "$BOOTSTRAP_KEYRING" 2>/dev/null; then
+    echo -e "${GREEN}  ✓ Retrieved bootstrap keyring from cluster${NC}"
+elif ceph auth list 2>/dev/null | grep -q "client.bootstrap-osd"; then
+    echo "  Bootstrap keyring exists in cluster, exporting..."
+    ceph auth export client.bootstrap-osd -o "$BOOTSTRAP_KEYRING" 2>/dev/null || \
+    ceph auth get client.bootstrap-osd -o "$BOOTSTRAP_KEYRING" 2>/dev/null || true
+else
+    echo -e "${YELLOW}  ⚠ Bootstrap keyring not found in cluster${NC}"
+    echo "  Attempting to create it..."
+    ceph auth add client.bootstrap-osd mon 'allow profile bootstrap-osd' -i "$BOOTSTRAP_KEYRING" 2>/dev/null || \
+    echo -e "${YELLOW}    Could not create bootstrap keyring automatically${NC}"
+fi
+
+# Check if keyring exists now
+if [ -f "$BOOTSTRAP_KEYRING" ]; then
+    echo -e "${GREEN}  ✓ Bootstrap keyring exists: $BOOTSTRAP_KEYRING${NC}"
+    ls -lh "$BOOTSTRAP_KEYRING"
+else
+    echo -e "${YELLOW}  ⚠ Bootstrap keyring still not found${NC}"
+    echo "  Will try alternative methods..."
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 3: Check for pveceph (Proxmox Ceph Tool) ===${NC}"
+echo "-----------------------------------"
+
+USE_PVECEPH=false
+if command -v pveceph &> /dev/null; then
+    echo -e "${GREEN}  ✓ pveceph found - will use Proxmox Ceph tool${NC}"
+    USE_PVECEPH=true
+else
+    echo -e "${YELLOW}  pveceph not found - will use ceph-volume${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 4: Create OSDs on All Drives ===${NC}"
+echo "-----------------------------------"
+
+osd_count=0
+failed_drives=()
+
+for drive in "${DRIVES[@]}"; do
+    if [ ! -b "/dev/$drive" ]; then
+        echo -e "${YELLOW}  /dev/$drive not found, skipping...${NC}"
+        continue
+    fi
+    
+    echo -e "${BLUE}  Creating OSD on /dev/$drive...${NC}"
+    
+    # Try pveceph first if available
+    if [ "$USE_PVECEPH" = true ]; then
+        if pveceph create "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
+            echo -e "${GREEN}    ✓ OSD created on /dev/$drive (using pveceph)${NC}"
+            osd_count=$((osd_count + 1))
+            continue
+        else
+            echo -e "${YELLOW}    pveceph failed, trying ceph-volume...${NC}"
+        fi
+    fi
+    
+    # Try ceph-volume
+    if ceph-volume lvm create --data "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
+        echo -e "${GREEN}    ✓ OSD created on /dev/$drive${NC}"
+        osd_count=$((osd_count + 1))
+    else
+        echo -e "${YELLOW}    ⚠ OSD creation had issues${NC}"
+        failed_drives+=("$drive")
+        
+        # Check if OSD was partially created
+        sleep 2
+        if ceph osd tree 2>/dev/null | grep -q "osd\."; then
+            echo "    Checking if OSD exists in cluster..."
+            # Count OSDs before and after to see if one was created
+        fi
+    fi
+    echo ""
+done
+
+echo "=========================================="
+echo -e "${GREEN}OSD Creation Summary${NC}"
+echo "=========================================="
+echo ""
+echo "Successfully created: $osd_count OSD(s) out of ${#DRIVES[@]} drives"
+
+if [ ${#failed_drives[@]} -gt 0 ]; then
+    echo -e "${YELLOW}Failed drives: ${failed_drives[@]}${NC}"
+    echo "Check logs in /tmp/ceph-osd-*.log for details"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 5: Verify Ceph Status ===${NC}"
+echo "-----------------------------------"
+
+echo "OSD Tree:"
+ceph osd tree 2>/dev/null || echo -e "${YELLOW}Cannot get OSD tree${NC}"
+echo ""
+
+echo "Ceph Health:"
+ceph health 2>/dev/null || echo -e "${YELLOW}Cannot get health${NC}"
+echo ""
+
+echo "OSD Details:"
+ceph osd df 2>/dev/null || echo -e "${YELLOW}Cannot get OSD details${NC}"
+echo ""
+
+echo "Current OSD count:"
+OSD_COUNT=$(ceph osd tree 2>/dev/null | grep -c "osd\." || echo "0")
+echo "  Total OSDs: $OSD_COUNT"
+
+if [ "$OSD_COUNT" -ge 3 ]; then
+    echo -e "${GREEN}  ✓ SUCCESS: Have 3+ OSDs (fixes TOO_FEW_OSDS error)${NC}"
+else
+    echo -e "${YELLOW}  ⚠ Still need more OSDs (currently have $OSD_COUNT, need 3+)${NC}"
+fi
+echo ""
+
+echo "=========================================="
+if [ "$osd_count" -gt 0 ]; then
+    echo -e "${GREEN}Process Complete!${NC}"
+    echo ""
+    echo "Next steps:"
+    echo "1. Monitor Ceph health: ceph health detail"
+    echo "2. Check for remaining issues: ceph health"
+    echo "3. If some OSDs failed, check logs: /tmp/ceph-osd-*.log"
+else
+    echo -e "${YELLOW}No OSDs were created${NC}"
+    echo ""
+    echo "Troubleshooting:"
+    echo "1. Check Ceph cluster connectivity: ceph health"
+    echo "2. Check bootstrap keyring: ls -la $BOOTSTRAP_KEYRING"
+    echo "3. Check logs: /tmp/ceph-osd-*.log"
+    echo "4. Try manual creation: ceph-volume lvm create --data /dev/sdc"
+fi
+echo "=========================================="
+echo ""
+

scripts/fix-ceph-bootstrap-and-create-osds.sh

@@ -0,0 +1,147 @@
+#!/bin/bash
+# Fix Ceph bootstrap keyring issue and create OSDs
+# Run on R630-01 as root
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")
+
+echo "=========================================="
+echo "Fix Ceph Bootstrap and Create OSDs"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+echo -e "${BLUE}=== Step 1: Check Ceph Bootstrap Keyring ===${NC}"
+echo "-----------------------------------"
+
+# Check for bootstrap keyring in standard locations
+BOOTSTRAP_KEYRING=""
+if [ -f "/var/lib/ceph/bootstrap-osd/ceph.keyring" ]; then
+    BOOTSTRAP_KEYRING="/var/lib/ceph/bootstrap-osd/ceph.keyring"
+    echo -e "${GREEN}  Found: $BOOTSTRAP_KEYRING${NC}"
+elif [ -f "/etc/pve/priv/ceph.client.bootstrap-osd.keyring" ]; then
+    BOOTSTRAP_KEYRING="/etc/pve/priv/ceph.client.bootstrap-osd.keyring"
+    echo -e "${GREEN}  Found: $BOOTSTRAP_KEYRING${NC}"
+else
+    echo -e "${YELLOW}  Bootstrap keyring not found in standard locations${NC}"
+    echo "  Searching for keyring files..."
+    find /var/lib/ceph /etc/pve -name "*bootstrap*keyring" 2>/dev/null | head -5
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 2: Check Ceph Cluster Status ===${NC}"
+echo "-----------------------------------"
+
+# Check if Ceph is accessible
+if command -v ceph &> /dev/null; then
+    echo "Testing Ceph connectivity..."
+    if ceph health &>/dev/null; then
+        echo -e "${GREEN}  ✓ Ceph cluster is accessible${NC}"
+        ceph health
+    else
+        echo -e "${RED}  ✗ Ceph cluster not accessible${NC}"
+        echo "  Checking Ceph services..."
+        systemctl status ceph.target 2>/dev/null || echo "  Ceph services not running"
+    fi
+else
+    echo -e "${YELLOW}  ceph command not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 3: Check Ceph Monitors ===${NC}"
+echo "-----------------------------------"
+if command -v ceph &> /dev/null; then
+    ceph mon stat 2>/dev/null || echo -e "${YELLOW}  Cannot get monitor status${NC}"
+    ceph mon dump 2>/dev/null || echo -e "${YELLOW}  Cannot get monitor dump${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 4: Try Creating OSD with Different Method ===${NC}"
+echo "-----------------------------------"
+
+# Try using pveceph if available (Proxmox Ceph management)
+if command -v pveceph &> /dev/null; then
+    echo "Using pveceph to create OSDs..."
+    for drive in "${DRIVES[@]}"; do
+        if [ ! -b "/dev/$drive" ]; then
+            continue
+        fi
+        echo -e "${BLUE}  Creating OSD on /dev/$drive using pveceph...${NC}"
+        pveceph create /dev/$drive 2>&1 || echo -e "${YELLOW}    pveceph method failed, will try ceph-volume${NC}"
+    done
+else
+    echo -e "${YELLOW}  pveceph not available${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 5: Create OSDs with ceph-volume (with fixes) ===${NC}"
+echo "-----------------------------------"
+
+# Try to fix bootstrap keyring issue
+if [ -z "$BOOTSTRAP_KEYRING" ] && [ -d "/var/lib/ceph/bootstrap-osd" ]; then
+    echo "Checking if we need to create bootstrap keyring..."
+    # Check if we can get it from the cluster
+    if command -v ceph &> /dev/null && ceph auth get client.bootstrap-osd &>/dev/null; then
+        echo "  Getting bootstrap keyring from cluster..."
+        mkdir -p /var/lib/ceph/bootstrap-osd
+        ceph auth get client.bootstrap-osd -o /var/lib/ceph/bootstrap-osd/ceph.keyring 2>/dev/null || true
+    fi
+fi
+
+# Try creating OSDs
+osd_count=0
+for drive in "${DRIVES[@]}"; do
+    if [ ! -b "/dev/$drive" ]; then
+        continue
+    fi
+    
+    echo -e "${BLUE}  Creating OSD on /dev/$drive...${NC}"
+    
+    # Try with explicit keyring path
+    if [ ! -z "$BOOTSTRAP_KEYRING" ]; then
+        export CEPH_ARGS="--keyring $BOOTSTRAP_KEYRING"
+    fi
+    
+    # Try creating OSD
+    if ceph-volume lvm create --data "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
+        echo -e "${GREEN}    ✓ OSD created on /dev/$drive${NC}"
+        osd_count=$((osd_count + 1))
+    else
+        echo -e "${YELLOW}    ⚠ OSD creation had issues, checking status...${NC}"
+        # Check if OSD was partially created
+        if ceph osd tree 2>/dev/null | grep -q "osd\."; then
+            echo "    Checking if OSD exists..."
+            ceph osd tree 2>/dev/null | tail -5
+        fi
+    fi
+    echo ""
+done
+
+echo "=========================================="
+echo -e "${GREEN}Process Complete!${NC}"
+echo "=========================================="
+echo ""
+echo "Created $osd_count OSD(s) out of ${#DRIVES[@]} drives"
+echo ""
+
+echo "Current OSD status:"
+ceph osd tree 2>/dev/null || echo "Cannot get OSD tree"
+echo ""
+
+echo "Ceph health:"
+ceph health 2>/dev/null || echo "Cannot get health"
+echo ""
+

scripts/fix-ceph-rbd-storage.sh

@@ -0,0 +1,163 @@
+#!/bin/bash
+# Fix Ceph RBD storage configuration on r630-01
+# This script fixes the "No such file or directory" error when accessing RBD storage
+
+set -e
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  FIX CEPH RBD STORAGE CONFIGURATION"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+# Check if running on r630-01
+HOSTNAME=$(hostname)
+if [ "$HOSTNAME" != "r630-01" ]; then
+    echo "⚠️  WARNING: This script should be run on r630-01"
+    echo "   Current hostname: $HOSTNAME"
+    echo ""
+    echo "To run remotely via SSH:"
+    echo "  ssh root@192.168.11.11 'bash -s' < $0"
+    echo ""
+    read -p "Continue anyway? (y/N) " -n 1 -r
+    echo
+    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+        exit 1
+    fi
+fi
+
+echo "Step 1: Checking Ceph cluster status..."
+if ! command -v ceph &> /dev/null; then
+    echo "❌ ERROR: ceph command not found. Ceph packages may not be installed."
+    exit 1
+fi
+
+CEPH_HEALTH=$(ceph health detail)
+echo "$CEPH_HEALTH"
+if [[ "$CEPH_HEALTH" == *"HEALTH_ERR"* ]]; then
+    echo "❌ ERROR: Ceph cluster is in HEALTH_ERR state. Please fix Ceph issues first."
+    exit 1
+fi
+echo ""
+
+echo "Step 2: Checking RBD pool 'rbd'..."
+if ! ceph osd pool ls | grep -q "^rbd$"; then
+    echo "  Creating RBD pool 'rbd'..."
+    ceph osd pool create rbd 32 32 || { echo "❌ Failed to create RBD pool 'rbd'"; exit 1; }
+    echo "✅ RBD pool 'rbd' created."
+else
+    echo "✅ RBD pool 'rbd' exists."
+fi
+echo ""
+
+echo "Step 3: Initializing RBD pool..."
+if ! rbd pool init rbd 2>/dev/null; then
+    echo "⚠️  RBD pool 'rbd' already initialized or initialization failed."
+else
+    echo "✅ RBD pool 'rbd' initialized."
+fi
+echo ""
+
+echo "Step 4: Enabling RBD application on pool..."
+if ! ceph osd pool application get rbd 2>/dev/null | grep -q "rbd"; then
+    echo "  Enabling RBD application..."
+    ceph osd pool application enable rbd rbd || { echo "❌ Failed to enable RBD application"; exit 1; }
+    echo "✅ RBD application enabled."
+else
+    echo "✅ RBD application already enabled."
+fi
+echo ""
+
+echo "Step 5: Checking RBD pool status..."
+if rbd ls -p rbd &>/dev/null; then
+    echo "✅ RBD pool is accessible."
+    echo "   Images in pool:"
+    rbd ls -p rbd | head -5 || echo "   (no images yet)"
+else
+    echo "⚠️  RBD pool access test failed, but pool exists."
+fi
+echo ""
+
+echo "Step 6: Checking Proxmox RBD storage configuration..."
+if pvesm status 2>/dev/null | grep -q "ceph-rbd"; then
+    echo "✅ ceph-rbd storage exists in Proxmox."
+    echo ""
+    echo "Storage details:"
+    pvesm status | grep "ceph-rbd"
+    echo ""
+    
+    # Check if storage needs to be removed and re-added
+    echo "Step 7: Verifying storage configuration..."
+    if pvesm status | grep "ceph-rbd" | grep -q "inactive"; then
+        echo "⚠️  Storage is inactive. Attempting to fix..."
+        
+        # Get storage config
+        STORAGE_CFG="/etc/pve/storage.cfg"
+        if [ -f "$STORAGE_CFG" ]; then
+            echo "   Current ceph-rbd storage config:"
+            grep -A 10 "^rbd: ceph-rbd" "$STORAGE_CFG" || echo "   (not found in config)"
+            echo ""
+            
+            echo "   Removing and re-adding ceph-rbd storage..."
+            # Remove storage
+            pvesm remove ceph-rbd 2>/dev/null || echo "   (storage may not exist or already removed)"
+            
+            # Re-add storage
+            echo "   Re-adding ceph-rbd storage..."
+            pvesm add rbd ceph-rbd \
+                --nodes r630-01 \
+                --pool rbd \
+                --content images \
+                --monhost 192.168.11.10:6789,192.168.11.11:6789 \
+                --username admin 2>&1 || {
+                echo "❌ Failed to re-add ceph-rbd storage."
+                echo "   Try manually via Web UI: Datacenter → Storage → Add → RBD"
+                exit 1
+            }
+            echo "✅ ceph-rbd storage re-added."
+        fi
+    else
+        echo "✅ Storage appears to be active."
+    fi
+else
+    echo "⚠️  ceph-rbd storage not found in Proxmox."
+    echo "   Adding ceph-rbd storage..."
+    pvesm add rbd ceph-rbd \
+        --nodes r630-01 \
+        --pool rbd \
+        --content images \
+        --monhost 192.168.11.10:6789,192.168.11.11:6789 \
+        --username admin 2>&1 || {
+        echo "❌ Failed to add ceph-rbd storage."
+        exit 1
+    }
+    echo "✅ ceph-rbd storage added."
+fi
+echo ""
+
+echo "Step 8: Testing RBD storage access..."
+if pvesm list ceph-rbd &>/dev/null; then
+    echo "✅ RBD storage is accessible via Proxmox."
+    echo "   Storage content:"
+    pvesm list ceph-rbd | head -5 || echo "   (empty - no images yet)"
+else
+    echo "⚠️  RBD storage listing failed, but storage exists."
+    echo "   Error details:"
+    pvesm list ceph-rbd 2>&1 | head -3
+fi
+echo ""
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  VERIFICATION"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "Checking storage status..."
+pvesm status | grep "ceph-rbd" || echo "Storage 'ceph-rbd' not found in pvesm status"
+echo ""
+
+echo "✅ Fix complete! RBD storage should now be working."
+echo ""
+echo "If errors persist, check:"
+echo "  1. Ceph cluster health: ceph health"
+echo "  2. RBD pool exists: ceph osd pool ls | grep rbd"
+echo "  3. RBD pool is initialized: rbd pool init rbd"
+echo "  4. Ceph keyring is accessible: ls -la /etc/pve/priv/ceph/ceph-rbd.keyring"
+

scripts/fix-ceph-services.sh

@@ -0,0 +1,155 @@
+#!/bin/bash
+# Fix failed Ceph services and retry OSD creation
+# Run on R630-01 as root
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")
+
+echo "=========================================="
+echo "Fix Ceph Services and Create OSDs"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+echo -e "${BLUE}=== Step 1: Check Failed Services ===${NC}"
+echo "-----------------------------------"
+
+echo "Monitor service status:"
+systemctl status ceph-mon@r630-01.service --no-pager | head -15 || true
+echo ""
+
+echo "OSD service status:"
+systemctl status ceph-osd@1.service --no-pager | head -15 || true
+echo ""
+
+echo -e "${BLUE}=== Step 2: Check Service Logs ===${NC}"
+echo "-----------------------------------"
+
+echo "Monitor service logs (last 20 lines):"
+journalctl -u ceph-mon@r630-01.service -n 20 --no-pager | tail -10 || true
+echo ""
+
+echo "OSD service logs (last 20 lines):"
+journalctl -u ceph-osd@1.service -n 20 --no-pager | tail -10 || true
+echo ""
+
+echo -e "${BLUE}=== Step 3: Attempt to Start Services ===${NC}"
+echo "-----------------------------------"
+
+echo "Starting monitor service..."
+if systemctl start ceph-mon@r630-01.service 2>&1; then
+    sleep 3
+    if systemctl is-active --quiet ceph-mon@r630-01.service; then
+        echo -e "${GREEN}  ✓ Monitor service started${NC}"
+    else
+        echo -e "${YELLOW}  ⚠ Monitor service started but not active${NC}"
+    fi
+else
+    echo -e "${RED}  ✗ Failed to start monitor service${NC}"
+    echo "  Check logs above for errors"
+fi
+echo ""
+
+echo "Starting OSD service..."
+if systemctl start ceph-osd@1.service 2>&1; then
+    sleep 3
+    if systemctl is-active --quiet ceph-osd@1.service; then
+        echo -e "${GREEN}  ✓ OSD service started${NC}"
+    else
+        echo -e "${YELLOW}  ⚠ OSD service started but not active${NC}"
+    fi
+else
+    echo -e "${RED}  ✗ Failed to start OSD service${NC}"
+    echo "  Check logs above for errors"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 4: Test Cluster Connectivity ===${NC}"
+echo "-----------------------------------"
+
+echo "Testing cluster connectivity (5 second timeout)..."
+if timeout 5 ceph health 2>&1; then
+    echo -e "${GREEN}  ✓ Cluster is accessible${NC}"
+    echo ""
+    echo "Ceph health:"
+    ceph health
+    echo ""
+    echo "OSD tree:"
+    ceph osd tree
+else
+    echo -e "${YELLOW}  ⚠ Cluster still not accessible${NC}"
+    echo "  Services may need more time or have other issues"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 5: Create OSDs (if cluster accessible) ===${NC}"
+echo "-----------------------------------"
+
+# Test if cluster is accessible
+if timeout 5 ceph health &>/dev/null; then
+    echo "Cluster is accessible. Creating OSDs..."
+    echo ""
+    
+    osd_count=0
+    for drive in "${DRIVES[@]}"; do
+        if [ ! -b "/dev/$drive" ]; then
+            continue
+        fi
+        
+        echo -e "${BLUE}  Creating OSD on /dev/$drive...${NC}"
+        
+        # Try pveceph first (Proxmox tool)
+        if command -v pveceph &> /dev/null; then
+            if pveceph create "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
+                echo -e "${GREEN}    ✓ OSD created on /dev/$drive (using pveceph)${NC}"
+                osd_count=$((osd_count + 1))
+                continue
+            fi
+        fi
+        
+        # Fallback to ceph-volume
+        if ceph-volume lvm create --data "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
+            echo -e "${GREEN}    ✓ OSD created on /dev/$drive${NC}"
+            osd_count=$((osd_count + 1))
+        else
+            echo -e "${YELLOW}    ⚠ OSD creation had issues${NC}"
+        fi
+        echo ""
+    done
+    
+    echo "Created $osd_count OSD(s) out of ${#DRIVES[@]} drives"
+    echo ""
+    
+    echo "Verifying OSDs:"
+    ceph osd tree
+    echo ""
+    ceph health
+else
+    echo -e "${YELLOW}Cluster not accessible. Cannot create OSDs yet.${NC}"
+    echo ""
+    echo "Next steps:"
+    echo "1. Fix service issues (check logs above)"
+    echo "2. Ensure services are running"
+    echo "3. Test cluster connectivity"
+    echo "4. Retry OSD creation"
+fi
+echo ""
+
+echo "=========================================="
+echo -e "${GREEN}Process Complete${NC}"
+echo "=========================================="
+echo ""
+

scripts/fix-containerd-tag.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+set -e
+
+echo "=========================================="
+echo "FIX CONTAINERD TAG - Force Latest to New Image"
+echo "=========================================="
+echo ""
+
+# The new image manifest digest
+NEW_DIGEST="sha256:219e9651ca232b6320614f2f2289ba4c9bf8d29809a90319038a6390871a4c26"
+IMAGE_NAME="docker.io/library/crossplane-provider-proxmox"
+
+echo "Step 1: Listing all crossplane images..."
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox || echo "  (No images found)"
+echo ""
+
+echo "Step 2: Removing ALL crossplane-provider-proxmox images..."
+ALL_IMAGES=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}" || true)
+if [ -n "$ALL_IMAGES" ]; then
+    echo "$ALL_IMAGES" | while read -r img; do
+        echo "  Removing: $img"
+        sudo ctr -n k8s.io images rm "$img" 2>/dev/null || true
+    done
+    echo "  ✅ All images removed"
+else
+    echo "  (No images to remove)"
+fi
+echo ""
+
+echo "Step 3: Re-importing fresh image..."
+if [ ! -f /tmp/crossplane-fresh-nuclear.tar ]; then
+    echo "  ❌ ERROR: Image tar not found at /tmp/crossplane-fresh-nuclear.tar"
+    exit 1
+fi
+sudo ctr -n k8s.io images import /tmp/crossplane-fresh-nuclear.tar
+echo "  ✅ Image imported"
+echo ""
+
+echo "Step 4: Verifying import and checking digest..."
+IMPORTED=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}:latest" || true)
+if [ -z "$IMPORTED" ]; then
+    echo "  ❌ ERROR: Image ${IMAGE_NAME}:latest not found after import!"
+    exit 1
+fi
+echo "  ✅ Image found: $IMPORTED"
+echo ""
+
+echo "Step 5: Checking image digest..."
+IMAGE_DIGEST=$(sudo ctr -n k8s.io images ls --format '{{.Target.Digest}}' | grep "${IMAGE_NAME}:latest" | head -1 || true)
+if [ -n "$IMAGE_DIGEST" ]; then
+    echo "  Image digest: $IMAGE_DIGEST"
+    if echo "$IMAGE_DIGEST" | grep -q "219e9651"; then
+        echo "  ✅ Correct digest (new image)"
+    else
+        echo "  ⚠️  Wrong digest - may still be old image"
+    fi
+else
+    echo "  ⚠️  Could not determine digest"
+fi
+echo ""
+
+echo "Step 6: Final image list:"
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox
+echo ""
+
+echo "=========================================="
+echo "CONTAINERD TAG FIX COMPLETE"
+echo "=========================================="
+echo ""
+echo "Now delete and recreate the pod:"
+echo "  kubectl delete pod -n crossplane-system -l app=crossplane-provider-proxmox"
+echo "  kubectl wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s"
+echo ""
+echo "Then verify:"
+echo "  kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox | grep '\[FIXED CODE\]'"
+

scripts/fix-deployment-issues.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+# Fix Deployment Issues
+# Diagnoses and fixes common deployment issues
+
+set -e
+
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+echo "=========================================="
+echo "Fixing Deployment Issues"
+echo "=========================================="
+echo ""
+
+# 1. Check connectivity
+echo -e "${BLUE}1. Checking connectivity...${NC}"
+if ping -c 2 192.168.11.10 &>/dev/null; then
+    echo -e "${GREEN}✓${NC} ML110-01 reachable"
+else
+    echo -e "${RED}✗${NC} ML110-01 not reachable"
+fi
+
+if ping -c 2 192.168.11.11 &>/dev/null; then
+    echo -e "${GREEN}✓${NC} R630-01 reachable"
+else
+    echo -e "${RED}✗${NC} R630-01 not reachable"
+fi
+
+echo ""
+
+# 2. Check provider pod
+echo -e "${BLUE}2. Checking provider pod...${NC}"
+POD_STATUS=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "NOT_FOUND")
+if [[ "$POD_STATUS" == "Running" ]]; then
+    echo -e "${GREEN}✓${NC} Provider pod is running"
+    READY=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].ready}' 2>/dev/null)
+    if [[ "$READY" == "true" ]]; then
+        echo -e "${GREEN}✓${NC} Provider pod is ready"
+    else
+        echo -e "${YELLOW}⚠${NC} Provider pod not ready, restarting..."
+        kubectl delete pod -n crossplane-system -l app=crossplane-provider-proxmox
+        echo "Waiting for pod to restart..."
+        sleep 10
+    fi
+else
+    echo -e "${RED}✗${NC} Provider pod not running (status: $POD_STATUS)"
+fi
+
+echo ""
+
+# 3. Check credentials
+echo -e "${BLUE}3. Checking credentials...${NC}"
+if kubectl get secret proxmox-credentials -n crossplane-system &>/dev/null; then
+    echo -e "${GREEN}✓${NC} Credentials secret exists"
+    KEYS=$(kubectl get secret proxmox-credentials -n crossplane-system -o jsonpath='{.data}' 2>/dev/null | jq -r 'keys[]' 2>/dev/null || echo "")
+    if [[ "$KEYS" == *"token"* && "$KEYS" == *"tokenid"* ]]; then
+        echo -e "${GREEN}✓${NC} Token format correct"
+    else
+        echo -e "${YELLOW}⚠${NC} Credentials format may be incorrect"
+    fi
+else
+    echo -e "${RED}✗${NC} Credentials secret not found"
+fi
+
+echo ""
+
+# 4. Check provider config
+echo -e "${BLUE}4. Checking provider config...${NC}"
+if kubectl get providerconfig proxmox-provider-config -n crossplane-system &>/dev/null; then
+    echo -e "${GREEN}✓${NC} ProviderConfig exists"
+    SITES=$(kubectl get providerconfig proxmox-provider-config -n crossplane-system -o jsonpath='{.spec.sites[*].name}' 2>/dev/null)
+    if [[ "$SITES" == *"site-1"* && "$SITES" == *"site-2"* ]]; then
+        echo -e "${GREEN}✓${NC} Both sites configured"
+    else
+        echo -e "${RED}✗${NC} Sites not correctly configured: $SITES"
+    fi
+else
+    echo -e "${RED}✗${NC} ProviderConfig not found"
+fi
+
+echo ""
+
+# 5. Check VM status
+echo -e "${BLUE}5. Checking VM status...${NC}"
+TOTAL=$(kubectl get proxmoxvm -A --no-headers 2>&1 | wc -l)
+echo "Total VMs: $TOTAL"
+
+# Count VMs with errors
+ERRORS=$(kubectl get proxmoxvm -A -o json 2>&1 | jq -r '.items[] | select(.status.conditions[]?.type=="Failed") | .metadata.name' 2>/dev/null | wc -l)
+if [[ $ERRORS -gt 0 ]]; then
+    echo -e "${YELLOW}⚠${NC} $ERRORS VMs with errors"
+    echo "Failed VMs:"
+    kubectl get proxmoxvm -A -o json 2>&1 | jq -r '.items[] | select(.status.conditions[]?.type=="Failed") | "  - \(.metadata.name): \(.status.conditions[]? | select(.type=="Failed") | .message)"' 2>/dev/null | head -5
+else
+    echo -e "${GREEN}✓${NC} No VMs with errors"
+fi
+
+echo ""
+
+# 6. Recommendations
+echo "=========================================="
+echo "Recommendations"
+echo "=========================================="
+echo ""
+
+if [[ $ERRORS -gt 0 ]]; then
+    echo "1. Check provider logs:"
+    echo "   kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox -f"
+    echo ""
+    echo "2. Verify Proxmox API access:"
+    echo "   Test from Kubernetes cluster to both Proxmox nodes"
+    echo ""
+    echo "3. Check credentials:"
+    echo "   Verify token is valid for both Proxmox nodes"
+    echo ""
+fi
+
+echo "4. Monitor VM creation:"
+echo "   kubectl get proxmoxvm -A -w"
+echo ""
+

scripts/fix-provider-image.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+# Script to fix provider image in containerd
+# Removes all old images and imports the fixed one
+
+set -e
+
+IMAGE_TAR="/tmp/crossplane-latest-fixed.tar"
+
+echo "=========================================="
+echo "Fixing Provider Image in Containerd"
+echo "=========================================="
+echo ""
+
+if [ ! -f "$IMAGE_TAR" ]; then
+    echo "❌ Error: Image tar file not found: $IMAGE_TAR"
+    exit 1
+fi
+
+echo "✅ Image tar file found: $IMAGE_TAR"
+echo ""
+
+echo "Step 1: Removing all old crossplane-provider-proxmox images..."
+sudo ctr -n k8s.io images rm docker.io/library/crossplane-provider-proxmox:latest 2>/dev/null || echo "  (latest not found or already removed)"
+sudo ctr -n k8s.io images rm docker.io/library/crossplane-provider-proxmox:v1.0.0-fixed 2>/dev/null || echo "  (v1.0.0-fixed not found or already removed)"
+
+# Try to remove by digest if they exist
+sudo ctr -n k8s.io images rm sha256:5d83df9f8d127aa8988ace01254f4adbb6336f0ad59be8193349b3eacd972cfc 2>/dev/null || echo "  (old image by digest not found)"
+
+echo ""
+echo "Step 2: Listing remaining crossplane images..."
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox || echo "  (no crossplane images found - good!)"
+echo ""
+
+echo "Step 3: Importing the fixed image..."
+if sudo ctr -n k8s.io images import "$IMAGE_TAR"; then
+    echo "✅ Image imported successfully"
+else
+    echo "❌ Failed to import image"
+    exit 1
+fi
+
+echo ""
+echo "Step 4: Verifying import..."
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox
+
+echo ""
+echo "=========================================="
+echo "Image Fix Complete!"
+echo "=========================================="
+echo ""
+echo "Next: Restart the provider pod:"
+echo "  kubectl delete pod -n crossplane-system -l app=crossplane-provider-proxmox"
+echo ""
+

scripts/fix-r630-mon-quorum.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# Fix R630-01 monitor to join quorum
+set -e
+
+echo "=== Fixing R630-01 Monitor Quorum ==="
+
+# Check current status
+echo "1. Checking monitor service status..."
+systemctl status ceph-mon@r630-01 --no-pager | head -15
+
+echo ""
+echo "2. Checking monitor directory..."
+ls -la /var/lib/ceph/mon/ceph-r630-01/ 2>&1 || echo "Monitor directory missing!"
+
+echo ""
+echo "3. Checking monitor logs..."
+journalctl -u ceph-mon@r630-01 -n 20 --no-pager | tail -20
+
+echo ""
+echo "4. Checking if monitor is listening..."
+ss -ltnp | grep -E "(:3300|:6789)" || echo "No monitor listeners found"
+
+echo ""
+echo "5. Attempting to add monitor to cluster..."
+# Get monmap from ml110-01
+echo "Fetching monmap from ml110-01..."
+ceph mon getmap -o /tmp/monmap 2>&1 || echo "Failed to get monmap"
+
+if [ -f /tmp/monmap ]; then
+    echo "Monmap retrieved, checking contents..."
+    monmaptool --print /tmp/monmap 2>&1
+    
+    # Check if r630-01 is in monmap
+    if ! monmaptool --print /tmp/monmap 2>&1 | grep -q "r630-01"; then
+        echo "r630-01 not in monmap, adding..."
+        # Get cluster FSID
+        FSID=$(ceph config get mon fsid 2>/dev/null || grep "fsid" /etc/pve/ceph.conf | awk '{print $3}')
+        if [ -z "$FSID" ]; then
+            FSID=$(grep "fsid" /etc/pve/ceph.conf | awk '{print $3}')
+        fi
+        
+        if [ -n "$FSID" ]; then
+            echo "Adding r630-01 to monmap with FSID: $FSID"
+            monmaptool --add r630-01 192.168.11.11 --fsid $FSID /tmp/monmap 2>&1
+            # Inject updated monmap
+            ceph mon setmap -i /tmp/monmap 2>&1 || echo "Failed to set monmap"
+        fi
+    else
+        echo "r630-01 already in monmap"
+    fi
+fi
+
+echo ""
+echo "6. Checking if monitor directory needs creation..."
+if [ ! -d "/var/lib/ceph/mon/ceph-r630-01" ] || [ -z "$(ls -A /var/lib/ceph/mon/ceph-r630-01 2>/dev/null)" ]; then
+    echo "Monitor directory missing or empty, creating..."
+    
+    # Get keyring
+    if [ -f "/var/lib/ceph/bootstrap-osd/ceph.keyring" ]; then
+        echo "Using bootstrap keyring to create monitor..."
+        mkdir -p /var/lib/ceph/mon/ceph-r630-01
+        chown ceph:ceph /var/lib/ceph/mon/ceph-r630-01
+        
+        # Get monmap and keyring
+        if [ -f /tmp/monmap ]; then
+            ceph-authtool --create-keyring /var/lib/ceph/mon/ceph-r630-01/keyring --gen-key -n mon. 2>&1 || true
+            ceph mon getmap -o /tmp/monmap 2>&1 || true
+            
+            if [ -f /tmp/monmap ] && [ -f /var/lib/ceph/mon/ceph-r630-01/keyring ]; then
+                echo "Creating monitor filesystem..."
+                ceph-mon --mkfs -i r630-01 --monmap /tmp/monmap --keyring /var/lib/ceph/mon/ceph-r630-01/keyring 2>&1
+                chown -R ceph:ceph /var/lib/ceph/mon/ceph-r630-01
+            fi
+        fi
+    else
+        echo "Bootstrap keyring not found, using pveceph..."
+        pveceph mon create 2>&1 || echo "pveceph mon create failed"
+    fi
+fi
+
+echo ""
+echo "7. Restarting monitor service..."
+systemctl restart ceph-mon@r630-01
+sleep 5
+systemctl status ceph-mon@r630-01 --no-pager | head -15
+
+echo ""
+echo "8. Waiting for quorum..."
+sleep 10
+ceph quorum_status 2>&1 | head -10
+
+echo ""
+echo "=== Done ==="
+echo "Check quorum with: ceph quorum_status"
+echo "Check cluster with: ceph -s"
+

scripts/force-new-image.sh

@@ -0,0 +1,163 @@
+#!/bin/bash
+set -e
+
+# Find kubectl
+KUBECTL=$(which kubectl 2>/dev/null || echo "")
+if [ -z "$KUBECTL" ]; then
+    for path in /usr/local/bin/kubectl /usr/bin/kubectl ~/.local/bin/kubectl; do
+        if [ -x "$path" ]; then
+            KUBECTL="$path"
+            break
+        fi
+    done
+fi
+
+IMAGE_NAME="docker.io/library/crossplane-provider-proxmox"
+NEW_DOCKER_ID="sha256:02dfb597dcb17ee74eab0bfe3483d5bbc5e0db66642bdbfd74bf9fcd2af40aaa"
+NEW_MANIFEST_DIGEST="sha256:219e9651ca232b6320614f2f2289ba4c9bf8d29809a90319038a6390871a4c26"
+
+echo "=========================================="
+echo "FORCE NEW IMAGE - Complete Cleanup"
+echo "=========================================="
+echo ""
+
+if [ -n "$KUBECTL" ]; then
+    echo "Using kubectl: $KUBECTL"
+    echo ""
+    echo "Step 1: Deleting deployment and pods..."
+    $KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo "  (Deployment not found)"
+    $KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo "  (Pods not found)"
+else
+    echo "⚠️  kubectl not found - skipping Kubernetes operations"
+    echo "  You'll need to manually delete/recreate the deployment"
+fi
+sleep 3
+
+echo ""
+echo "Step 2: Listing ALL crossplane images in containerd..."
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox || echo "  (No images found)"
+echo ""
+
+echo "Step 3: Removing ALL crossplane-provider-proxmox images (by name and digest)..."
+# Remove by name
+ALL_IMAGES=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}" || true)
+if [ -n "$ALL_IMAGES" ]; then
+    echo "$ALL_IMAGES" | while read -r img; do
+        echo "  Removing: $img"
+        sudo ctr -n k8s.io images rm "$img" 2>/dev/null || true
+    done
+fi
+
+# Also try to remove by the old image ID if it exists
+echo "  Attempting to remove old image by ID: 5d83df9f..."
+sudo ctr -n k8s.io images rm "${IMAGE_NAME}@sha256:5d83df9f8d127aa8988ace01254f4adbb6336f0ad59be8193349b3eacd972cfc" 2>/dev/null || true
+
+echo "  ✅ All images removed"
+echo ""
+
+echo "Step 4: Pruning containerd images..."
+sudo ctr -n k8s.io images prune 2>&1 | head -5 || echo "  (Prune completed)"
+echo ""
+
+echo "Step 5: Verifying image tar exists..."
+if [ ! -f /tmp/crossplane-fresh-nuclear.tar ]; then
+    echo "  ❌ ERROR: Image tar not found at /tmp/crossplane-fresh-nuclear.tar"
+    echo "  Please rebuild: sudo /home/intlc/projects/Sankofa/scripts/nuclear-cleanup-rebuild.sh"
+    exit 1
+fi
+echo "  ✅ Image tar found: $(ls -lh /tmp/crossplane-fresh-nuclear.tar | awk '{print $5}')"
+echo ""
+
+echo "Step 6: Re-importing fresh image..."
+sudo ctr -n k8s.io images import /tmp/crossplane-fresh-nuclear.tar
+echo "  ✅ Image imported"
+echo ""
+
+echo "Step 7: Verifying import and checking digest..."
+IMPORTED=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}:latest" || true)
+if [ -z "$IMPORTED" ]; then
+    echo "  ❌ ERROR: Image ${IMAGE_NAME}:latest not found after import!"
+    exit 1
+fi
+echo "  ✅ Image found: $IMPORTED"
+
+# Check the actual digest from the image list output
+IMAGE_DIGEST=$(sudo ctr -n k8s.io images ls | grep "${IMAGE_NAME}:latest" | grep -o "sha256:[a-f0-9]\{64\}" | head -1 || true)
+if [ -n "$IMAGE_DIGEST" ]; then
+    echo "  Image manifest digest: $IMAGE_DIGEST"
+    if echo "$IMAGE_DIGEST" | grep -q "219e9651"; then
+        echo "  ✅ Correct digest (new image)"
+    else
+        echo "  ⚠️  WARNING: Digest doesn't match expected new image!"
+        echo "     Expected: ...219e9651..."
+        echo "     Got: $IMAGE_DIGEST"
+    fi
+fi
+echo ""
+
+echo "Step 8: Final image list in containerd:"
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox
+echo ""
+
+if [ -n "$KUBECTL" ]; then
+    echo "Step 9: Recreating deployment..."
+    $KUBECTL apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml
+    echo "  ✅ Deployment created"
+    echo ""
+
+    echo "Step 10: Waiting for pod to be ready..."
+    $KUBECTL wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s || {
+        echo "  ⚠️  Pod not ready, checking status..."
+        $KUBECTL get pod -n crossplane-system -l app=crossplane-provider-proxmox
+        exit 1
+    }
+    echo "  ✅ Pod is ready"
+    echo ""
+
+    echo "Step 11: Checking pod image ID..."
+    POD_IMAGE_ID=$($KUBECTL get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].imageID}')
+    echo "  Pod image ID: $POD_IMAGE_ID"
+    if echo "$POD_IMAGE_ID" | grep -q "5d83df9f"; then
+        echo "  ⚠️  WARNING: Pod is still using OLD image ID!"
+        echo "     This means containerd is still resolving 'latest' to old image."
+    elif echo "$POD_IMAGE_ID" | grep -q "219e9651\|02dfb597"; then
+        echo "  ✅ Pod is using NEW image!"
+    else
+        echo "  ⚠️  Unknown image ID"
+    fi
+    echo ""
+
+    echo "Step 12: Waiting for logs to generate..."
+    sleep 5
+
+    echo ""
+    echo "Step 13: Checking for [FIXED CODE] message..."
+    if $KUBECTL logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 2>&1 | grep -q '\[FIXED CODE\]'; then
+        echo "  ✅ [FIXED CODE] message found! Fix is active."
+        $KUBECTL logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 | grep '\[FIXED CODE\]' | head -3
+    else
+        echo "  ❌ [FIXED CODE] message NOT found!"
+        echo ""
+        echo "  Checking recent logs for Cookie header..."
+        $KUBECTL logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=20 | grep -E "Cookie|Authorization" | head -5 || echo "  (No relevant logs found)"
+        echo ""
+        echo "  This indicates the old code is still running."
+        echo "  Possible causes:"
+        echo "    1. Containerd is caching the old image"
+        echo "    2. The image ID shown is a layer digest, not manifest digest"
+        echo "    3. Need to restart containerd (disruptive)"
+    fi
+else
+    echo "Step 9: Skipping Kubernetes operations (kubectl not found)"
+    echo ""
+    echo "  Please manually run:"
+    echo "    kubectl apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml"
+    echo "    kubectl wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s"
+    echo "    kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox | grep '\[FIXED CODE\]'"
+fi
+
+echo ""
+echo "=========================================="
+echo "FORCE NEW IMAGE COMPLETE"
+echo "=========================================="
+

scripts/format-ssds-and-check-proxmox.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+# Format 6x 250GB SSDs and check if Proxmox recognizes them
+# Run on R630-01 as root
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")
+
+echo "=========================================="
+echo "Format 250GB SSDs and Check Proxmox Recognition"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+echo -e "${BLUE}=== Step 1: Current Drive Status ===${NC}"
+echo "-----------------------------------"
+echo "Checking current status of 250GB drives:"
+for drive in "${DRIVES[@]}"; do
+    if [ -b "/dev/$drive" ]; then
+        size=$(fdisk -l "/dev/$drive" 2>/dev/null | grep "^Disk /dev/$drive" | awk '{print $3, $4}')
+        echo "  /dev/$drive: $size"
+        # Check if has partitions
+        partitions=$(lsblk -n /dev/$drive 2>/dev/null | grep -c "part" || echo "0")
+        if [ "$partitions" -gt 0 ]; then
+            echo -e "    ${YELLOW}Has $partitions partition(s)${NC}"
+        else
+            echo -e "    ${GREEN}No partitions (clean)${NC}"
+        fi
+    else
+        echo -e "  ${YELLOW}/dev/$drive not found${NC}"
+    fi
+done
+echo ""
+
+echo -e "${BLUE}=== Step 2: Format Drives (Create Partition Table) ===${NC}"
+echo "-----------------------------------"
+echo -e "${YELLOW}This will create a new partition table on each drive${NC}"
+echo ""
+
+for drive in "${DRIVES[@]}"; do
+    if [ ! -b "/dev/$drive" ]; then
+        echo -e "${YELLOW}  /dev/$drive not found, skipping...${NC}"
+        continue
+    fi
+    
+    echo -e "${BLUE}  Formatting /dev/$drive...${NC}"
+    
+    # Unmount any partitions
+    umount /dev/${drive}* 2>/dev/null || true
+    
+    # Create new GPT partition table
+    echo "    Creating GPT partition table..."
+    parted -s "/dev/$drive" mklabel gpt 2>&1 || echo "    (Partition table may already be GPT)"
+    
+    # Optionally create a single partition (we'll leave it unformatted for now)
+    # This makes the drive visible but doesn't format it with a filesystem
+    echo "    Creating single partition..."
+    parted -s "/dev/$drive" mkpart primary 0% 100% 2>&1 || echo "    (Partition may already exist)"
+    
+    # Re-read partition table
+    partprobe "/dev/$drive" 2>/dev/null || true
+    
+    echo -e "${GREEN}    ✓ /dev/$drive formatted${NC}"
+    echo ""
+done
+
+echo -e "${BLUE}=== Step 3: Check Block Devices ===${NC}"
+echo "-----------------------------------"
+echo "All block devices:"
+lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL | grep -E "NAME|sd"
+echo ""
+
+echo -e "${BLUE}=== Step 4: Check Proxmox Storage ===${NC}"
+echo "-----------------------------------"
+echo "Proxmox storage status:"
+if command -v pvesm &> /dev/null; then
+    pvesm status 2>/dev/null || echo -e "${YELLOW}pvesm command failed${NC}"
+else
+    echo -e "${YELLOW}pvesm command not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 5: Check Available Disks in Proxmox ===${NC}"
+echo "-----------------------------------"
+echo "Checking if Proxmox can see the drives..."
+
+# Check via pvesm
+if command -v pvesm &> /dev/null; then
+    echo "Available storage:"
+    pvesm status 2>/dev/null | grep -E "Name|Type|Status" || true
+fi
+
+# Check via lsblk (what Proxmox would see)
+echo ""
+echo "Drives visible to system:"
+for drive in "${DRIVES[@]}"; do
+    if [ -b "/dev/$drive" ]; then
+        info=$(lsblk -n -o SIZE,TYPE,MOUNTPOINT,FSTYPE /dev/$drive 2>/dev/null | head -1)
+        echo "  /dev/$drive: $info"
+    fi
+done
+echo ""
+
+echo -e "${BLUE}=== Step 6: Check via Proxmox API (if accessible) ===${NC}"
+echo "-----------------------------------"
+echo "Note: To check via Proxmox web UI:"
+echo "  1. Go to Datacenter > Storage"
+echo "  2. Click 'Add' > 'Disk' or 'Directory'"
+echo "  3. Check if drives are listed"
+echo ""
+
+echo "=========================================="
+echo -e "${GREEN}Formatting Complete!${NC}"
+echo "=========================================="
+echo ""
+echo "Summary:"
+echo "--------"
+echo "All 6 drives have been formatted with GPT partition tables"
+echo ""
+echo "To check in Proxmox Web UI:"
+echo "  1. Login to Proxmox: https://192.168.11.11:8006"
+echo "  2. Go to: Datacenter > Storage"
+echo "  3. Click 'Add' to see available disks"
+echo ""
+echo "To check via command line:"
+echo "  pvesm status"
+echo "  lsblk"
+echo ""
+

scripts/fresh-ceph-install-r630.sh

@@ -0,0 +1,163 @@
+#!/bin/bash
+# Fresh Ceph installation for R630-01 (r630-01.sankofa.nexus)
+set -e
+
+HOSTNAME="r630-01"
+FQDN="r630-01.sankofa.nexus"
+PUBLIC_NETWORK="192.168.11.0/24"
+CLUSTER_NETWORK="192.168.11.0/24"
+FSID="5fb968ae-12ab-405f-b05f-0df29a168328"
+
+echo "=== Fresh Ceph Installation for $FQDN ==="
+echo ""
+
+# Verify we're on the right host
+CURRENT_HOST=$(hostname)
+if [ "$CURRENT_HOST" != "$HOSTNAME" ]; then
+    echo "WARNING: Hostname is $CURRENT_HOST, expected $HOSTNAME"
+    read -p "Continue anyway? (y/N) " -n 1 -r
+    echo
+    if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+        exit 1
+    fi
+fi
+
+echo "=== Step 1: Installing Ceph packages ==="
+apt-get update
+apt-get install -y ceph ceph-common ceph-base ceph-volume
+
+echo ""
+echo "=== Step 2: Cleaning up any existing Ceph data ==="
+systemctl stop ceph-*.service 2>/dev/null || true
+systemctl disable ceph-*.service 2>/dev/null || true
+rm -rf /var/lib/ceph/*
+rm -rf /var/log/ceph/*
+rm -rf /etc/ceph/*
+rm -rf /etc/pve/ceph.conf
+rm -rf /run/ceph/*
+
+# Unmount any OSD directories
+for osd_dir in /var/lib/ceph/osd/ceph-*; do
+    if mountpoint -q "$osd_dir" 2>/dev/null; then
+        umount -f "$osd_dir" 2>/dev/null || true
+    fi
+done
+
+# Remove Ceph LVM volumes
+for vg in $(vgs --noheadings -o vg_name | grep ceph); do
+    vgremove -f "$vg" 2>/dev/null || true
+done
+
+# Wipe Ceph drives
+for disk in sdc sdd sde sdf sdg sdh; do
+    if [ -e "/dev/$disk" ]; then
+        echo "Wiping /dev/$disk..."
+        wipefs -a /dev/$disk 2>/dev/null || true
+        dd if=/dev/zero of=/dev/$disk bs=1M count=100 2>/dev/null || true
+    fi
+done
+
+echo ""
+echo "=== Step 3: Creating Ceph configuration ==="
+cat > /etc/pve/ceph.conf <<EOF
+[global]
+	auth_client_required = cephx
+	auth_cluster_required = cephx
+	auth_service_required = cephx
+	cluster_network = $CLUSTER_NETWORK
+	fsid = $FSID
+	mon_allow_pool_delete = true
+	mon_host = 192.168.11.10 192.168.11.11
+	ms_bind_ipv4 = true
+	ms_bind_ipv6 = false
+	osd_pool_default_min_size = 2
+	osd_pool_default_size = 3
+	public_network = $PUBLIC_NETWORK
+
+[client]
+	keyring = /etc/pve/priv/\$cluster.\$name.keyring
+
+[client.crash]
+	keyring = /etc/pve/ceph/\$cluster.\$name.keyring
+
+[mds]
+	keyring = /var/lib/ceph/mds/ceph-\$id/keyring
+
+[mds.ml110-01]
+	host = ml110-01
+	mds_standby_for_name = pve
+
+[mds.r630-01]
+	host = r630-01
+	mds_standby_for_name = pve
+
+[mon.ml110-01]
+	public_addr = 192.168.11.10
+
+[mon.r630-01]
+	public_addr = 192.168.11.11
+EOF
+
+# Create symlink
+ln -sf /etc/pve/ceph.conf /etc/ceph/ceph.conf
+
+echo "Configuration created"
+
+echo ""
+echo "=== Step 4: Initializing Ceph cluster (if not already done) ==="
+# Check if cluster is already initialized by checking for bootstrap keyring
+if [ ! -f "/var/lib/ceph/bootstrap-osd/ceph.keyring" ]; then
+    echo "Cluster not initialized. Initializing..."
+    pveceph init --network $PUBLIC_NETWORK --cluster-network $CLUSTER_NETWORK 2>&1 || {
+        echo "Init may have already been done or failed. Continuing..."
+    }
+else
+    echo "Cluster appears to be initialized (bootstrap keyring exists)"
+fi
+
+echo ""
+echo "=== Step 5: Creating monitor on $HOSTNAME ==="
+# Remove any existing monitor directory
+rm -rf /var/lib/ceph/mon/ceph-$HOSTNAME/*
+
+# Create monitor
+pveceph mon create 2>&1 || {
+    echo "Monitor creation failed or already exists. Checking status..."
+    systemctl status ceph-mon@$HOSTNAME --no-pager | head -10 || true
+}
+
+echo ""
+echo "=== Step 6: Starting monitor service ==="
+systemctl enable ceph-mon@$HOSTNAME
+systemctl start ceph-mon@$HOSTNAME
+sleep 5
+
+echo ""
+echo "=== Step 7: Verifying monitor status ==="
+systemctl status ceph-mon@$HOSTNAME --no-pager | head -20
+
+echo ""
+echo "=== Step 8: Checking quorum status ==="
+sleep 10
+if ceph quorum_status 2>&1 | head -20; then
+    echo "Quorum established!"
+else
+    echo "Quorum not yet established. This is normal if ml110-01 monitor is not running."
+    echo "Monitor is running and will join quorum when both monitors are up."
+fi
+
+echo ""
+echo "=== Step 9: Preparing drives for OSD creation ==="
+echo "Available drives:"
+lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE | grep -E "(sd[cd]|sd[ef]|sd[gh]|NAME)"
+
+echo ""
+echo "=== COMPLETE ==="
+echo "Ceph monitor installed and configured on $FQDN"
+echo ""
+echo "Next steps:"
+echo "1. Verify monitor is running: systemctl status ceph-mon@$HOSTNAME"
+echo "2. Check quorum: ceph quorum_status"
+echo "3. Create OSDs: ceph-volume lvm create --data /dev/<disk>"
+echo "4. Check cluster health: ceph -s"
+

scripts/fresh-ceph-setup.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+# Fresh Ceph setup on Proxmox VE nodes
+set -e
+
+PUBLIC_NETWORK="192.168.11.0/24"
+CLUSTER_NETWORK="192.168.11.0/24"
+HOSTNAME=$(hostname -s)
+
+echo "=== Fresh Ceph Setup on $HOSTNAME ==="
+
+# Check if we're on a Proxmox node
+if [ ! -d "/etc/pve" ]; then
+    echo "ERROR: This script must be run on a Proxmox node"
+    exit 1
+fi
+
+echo ""
+echo "=== Step 1: Installing Ceph packages ==="
+apt-get update
+apt-get install -y ceph ceph-common ceph-base
+
+echo ""
+echo "=== Step 2: Initializing Ceph cluster (only on first node) ==="
+if [ "$HOSTNAME" = "ml110-01" ]; then
+    echo "Initializing Ceph cluster on ml110-01..."
+    if [ ! -f "/etc/pve/ceph.conf" ]; then
+        pveceph init --network $PUBLIC_NETWORK --cluster-network $CLUSTER_NETWORK
+    else
+        echo "Ceph already initialized"
+    fi
+else
+    echo "Skipping init on $HOSTNAME (should be done on ml110-01 first)"
+fi
+
+echo ""
+echo "=== Step 3: Creating monitor ==="
+if [ "$HOSTNAME" = "ml110-01" ]; then
+    echo "Creating monitor on ml110-01..."
+    pveceph mon create 2>&1 || echo "Monitor may already exist"
+elif [ "$HOSTNAME" = "r630-01" ]; then
+    echo "Creating monitor on r630-01..."
+    pveceph mon create 2>&1 || echo "Monitor may already exist"
+fi
+
+echo ""
+echo "=== Step 4: Starting monitor service ==="
+systemctl enable ceph-mon@$HOSTNAME
+systemctl start ceph-mon@$HOSTNAME
+sleep 5
+systemctl status ceph-mon@$HOSTNAME --no-pager | head -15
+
+echo ""
+echo "=== Step 5: Verifying quorum ==="
+sleep 10
+if ceph quorum_status &>/dev/null; then
+    echo "Quorum established!"
+    ceph quorum_status 2>&1 | head -10
+else
+    echo "Quorum not yet established (may take a few minutes)"
+fi
+
+echo ""
+echo "=== COMPLETE ==="
+echo "Next steps:"
+echo "1. Verify quorum: ceph quorum_status"
+echo "2. Create OSDs: ceph-volume lvm create --data /dev/<disk>"
+echo "3. Check cluster health: ceph -s"
+

scripts/import-image-to-rbd.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+# Import cloud image directly to RBD pool using rbd import
+# This is the correct method for copying images to ceph-rbd storage
+
+set -e
+
+SOURCE_IMAGE="/var/lib/vz/template/iso/ubuntu-22.04-cloud.img"
+RBD_POOL="rbd"
+RBD_IMAGE_NAME="template_cache_ubuntu_22_04_cloud"
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  IMPORT IMAGE TO RBD POOL"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+# Check if running on r630-01
+HOSTNAME=$(hostname)
+if [ "$HOSTNAME" != "r630-01" ]; then
+    echo "⚠️  WARNING: This script should be run on r630-01"
+    echo "   Current hostname: $HOSTNAME"
+    exit 1
+fi
+
+echo "Step 1: Checking if source image exists..."
+if [ ! -f "$SOURCE_IMAGE" ]; then
+    echo "❌ Source image not found: $SOURCE_IMAGE"
+    echo "   Available images in /var/lib/vz/template/iso/:"
+    ls -lh /var/lib/vz/template/iso/*.img 2>/dev/null | head -5 || echo "   (no .img files found)"
+    exit 1
+fi
+
+IMAGE_SIZE=$(du -h "$SOURCE_IMAGE" | cut -f1)
+echo "✅ Source image found: $SOURCE_IMAGE ($IMAGE_SIZE)"
+echo ""
+
+echo "Step 2: Checking RBD pool..."
+if ! ceph osd pool ls 2>/dev/null | grep -q "^${RBD_POOL}$"; then
+    echo "❌ RBD pool '$RBD_POOL' does not exist"
+    exit 1
+fi
+echo "✅ RBD pool '$RBD_POOL' exists"
+echo ""
+
+echo "Step 3: Checking if image already exists in RBD pool..."
+if rbd ls -p "$RBD_POOL" 2>/dev/null | grep -q "^${RBD_IMAGE_NAME}$"; then
+    echo "⚠️  Image '$RBD_IMAGE_NAME' already exists in RBD pool"
+    echo "   Removing existing image to overwrite..."
+    rbd rm -p "$RBD_POOL" "$RBD_IMAGE_NAME" 2>/dev/null || true
+    echo "   ✅ Existing image removed"
+fi
+echo ""
+
+echo "Step 4: Importing image to RBD pool..."
+echo "   This may take a few minutes for large images..."
+echo "   Source: $SOURCE_IMAGE"
+echo "   Target: rbd/$RBD_IMAGE_NAME"
+echo ""
+
+if rbd import "$SOURCE_IMAGE" "${RBD_POOL}/${RBD_IMAGE_NAME}" --image-format 2 2>&1; then
+    echo "✅ Image imported successfully to RBD pool"
+else
+    echo "❌ RBD import failed"
+    exit 1
+fi
+
+echo ""
+echo "Step 5: Verifying imported image..."
+if rbd ls -p "$RBD_POOL" 2>/dev/null | grep -q "^${RBD_IMAGE_NAME}$"; then
+    RBD_SIZE=$(rbd info -p "$RBD_POOL" "$RBD_IMAGE_NAME" 2>/dev/null | grep "size" | awk '{print $2, $3}' || echo "unknown")
+    echo "✅ Image verified in RBD pool"
+    echo "   Image: ${RBD_POOL}/${RBD_IMAGE_NAME}"
+    echo "   Size: $RBD_SIZE"
+    echo ""
+    echo "⚠️  Note: Proxmox may need to refresh to see this image."
+    echo "   The image is now available in the RBD pool and can be used by VMs."
+    echo "   The provider will search for images in ceph-rbd storage."
+else
+    echo "⚠️  Image not found in RBD pool after import"
+    exit 1
+fi
+
+echo ""
+echo "✅ Import complete!"
+

scripts/import-provider-image.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Script to import the fixed Crossplane provider image into containerd
+
+set -e
+
+IMAGE_TAR="/tmp/crossplane-provider-proxmox-fixed-v2.tar"
+IMAGE_NAME="crossplane-provider-proxmox:fixed-v2"
+
+echo "=========================================="
+echo "Importing Crossplane Provider Image"
+echo "=========================================="
+echo ""
+
+if [ ! -f "$IMAGE_TAR" ]; then
+    echo "❌ Error: Image tar file not found: $IMAGE_TAR"
+    echo "   Please ensure the image was built and saved."
+    exit 1
+fi
+
+echo "✅ Image tar file found: $IMAGE_TAR"
+echo ""
+
+echo "Importing image to containerd (k8s.io namespace)..."
+if sudo ctr -n k8s.io images import "$IMAGE_TAR"; then
+    echo "✅ Image imported successfully"
+else
+    echo "❌ Failed to import image"
+    exit 1
+fi
+
+echo ""
+echo "Verifying image in containerd..."
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox || echo "⚠️  Image not found with expected name"
+
+echo ""
+echo "=========================================="
+echo "Next Steps:"
+echo "=========================================="
+echo "1. Update deployment:"
+echo "   kubectl set image deployment/crossplane-provider-proxmox provider=$IMAGE_NAME -n crossplane-system"
+echo ""
+echo "2. Restart provider pod:"
+echo "   kubectl delete pod -n crossplane-system -l app=crossplane-provider-proxmox"
+echo ""
+echo "3. Verify new code is running:"
+echo "   kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50"
+echo ""
+

scripts/import-provider-to-containerd.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+# Script to import Crossplane provider image to containerd
+# Must be run with sudo on the Kubernetes node
+
+set -e
+
+IMAGE_TAR="${1:-/tmp/crossplane-provider-proxmox-latest-fresh.tar}"
+
+if [ ! -f "$IMAGE_TAR" ]; then
+    echo "❌ Error: Image tar file not found: $IMAGE_TAR"
+    exit 1
+fi
+
+echo "=========================================="
+echo "Importing Provider Image to Containerd"
+echo "=========================================="
+echo ""
+echo "Image: $IMAGE_TAR"
+echo "Size: $(ls -lh "$IMAGE_TAR" | awk '{print $5}')"
+echo ""
+
+echo "Importing to containerd (k8s.io namespace)..."
+if ctr -n k8s.io images import "$IMAGE_TAR"; then
+    echo "✅ Image imported successfully"
+else
+    echo "❌ Failed to import image"
+    exit 1
+fi
+
+echo ""
+echo "Verifying import..."
+ctr -n k8s.io images ls | grep crossplane-provider-proxmox || echo "⚠️  Image not found with expected name"
+
+echo ""
+echo "=========================================="
+echo "Import Complete!"
+echo "=========================================="
+echo ""
+echo "Next: Restart the provider pod:"
+echo "  kubectl delete pod -n crossplane-system -l app=crossplane-provider-proxmox"

scripts/install-r630-packages.sh

@@ -0,0 +1,279 @@
+#!/bin/bash
+# Install required packages on R630-01 for Ceph and system management
+# Run on R630-01 as root
+
+set -e
+
+# Set non-interactive mode to avoid prompts
+export DEBIAN_FRONTEND=noninteractive
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+echo "=========================================="
+echo -e "${CYAN}R630-01 Package Installation${NC}"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+# Update package list
+echo -e "${BLUE}=== Step 1: Update Package Lists ===${NC}"
+echo "-----------------------------------"
+apt-get update
+echo -e "${GREEN}✓ Package lists updated${NC}"
+echo ""
+
+# Essential utilities
+echo -e "${BLUE}=== Step 2: Install Essential Utilities ===${NC}"
+echo "-----------------------------------"
+ESSENTIAL_PACKAGES=(
+    "curl"
+    "wget"
+    "git"
+    "jq"
+    "vim"
+    "nano"
+    "htop"
+    "tree"
+    "unzip"
+    "zip"
+    "rsync"
+    "screen"
+    "tmux"
+)
+
+for pkg in "${ESSENTIAL_PACKAGES[@]}"; do
+    if dpkg -l | grep -q "^ii  $pkg "; then
+        echo -e "  ${GREEN}✓${NC} $pkg (already installed)"
+    else
+        echo "  Installing $pkg..."
+        apt-get install -y "$pkg" && echo -e "  ${GREEN}✓${NC} $pkg installed"
+    fi
+done
+echo ""
+
+# Disk and storage management
+echo -e "${BLUE}=== Step 3: Install Disk & Storage Tools ===${NC}"
+echo "-----------------------------------"
+STORAGE_PACKAGES=(
+    "lvm2"           # LVM management
+    "parted"          # Disk partitioning
+    "gdisk"           # GPT partition table tools
+    "fdisk"           # Partition table manipulator
+    "util-linux"      # lsblk and other utilities
+    "smartmontools"   # SMART disk monitoring
+)
+
+for pkg in "${STORAGE_PACKAGES[@]}"; do
+    if dpkg -l | grep -q "^ii  $pkg "; then
+        echo -e "  ${GREEN}✓${NC} $pkg (already installed)"
+    else
+        echo "  Installing $pkg..."
+        apt-get install -y "$pkg" && echo -e "  ${GREEN}✓${NC} $pkg installed"
+    fi
+done
+echo ""
+
+# Time synchronization
+echo -e "${BLUE}=== Step 4: Install Time Synchronization ===${NC}"
+echo "-----------------------------------"
+if dpkg -l | grep -q "^ii  chrony "; then
+    echo -e "${GREEN}✓ chrony (already installed)${NC}"
+else
+    echo "Installing chrony..."
+    apt-get install -y chrony
+    systemctl enable chronyd || systemctl enable chrony
+    systemctl start chronyd || systemctl start chrony
+    echo -e "${GREEN}✓ chrony installed and started${NC}"
+fi
+echo ""
+
+# Python and development tools
+echo -e "${BLUE}=== Step 5: Install Python & Development Tools ===${NC}"
+echo "-----------------------------------"
+PYTHON_PACKAGES=(
+    "python3"
+    "python3-pip"
+    "python3-venv"
+    "build-essential"
+)
+
+for pkg in "${PYTHON_PACKAGES[@]}"; do
+    if dpkg -l | grep -q "^ii  $pkg "; then
+        echo -e "  ${GREEN}✓${NC} $pkg (already installed)"
+    else
+        echo "  Installing $pkg..."
+        apt-get install -y "$pkg" && echo -e "  ${GREEN}✓${NC} $pkg installed"
+    fi
+done
+echo ""
+
+# Ceph packages (if Ceph repository is configured)
+echo -e "${BLUE}=== Step 6: Install Ceph Packages ===${NC}"
+echo "-----------------------------------"
+
+CEPH_VERSION="${CEPH_VERSION:-quincy}"
+CEPH_REPO_CONFIGURED=false
+
+# Check if Ceph repository is configured
+if [ -f /etc/apt/sources.list.d/ceph.list ] || [ -f /etc/apt/sources.list.d/ceph-no-sub.list ]; then
+    CEPH_REPO_CONFIGURED=true
+    echo -e "${GREEN}✓ Ceph repository already configured${NC}"
+else
+    echo "Ceph repository not found. Setting up..."
+    
+    # Create keyrings directory
+    mkdir -p /etc/apt/keyrings
+    
+    # Download Ceph release key
+    if wget -q -O /etc/apt/keyrings/ceph-release.asc 'https://download.ceph.com/keys/release.asc'; then
+        # Detect Debian/Proxmox version
+        if [ -f /etc/debian_version ]; then
+            DEBIAN_VERSION=$(cat /etc/debian_version | cut -d. -f1)
+            if [ "$DEBIAN_VERSION" = "11" ]; then
+                CODENAME="bullseye"
+            elif [ "$DEBIAN_VERSION" = "12" ]; then
+                CODENAME="bookworm"
+            else
+                CODENAME="bullseye"  # Default
+            fi
+        else
+            CODENAME="bullseye"  # Default for Proxmox
+        fi
+        
+        # Add Ceph repository
+        echo "deb [signed-by=/etc/apt/keyrings/ceph-release.asc] https://download.ceph.com/debian-${CEPH_VERSION}/ ${CODENAME} main" > /etc/apt/sources.list.d/ceph.list
+        
+        # Try Proxmox Ceph repo as fallback
+        if [ -f /etc/os-release ] && grep -q "Proxmox" /etc/os-release; then
+            echo "deb http://download.proxmox.com/debian/ceph-${CEPH_VERSION} ${CODENAME} no-subscription" >> /etc/apt/sources.list.d/ceph-no-sub.list
+        fi
+        
+        apt-get update || apt-get update --allow-releaseinfo-change || true
+        CEPH_REPO_CONFIGURED=true
+        echo -e "${GREEN}✓ Ceph repository configured${NC}"
+    else
+        echo -e "${YELLOW}⚠ Could not download Ceph repository key${NC}"
+        echo "  You may need to configure Ceph repository manually"
+    fi
+fi
+
+# Install Ceph packages if repository is configured
+if [ "$CEPH_REPO_CONFIGURED" = true ]; then
+    CEPH_PACKAGES=(
+        "ceph"
+        "ceph-common"
+        "ceph-base"
+        "ceph-volume"
+        "ceph-mds"
+    )
+    
+    for pkg in "${CEPH_PACKAGES[@]}"; do
+        if dpkg -l | grep -q "^ii  $pkg "; then
+            echo -e "  ${GREEN}✓${NC} $pkg (already installed)"
+        else
+            echo "  Installing $pkg..."
+            if apt-get install -y "$pkg" 2>&1 | grep -v "WARNING: apt does not have a stable CLI interface"; then
+                echo -e "  ${GREEN}✓${NC} $pkg installed"
+            else
+                echo -e "  ${YELLOW}⚠${NC} $pkg installation had issues (may already be installed)"
+            fi
+        fi
+    done
+else
+    echo -e "${YELLOW}⚠ Skipping Ceph package installation (repository not configured)${NC}"
+    echo "  To install Ceph later, configure the repository and run:"
+    echo "  apt-get install -y ceph ceph-common ceph-base ceph-volume"
+fi
+echo ""
+
+# Network tools
+echo -e "${BLUE}=== Step 7: Install Network Tools ===${NC}"
+echo "-----------------------------------"
+NETWORK_PACKAGES=(
+    "net-tools"       # ifconfig, netstat
+    "iproute2"        # ip command
+    "tcpdump"         # Network packet analyzer
+    "nmap"            # Network scanner
+)
+
+for pkg in "${NETWORK_PACKAGES[@]}"; do
+    if dpkg -l | grep -q "^ii  $pkg "; then
+        echo -e "  ${GREEN}✓${NC} $pkg (already installed)"
+    else
+        echo "  Installing $pkg..."
+        apt-get install -y "$pkg" && echo -e "  ${GREEN}✓${NC} $pkg installed"
+    fi
+done
+
+# iperf3 with pre-configured answer (don't start as daemon)
+if dpkg -l | grep -q "^ii  iperf3 "; then
+    echo -e "  ${GREEN}✓${NC} iperf3 (already installed)"
+else
+    echo "  Installing iperf3..."
+    echo "iperf3 iperf3/start_daemon boolean false" | debconf-set-selections
+    apt-get install -y iperf3 && echo -e "  ${GREEN}✓${NC} iperf3 installed"
+fi
+echo ""
+
+# Monitoring and system info
+echo -e "${BLUE}=== Step 8: Install Monitoring Tools ===${NC}"
+echo "-----------------------------------"
+MONITORING_PACKAGES=(
+    "iotop"           # I/O monitoring
+    "nethogs"         # Network usage by process
+    "iftop"           # Network bandwidth monitor
+)
+
+for pkg in "${MONITORING_PACKAGES[@]}"; do
+    if dpkg -l | grep -q "^ii  $pkg "; then
+        echo -e "  ${GREEN}✓${NC} $pkg (already installed)"
+    else
+        echo "  Installing $pkg..."
+        apt-get install -y "$pkg" && echo -e "  ${GREEN}✓${NC} $pkg installed"
+    fi
+done
+echo ""
+
+# Cleanup
+echo -e "${BLUE}=== Step 9: Cleanup ===${NC}"
+echo "-----------------------------------"
+apt-get autoremove -y
+apt-get autoclean -y
+echo -e "${GREEN}✓ Cleanup complete${NC}"
+echo ""
+
+# Summary
+echo "=========================================="
+echo -e "${GREEN}Installation Complete!${NC}"
+echo "=========================================="
+echo ""
+echo "Installed packages:"
+echo "  ✓ Essential utilities (curl, wget, git, jq, etc.)"
+echo "  ✓ Disk & storage tools (lvm2, parted, gdisk, etc.)"
+echo "  ✓ Time synchronization (chrony)"
+echo "  ✓ Python & development tools"
+if [ "$CEPH_REPO_CONFIGURED" = true ]; then
+    echo "  ✓ Ceph packages (ceph, ceph-common, ceph-volume, etc.)"
+else
+    echo "  ⚠ Ceph packages (repository needs configuration)"
+fi
+echo "  ✓ Network tools"
+echo "  ✓ Monitoring tools"
+echo ""
+echo "Next steps:"
+echo "1. Verify Ceph installation: ceph --version"
+echo "2. Check disk layout: ./scripts/analyze-r630-disk-layout.sh"
+echo "3. Setup Ceph OSDs: ./scripts/setup-ceph-osds-r630.sh"
+echo ""
+

scripts/kill-and-fix.sh

@@ -0,0 +1,141 @@
+#!/bin/bash
+set -e
+
+# Find kubectl in common locations
+KUBECTL=$(which kubectl 2>/dev/null || echo "")
+if [ -z "$KUBECTL" ]; then
+    # Try common locations
+    for path in /usr/local/bin/kubectl /usr/bin/kubectl ~/.local/bin/kubectl; do
+        if [ -x "$path" ]; then
+            KUBECTL="$path"
+            break
+        fi
+    done
+fi
+
+if [ -z "$KUBECTL" ]; then
+    echo "ERROR: kubectl not found. Please ensure kubectl is in PATH or update this script."
+    exit 1
+fi
+
+echo "=========================================="
+echo "KILL AND FIX - Aggressive Cleanup"
+echo "=========================================="
+echo "Using kubectl: $KUBECTL"
+echo ""
+
+# Step 1: Force kill pod
+echo "Step 1: Force killing pod..."
+$KUBECTL delete pod -n crossplane-system -l app=crossplane-provider-proxmox --force --grace-period=0 2>&1 || echo "  (Pod not found)"
+sleep 2
+
+# Step 2: Force delete deployment
+echo ""
+echo "Step 2: Force deleting deployment..."
+$KUBECTL delete deployment crossplane-provider-proxmox -n crossplane-system --force --grace-period=0 2>&1 || echo "  (Deployment not found)"
+sleep 2
+
+# Step 3: Remove finalizers if stuck
+echo ""
+echo "Step 3: Removing finalizers..."
+$KUBECTL patch deployment crossplane-provider-proxmox -n crossplane-system -p '{"metadata":{"finalizers":[]}}' --type=merge 2>&1 || echo "  (No finalizers to remove)"
+sleep 2
+
+# Step 4: Verify cleanup
+echo ""
+echo "Step 4: Verifying cleanup..."
+REMAINING=$($KUBECTL get all -n crossplane-system -l app=crossplane-provider-proxmox 2>&1 | grep -v "No resources" || true)
+if [ -n "$REMAINING" ]; then
+    echo "  ⚠️  Some resources still exist:"
+    echo "$REMAINING"
+else
+    echo "  ✅ All resources cleaned up"
+fi
+
+# Step 5: Remove ALL crossplane-provider-proxmox images from containerd
+echo ""
+echo "Step 5: Removing ALL crossplane-provider-proxmox images from containerd..."
+ALL_IMAGES=$(sudo ctr -n k8s.io images ls -q | grep crossplane-provider-proxmox || true)
+if [ -n "$ALL_IMAGES" ]; then
+    echo "$ALL_IMAGES" | while read -r img; do
+        echo "  Removing: $img"
+        sudo ctr -n k8s.io images rm "$img" 2>/dev/null || true
+    done
+    echo "  ✅ All images removed"
+else
+    echo "  (No images found)"
+fi
+
+# Step 6: Verify image tar exists
+echo ""
+echo "Step 6: Verifying image tar exists..."
+if [ ! -f /tmp/crossplane-fresh-nuclear.tar ]; then
+    echo "  ❌ ERROR: Image tar not found at /tmp/crossplane-fresh-nuclear.tar"
+    echo "  Please rebuild the image first using:"
+    echo "    sudo /home/intlc/projects/Sankofa/scripts/nuclear-cleanup-rebuild.sh"
+    exit 1
+fi
+echo "  ✅ Image tar found"
+
+# Step 7: Re-import fresh image
+echo ""
+echo "Step 7: Re-importing fresh image..."
+sudo ctr -n k8s.io images import /tmp/crossplane-fresh-nuclear.tar
+echo "  ✅ Image imported"
+
+# Step 8: Verify import
+echo ""
+echo "Step 8: Verifying import..."
+IMPORTED=$(sudo ctr -n k8s.io images ls -q | grep "crossplane-provider-proxmox:latest" || true)
+if [ -z "$IMPORTED" ]; then
+    echo "  ❌ ERROR: Image not found after import!"
+    exit 1
+fi
+echo "  ✅ Image verified: $IMPORTED"
+
+# Step 9: Show image details
+echo ""
+echo "Step 9: Image details:"
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox
+
+# Step 10: Recreate deployment
+echo ""
+echo "Step 10: Recreating deployment..."
+$KUBECTL apply -f /home/intlc/projects/Sankofa/crossplane-provider-proxmox/config/provider.yaml
+
+# Step 11: Wait for pod
+echo ""
+echo "Step 11: Waiting for pod to be ready..."
+$KUBECTL wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s || {
+    echo "  ⚠️  Pod not ready within timeout, checking status..."
+    $KUBECTL get pod -n crossplane-system -l app=crossplane-provider-proxmox
+    exit 1
+}
+echo "  ✅ Pod is ready"
+
+# Step 12: Check image ID
+echo ""
+echo "Step 12: Checking pod image ID..."
+POD_IMAGE_ID=$($KUBECTL get pod -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].imageID}')
+echo "  Pod image ID: $POD_IMAGE_ID"
+
+# Step 13: Wait for logs
+echo ""
+echo "Step 13: Waiting for logs to generate..."
+sleep 5
+
+# Step 14: Check for [FIXED CODE] message
+echo ""
+echo "Step 14: Checking for [FIXED CODE] message in logs..."
+if $KUBECTL logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 | grep -q '\[FIXED CODE\]'; then
+    echo "  ✅ [FIXED CODE] message found! Fix is active."
+else
+    echo "  ⚠️  [FIXED CODE] message NOT found. Checking logs..."
+    $KUBECTL logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=20 | grep -E "Cookie|Authorization|FIXED" || echo "  (No relevant log lines found)"
+fi
+
+echo ""
+echo "=========================================="
+echo "KILL AND FIX COMPLETE"
+echo "=========================================="
+

scripts/monitor-vm-deployment.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# VM Deployment Monitoring Script
+# Continuously monitors VM deployment progress
+
+echo "=========================================="
+echo "📊 VM DEPLOYMENT MONITORING"
+echo "=========================================="
+echo ""
+echo "Press Ctrl+C to stop"
+echo ""
+
+while true; do
+    clear
+    echo "=========================================="
+    echo "📊 VM DEPLOYMENT STATUS - $(date '+%Y-%m-%d %H:%M:%S')"
+    echo "=========================================="
+    echo ""
+    
+    # Authentication Status
+    echo "1. AUTHENTICATION STATUS"
+    echo "----------------------------------------"
+    AUTH_ERR=$(kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --since=2m 2>&1 | grep -i "invalid PVE ticket" | wc -l)
+    echo "   Errors (last 2m): $AUTH_ERR"
+    if [ $AUTH_ERR -eq 0 ]; then
+        echo "   Status: ✅ Working"
+    else
+        echo "   Status: ⚠️  Issues detected"
+    fi
+    echo ""
+    
+    # Provider Status
+    echo "2. PROVIDER STATUS"
+    echo "----------------------------------------"
+    PROVIDER_STATUS=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.phase}' 2>/dev/null)
+    PROVIDER_READY=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].ready}' 2>/dev/null)
+    echo "   Status: $PROVIDER_STATUS"
+    echo "   Ready: $PROVIDER_READY"
+    echo ""
+    
+    # VM Creation Status
+    echo "3. VM CREATION STATUS"
+    echo "----------------------------------------"
+    TOTAL=$(kubectl get proxmoxvm -A --no-headers 2>&1 | wc -l)
+    WITH_VMID=$(kubectl get proxmoxvm -A -o jsonpath='{range .items[*]}{.status.vmId}{"\n"}{end}' 2>&1 | grep -v '^$' | wc -l)
+    PENDING=$((TOTAL - WITH_VMID))
+    if [ $TOTAL -gt 0 ]; then
+        PERCENT=$((WITH_VMID * 100 / TOTAL))
+    else
+        PERCENT=0
+    fi
+    echo "   Total VMs: $TOTAL"
+    echo "   Created (with VMID): $WITH_VMID"
+    echo "   Pending: $PENDING"
+    echo "   Progress: $PERCENT%"
+    echo ""
+    
+    # Recent VMs Created
+    if [ $WITH_VMID -gt 0 ]; then
+        echo "   Recently Created VMs:"
+        kubectl get proxmoxvm -A -o custom-columns=NAME:.metadata.name,NODE:.spec.forProvider.node,VMID:.status.vmId 2>&1 | grep -v "VMID.*<none>" | head -10
+        echo ""
+    fi
+    
+    # Node Health Checks
+    echo "4. NODE HEALTH CHECKS"
+    echo "----------------------------------------"
+    HEALTH_CHECKS=$(kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --since=2m 2>&1 | grep -E "node.*healthy|node.*online" -i | wc -l)
+    HEALTH_FAILS=$(kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --since=2m 2>&1 | grep -i "node health check failed" | wc -l)
+    echo "   Successful: $HEALTH_CHECKS"
+    echo "   Failed: $HEALTH_FAILS"
+    echo ""
+    
+    # Recent Activity
+    echo "5. RECENT ACTIVITY"
+    echo "----------------------------------------"
+    kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --since=1m 2>&1 | grep -E "Creating VM|VM.*created|VMID|Reconciling" -i | tail -5 || echo "   No recent activity"
+    echo ""
+    
+    # Resource Allocation
+    echo "6. RESOURCE ALLOCATION"
+    echo "----------------------------------------"
+    ML_CPU=$(kubectl get proxmoxvm -A -o jsonpath='{range .items[?(@.spec.forProvider.node=="ml110-01")]}{.spec.forProvider.cpu}{"\n"}{end}' 2>&1 | awk '{sum+=$1} END {print sum}')
+    R_CPU=$(kubectl get proxmoxvm -A -o jsonpath='{range .items[?(@.spec.forProvider.node=="r630-01")]}{.spec.forProvider.cpu}{"\n"}{end}' 2>&1 | awk '{sum+=$1} END {print sum}')
+    echo "   ML110-01: $ML_CPU CPU (5-6 available)"
+    echo "   R630-01: $R_CPU CPU (52 available)"
+    echo ""
+    
+    echo "=========================================="
+    echo "Refreshing in 10 seconds... (Ctrl+C to stop)"
+    echo "=========================================="
+    
+    sleep 10
+done

scripts/nuclear-cleanup-rebuild.sh

@@ -0,0 +1,124 @@
+#!/bin/bash
+set -euo pipefail
+
+log() {
+    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" >&2
+}
+
+error() {
+    log "ERROR: $*"
+    exit 1
+}
+
+IMAGE_NAME="docker.io/library/crossplane-provider-proxmox"
+PROJECT_DIR="/home/intlc/projects/Sankofa/crossplane-provider-proxmox"
+
+log "=========================================="
+log "NUCLEAR CLEANUP AND REBUILD"
+log "=========================================="
+log ""
+
+# Step 1: Remove ALL crossplane-provider-proxmox images from containerd
+log "Step 1: Removing ALL crossplane-provider-proxmox images from containerd..."
+ALL_IMAGES=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}" || true)
+if [ -n "$ALL_IMAGES" ]; then
+    echo "$ALL_IMAGES" | while read -r img; do
+        log "  Removing: $img"
+        sudo ctr -n k8s.io images rm "$img" 2>/dev/null || true
+    done
+    log "  ✅ All images removed"
+else
+    log "  No images found (already clean)"
+fi
+
+# Step 2: Prune containerd images (remove unused)
+log ""
+log "Step 2: Pruning unused containerd images..."
+sudo ctr -n k8s.io images prune 2>&1 | head -10 || log "  (Prune completed)"
+
+# Step 3: Check Docker daemon and determine access method
+log ""
+log "Step 3: Checking Docker daemon..."
+# If running as root (via sudo), use original user for Docker commands
+# since Docker is running in rootless mode with user-specific socket
+DOCKER_CMD="docker"
+if [ "$EUID" -eq 0 ]; then
+    ORIGINAL_USER="${SUDO_USER:-${USER}}"
+    if [ -n "$ORIGINAL_USER" ] && [ "$ORIGINAL_USER" != "root" ]; then
+        log "  Running Docker commands as user: $ORIGINAL_USER (rootless Docker)"
+        # Preserve DOCKER_HOST and other Docker env vars
+        DOCKER_HOST_VAL="${DOCKER_HOST:-unix:///run/user/$(id -u $ORIGINAL_USER)/docker.sock}"
+        DOCKER_CMD="sudo -u $ORIGINAL_USER env DOCKER_HOST=$DOCKER_HOST_VAL docker"
+    fi
+else
+    # Not root - preserve current DOCKER_HOST if set
+    if [ -n "${DOCKER_HOST:-}" ]; then
+        DOCKER_CMD="env DOCKER_HOST=$DOCKER_HOST docker"
+    fi
+fi
+
+# Verify Docker access
+if ! $DOCKER_CMD ps >/dev/null 2>&1; then
+    error "Docker daemon is not accessible! Please ensure Docker is running. Try: docker ps"
+fi
+log "  ✅ Docker daemon is accessible (using: $DOCKER_CMD)"
+
+# Step 4: Remove Docker images locally
+log ""
+log "Step 4: Removing local Docker images..."
+$DOCKER_CMD rmi crossplane-provider-proxmox:latest crossplane-provider-proxmox:final-fix 2>/dev/null || log "  (Images not found in Docker, continuing...)"
+
+# Step 5: Clean Docker build cache
+log ""
+log "Step 5: Cleaning Docker build cache..."
+$DOCKER_CMD builder prune -af --filter "until=24h" 2>&1 | tail -5 || log "  (Cache clean completed)"
+
+# Step 6: Rebuild image from scratch (no cache)
+log ""
+log "Step 6: Rebuilding image from scratch (NO CACHE)..."
+cd "$PROJECT_DIR" || error "Cannot cd to $PROJECT_DIR"
+$DOCKER_CMD build --no-cache --pull -t crossplane-provider-proxmox:latest . 2>&1 | tail -20
+
+# Step 7: Save the fresh image
+log ""
+log "Step 7: Saving fresh image..."
+$DOCKER_CMD save crossplane-provider-proxmox:latest -o /tmp/crossplane-fresh-nuclear.tar
+log "  ✅ Image saved to /tmp/crossplane-fresh-nuclear.tar"
+ls -lh /tmp/crossplane-fresh-nuclear.tar
+
+# Step 8: Import to containerd
+log ""
+log "Step 8: Importing fresh image to containerd..."
+sudo ctr -n k8s.io images import /tmp/crossplane-fresh-nuclear.tar
+log "  ✅ Image imported"
+
+# Step 9: Verify import
+log ""
+log "Step 9: Verifying import..."
+IMPORTED=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}:latest" || true)
+if [ -z "$IMPORTED" ]; then
+    error "Image ${IMAGE_NAME}:latest not found after import!"
+fi
+log "  ✅ Image verified: $IMPORTED"
+
+# Step 10: Show final state
+log ""
+log "Step 10: Final image state:"
+sudo ctr -n k8s.io images ls | grep "${IMAGE_NAME}" || error "No crossplane images found!"
+
+log ""
+log "=========================================="
+log "CLEANUP AND REBUILD COMPLETE"
+log "=========================================="
+log ""
+log "Next steps:"
+log "  1. Apply the deployment:"
+log "     kubectl apply -f $PROJECT_DIR/config/provider.yaml"
+log ""
+log "  2. Wait for pod to be ready:"
+log "     kubectl wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s"
+log ""
+log "  3. Verify [FIXED CODE] appears in logs:"
+log "     kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox | grep '\[FIXED CODE\]'"
+
+

scripts/nuclear-image-fix.sh

@@ -0,0 +1,86 @@
+#!/bin/bash
+set -euo pipefail
+
+log() {
+    echo "[$(date +'%Y-%m-%d %H:%M:%S')] $*" >&2
+}
+
+error() {
+    log "ERROR: $*"
+    exit 1
+}
+
+IMAGE_NAME="docker.io/library/crossplane-provider-proxmox"
+IMAGE_TAR="/tmp/crossplane-cookie-fix.tar"
+
+log "=========================================="
+log "NUCLEAR IMAGE CLEANUP AND RE-IMPORT"
+log "=========================================="
+log ""
+
+# 1. Remove ALL crossplane images by all possible references
+log "Step 1: Removing ALL crossplane-provider-proxmox images..."
+sudo ctr -n k8s.io images rm "${IMAGE_NAME}:latest" 2>/dev/null || log "  (latest not found, continuing...)"
+sudo ctr -n k8s.io images rm "${IMAGE_NAME}:v1.0.0-fixed" 2>/dev/null || log "  (v1.0.0-fixed not found, continuing...)"
+sudo ctr -n k8s.io images rm "sha256:5d83df9f8d127aa8988ace01254f4adbb6336f0ad59be8193349b3eacd972cfc" 2>/dev/null || log "  (old image ID not found, continuing...)"
+
+# 2. List and remove any remaining by name pattern
+log ""
+log "Step 2: Finding and removing any remaining images..."
+REMAINING=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}" || true)
+if [ -n "$REMAINING" ]; then
+    log "  Found remaining images:"
+    echo "$REMAINING" | while read -r img; do
+        log "    Removing: $img"
+        sudo ctr -n k8s.io images rm "$img" 2>/dev/null || true
+    done
+else
+    log "  No remaining images found (good!)"
+fi
+
+# 3. Verify cleanup
+log ""
+log "Step 3: Verifying cleanup..."
+REMAINING_AFTER=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}" || true)
+if [ -n "$REMAINING_AFTER" ]; then
+    error "Cleanup incomplete! Remaining images: $REMAINING_AFTER"
+fi
+log "  ✅ All old images removed"
+
+# 4. Import the fresh image
+log ""
+log "Step 4: Importing fresh image..."
+if [ ! -f "${IMAGE_TAR}" ]; then
+    error "Image tar file not found at ${IMAGE_TAR}"
+fi
+sudo ctr -n k8s.io images import "${IMAGE_TAR}"
+log "  ✅ Image imported"
+
+# 5. Verify import
+log ""
+log "Step 5: Verifying import..."
+IMPORTED=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}:latest" || true)
+if [ -z "$IMPORTED" ]; then
+    error "Image ${IMAGE_NAME}:latest not found after import!"
+fi
+log "  ✅ Image verified: $IMPORTED"
+
+# 6. Show final state
+log ""
+log "Step 6: Final image state:"
+sudo ctr -n k8s.io images ls | grep "${IMAGE_NAME}" || error "No crossplane images found!"
+
+log ""
+log "=========================================="
+log "CLEANUP COMPLETE"
+log "=========================================="
+log ""
+log "Next steps:"
+log "  1. Delete the deployment to force recreation:"
+log "     kubectl delete deployment crossplane-provider-proxmox -n crossplane-system"
+log ""
+log "  2. Reapply the provider:"
+log "     kubectl apply -f crossplane-provider-proxmox/config/provider.yaml"
+log ""
+log "  3. Wait for pod to be ready and verify fixes are active"
+

scripts/pre-deployment-verification.sh

@@ -0,0 +1,146 @@
+#!/bin/bash
+# Pre-Deployment Verification Script
+# Verifies all pre-deployment requirements are met
+
+set -e
+
+GREEN='\033[0;32m'
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+ERRORS=0
+WARNINGS=0
+
+echo "=========================================="
+echo "Pre-Deployment Verification"
+echo "=========================================="
+echo ""
+
+# 1. Check namespace
+echo "1. Checking crossplane-system namespace..."
+if kubectl get namespace crossplane-system &>/dev/null; then
+    echo -e "${GREEN}✓${NC} Namespace exists"
+else
+    echo -e "${RED}✗${NC} Namespace missing"
+    ((ERRORS++))
+fi
+
+# 2. Check provider pod
+echo ""
+echo "2. Checking provider pod..."
+POD_STATUS=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.phase}' 2>/dev/null || echo "NOT_FOUND")
+if [[ "$POD_STATUS" == "Running" ]]; then
+    echo -e "${GREEN}✓${NC} Provider pod is running"
+    READY=$(kubectl get pods -n crossplane-system -l app=crossplane-provider-proxmox -o jsonpath='{.items[0].status.containerStatuses[0].ready}' 2>/dev/null)
+    if [[ "$READY" == "true" ]]; then
+        echo -e "${GREEN}✓${NC} Provider pod is ready"
+    else
+        echo -e "${YELLOW}⚠${NC} Provider pod not ready yet"
+        ((WARNINGS++))
+    fi
+else
+    echo -e "${RED}✗${NC} Provider pod not running (status: $POD_STATUS)"
+    ((ERRORS++))
+fi
+
+# 3. Check ProviderConfig
+echo ""
+echo "3. Checking ProviderConfig..."
+if kubectl get providerconfig proxmox-provider-config -n crossplane-system &>/dev/null; then
+    echo -e "${GREEN}✓${NC} ProviderConfig exists"
+    
+    # Check sites
+    SITES=$(kubectl get providerconfig proxmox-provider-config -n crossplane-system -o jsonpath='{.spec.sites[*].name}' 2>/dev/null)
+    if [[ "$SITES" == *"site-1"* && "$SITES" == *"site-2"* ]]; then
+        echo -e "${GREEN}✓${NC} Both sites configured (site-1, site-2)"
+    else
+        echo -e "${RED}✗${NC} Sites not correctly configured: $SITES"
+        ((ERRORS++))
+    fi
+else
+    echo -e "${RED}✗${NC} ProviderConfig not found"
+    ((ERRORS++))
+fi
+
+# 4. Check secret
+echo ""
+echo "4. Checking credentials secret..."
+if kubectl get secret proxmox-credentials -n crossplane-system &>/dev/null; then
+    echo -e "${GREEN}✓${NC} Secret exists"
+    
+    # Check secret keys
+    KEYS=$(kubectl get secret proxmox-credentials -n crossplane-system -o jsonpath='{.data}' 2>/dev/null | jq -r 'keys[]' 2>/dev/null || echo "")
+    if [[ "$KEYS" == *"tokenid"* && "$KEYS" == *"token"* ]] || [[ "$KEYS" == *"username"* && "$KEYS" == *"password"* ]]; then
+        echo -e "${GREEN}✓${NC} Secret has correct format"
+    else
+        echo -e "${YELLOW}⚠${NC} Secret format may be incorrect (keys: $KEYS)"
+        ((WARNINGS++))
+    fi
+else
+    echo -e "${RED}✗${NC} Secret not found"
+    ((ERRORS++))
+fi
+
+# 5. Check CRDs
+echo ""
+echo "5. Checking CRDs..."
+REQUIRED_CRDS=("proxmoxvms.proxmox.sankofa.nexus" "providerconfigs.proxmox.sankofa.nexus")
+for CRD in "${REQUIRED_CRDS[@]}"; do
+    if kubectl get crd "$CRD" &>/dev/null; then
+        echo -e "${GREEN}✓${NC} $CRD installed"
+    else
+        echo -e "${RED}✗${NC} $CRD not installed"
+        ((ERRORS++))
+    fi
+done
+
+# 6. Check provider logs for errors
+echo ""
+echo "6. Checking provider logs..."
+LOG_ERRORS=$(kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox --tail=50 2>&1 | grep -i "error" | grep -v "CRD" | wc -l)
+if [[ $LOG_ERRORS -eq 0 ]]; then
+    echo -e "${GREEN}✓${NC} No critical errors in logs"
+else
+    echo -e "${YELLOW}⚠${NC} Found $LOG_ERRORS error messages in logs (may be non-critical)"
+    ((WARNINGS++))
+fi
+
+# 7. Verify site endpoints
+echo ""
+echo "7. Verifying site endpoints..."
+SITE1_ENDPOINT=$(kubectl get providerconfig proxmox-provider-config -n crossplane-system -o jsonpath='{.spec.sites[?(@.name=="site-1")].endpoint}' 2>/dev/null)
+SITE2_ENDPOINT=$(kubectl get providerconfig proxmox-provider-config -n crossplane-system -o jsonpath='{.spec.sites[?(@.name=="site-2")].endpoint}' 2>/dev/null)
+
+if [[ "$SITE1_ENDPOINT" == "https://192.168.11.10:8006" ]]; then
+    echo -e "${GREEN}✓${NC} Site-1 endpoint correct: $SITE1_ENDPOINT"
+else
+    echo -e "${RED}✗${NC} Site-1 endpoint incorrect: $SITE1_ENDPOINT"
+    ((ERRORS++))
+fi
+
+if [[ "$SITE2_ENDPOINT" == "https://192.168.11.11:8006" ]]; then
+    echo -e "${GREEN}✓${NC} Site-2 endpoint correct: $SITE2_ENDPOINT"
+else
+    echo -e "${RED}✗${NC} Site-2 endpoint incorrect: $SITE2_ENDPOINT"
+    ((ERRORS++))
+fi
+
+echo ""
+echo "=========================================="
+echo "Verification Complete"
+echo "=========================================="
+echo "Errors: $ERRORS"
+echo "Warnings: $WARNINGS"
+
+if [[ $ERRORS -eq 0 && $WARNINGS -eq 0 ]]; then
+    echo -e "${GREEN}✓ All checks passed! Ready for deployment.${NC}"
+    exit 0
+elif [[ $ERRORS -eq 0 ]]; then
+    echo -e "${YELLOW}⚠ All critical checks passed with minor warnings.${NC}"
+    exit 0
+else
+    echo -e "${RED}✗ Critical errors found. Please fix before deployment.${NC}"
+    exit 1
+fi
+

scripts/prepare-ceph-osd-from-ubuntu-vg.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+# Script to free a drive from ubuntu-vg and create Ceph OSD
+# Run on R630-01 as root
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+echo "=========================================="
+echo "Prepare Drive from ubuntu-vg for Ceph OSD"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+echo -e "${BLUE}=== Step 1: Current ubuntu-vg Status ===${NC}"
+echo "-----------------------------------"
+if vgs ubuntu-vg &>/dev/null; then
+    echo "Volume Group Information:"
+    vgs ubuntu-vg
+    echo ""
+    echo "Logical Volumes:"
+    lvs ubuntu-vg
+    echo ""
+    echo "Physical Volumes:"
+    pvs | grep ubuntu-vg
+    echo ""
+else
+    echo -e "${YELLOW}ubuntu-vg not found${NC}"
+    echo ""
+fi
+
+echo -e "${BLUE}=== Step 2: Available 250GB Drives ===${NC}"
+echo "-----------------------------------"
+echo "250GB drives detected:"
+drives=()
+for disk in sdc sdd sde sdf sdg sdh; do
+    if [ -b "/dev/$disk" ]; then
+        size=$(fdisk -l "/dev/$disk" 2>/dev/null | grep "^Disk /dev/$disk" | awk '{print $3, $4}')
+        in_vg=$(pvs 2>/dev/null | grep "/dev/${disk}3" | grep ubuntu-vg | wc -l)
+        if [ "$in_vg" -gt 0 ]; then
+            echo -e "  ${YELLOW}/dev/$disk: ${size} - In ubuntu-vg${NC}"
+            drives+=("$disk")
+        else
+            echo -e "  ${GREEN}/dev/$disk: ${size} - Available${NC}"
+            drives+=("$disk")
+        fi
+    fi
+done
+echo ""
+
+echo -e "${BLUE}=== Step 3: Current Ceph OSD Status ===${NC}"
+echo "-----------------------------------"
+ceph osd tree 2>/dev/null || echo -e "${YELLOW}Ceph not accessible${NC}"
+echo ""
+
+echo "=========================================="
+echo -e "${GREEN}Ready to Prepare Drive${NC}"
+echo "=========================================="
+echo ""
+echo "To prepare a drive for Ceph OSD, run:"
+echo ""
+echo -e "${YELLOW}# Choose a drive (e.g., sdc, sdd, sde, sdf, sdg, or sdh)${NC}"
+echo -e "${YELLOW}# Replace DRIVE with your choice${NC}"
+echo ""
+echo "# 1. Check if drive is in ubuntu-vg"
+echo "pvs | grep DRIVE"
+echo ""
+echo "# 2. If in ubuntu-vg, unmount and remove (WARNING: Destroys data!)"
+echo "umount /dev/ubuntu-vg/* 2>/dev/null || true"
+echo "vgchange -a n ubuntu-vg"
+echo "pvremove /dev/DRIVE3"
+echo ""
+echo "# 3. Wipe the entire drive"
+echo "wipefs -a /dev/DRIVE"
+echo "dd if=/dev/zero of=/dev/DRIVE bs=1M count=100"
+echo ""
+echo "# 4. Create Ceph OSD"
+echo "ceph-volume lvm create --data /dev/DRIVE"
+echo ""
+echo "# 5. Verify"
+echo "ceph osd tree"
+echo "ceph health"
+echo ""
+

scripts/reinitialize-ceph-cluster.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+# Reinitialize Ceph cluster after reinstall
+set -e
+
+FSID="5fb968ae-12ab-405f-b05f-0df29a168328"
+PUBLIC_NETWORK="192.168.11.0/24"
+CLUSTER_NETWORK="192.168.11.0/24"
+
+echo "=== Reinitializing Ceph Cluster ==="
+
+# Check if we're on a Proxmox node
+if [ ! -d "/etc/pve" ]; then
+    echo "ERROR: This script must be run on a Proxmox node"
+    exit 1
+fi
+
+echo ""
+echo "=== Step 1: Creating Ceph configuration ==="
+cat > /etc/pve/ceph.conf <<EOF
+[global]
+	auth_client_required = cephx
+	auth_cluster_required = cephx
+	auth_service_required = cephx
+	cluster_network = $CLUSTER_NETWORK
+	fsid = $FSID
+	mon_allow_pool_delete = true
+	mon_host = 192.168.11.10 192.168.11.11
+	ms_bind_ipv4 = true
+	ms_bind_ipv6 = false
+	osd_pool_default_min_size = 2
+	osd_pool_default_size = 3
+	public_network = $PUBLIC_NETWORK
+
+[client]
+	keyring = /etc/pve/priv/\$cluster.\$name.keyring
+
+[client.crash]
+	keyring = /etc/pve/ceph/\$cluster.\$name.keyring
+
+[mds]
+	keyring = /var/lib/ceph/mds/ceph-\$id/keyring
+
+[mds.ml110-01]
+	host = ml110-01
+	mds_standby_for_name = pve
+
+[mds.r630-01]
+	host = r630-01
+	mds_standby_for_name = pve
+
+[mon.ml110-01]
+	public_addr = 192.168.11.10
+
+[mon.r630-01]
+	public_addr = 192.168.11.11
+EOF
+
+# Create symlink (Proxmox uses /etc/pve/ceph.conf which is symlinked to /etc/ceph/ceph.conf)
+if [ ! -L /etc/ceph/ceph.conf ]; then
+    ln -sf /etc/pve/ceph.conf /etc/ceph/ceph.conf 2>/dev/null || true
+fi
+
+echo "Configuration created"
+
+echo ""
+echo "=== Step 2: Initializing Ceph cluster ==="
+HOSTNAME=$(hostname)
+if [ "$HOSTNAME" = "ml110-01" ]; then
+    echo "Initializing cluster on ml110-01..."
+    pveceph init --network $PUBLIC_NETWORK --cluster-network $CLUSTER_NETWORK 2>&1 || echo "Init may have already been done"
+fi
+
+echo ""
+echo "=== Step 3: Creating monitors ==="
+if [ "$HOSTNAME" = "ml110-01" ]; then
+    echo "Creating monitor on ml110-01..."
+    pveceph mon create ml110-01 2>&1 || echo "Monitor may already exist"
+fi
+
+if [ "$HOSTNAME" = "r630-01" ]; then
+    echo "Creating monitor on r630-01..."
+    pveceph mon create r630-01 2>&1 || echo "Monitor may already exist"
+fi
+
+echo ""
+echo "=== Step 4: Starting monitor services ==="
+systemctl enable ceph-mon@$HOSTNAME
+systemctl start ceph-mon@$HOSTNAME
+sleep 5
+systemctl status ceph-mon@$HOSTNAME --no-pager | head -15
+
+echo ""
+echo "=== Step 5: Verifying quorum ==="
+sleep 10
+ceph quorum_status 2>&1 | head -20 || echo "Quorum not yet established, may take a few minutes"
+
+echo ""
+echo "=== COMPLETE ==="
+echo "Ceph cluster reinitialized. Next:"
+echo "1. Verify quorum: ceph quorum_status"
+echo "2. Create OSDs: ceph-volume lvm create --data /dev/<disk>"
+echo "3. Check cluster health: ceph -s"
+

scripts/remove-old-image-by-id.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+set -e
+
+echo "=========================================="
+echo "REMOVE OLD IMAGE BY ID"
+echo "=========================================="
+echo ""
+
+OLD_IMAGE_ID="sha256:5d83df9f8d127aa8988ace01254f4adbb6336f0ad59be8193349b3eacd972cfc"
+IMAGE_NAME="docker.io/library/crossplane-provider-proxmox"
+
+echo "Step 1: Listing all crossplane images..."
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox || echo "  (No images found)"
+echo ""
+
+echo "Step 2: Removing old image by specific ID..."
+sudo ctr -n k8s.io images rm "${IMAGE_NAME}@${OLD_IMAGE_ID}" 2>&1 || echo "  (Image not found by ID)"
+echo ""
+
+echo "Step 3: Removing all images with the name (to be safe)..."
+ALL_IMAGES=$(sudo ctr -n k8s.io images ls -q | grep "${IMAGE_NAME}" || true)
+if [ -n "$ALL_IMAGES" ]; then
+    echo "$ALL_IMAGES" | while read -r img; do
+        echo "  Removing: $img"
+        sudo ctr -n k8s.io images rm "$img" 2>/dev/null || true
+    done
+fi
+echo ""
+
+echo "Step 4: Re-importing fresh image..."
+if [ ! -f /tmp/crossplane-fresh-nuclear.tar ]; then
+    echo "  ❌ ERROR: Image tar not found!"
+    exit 1
+fi
+sudo ctr -n k8s.io images import /tmp/crossplane-fresh-nuclear.tar
+echo "  ✅ Image imported"
+echo ""
+
+echo "Step 5: Verifying import..."
+sudo ctr -n k8s.io images ls | grep crossplane-provider-proxmox
+echo ""
+
+echo "=========================================="
+echo "OLD IMAGE REMOVAL COMPLETE"
+echo "=========================================="
+echo ""
+echo "Now restart the pod:"
+echo "  kubectl delete pod -n crossplane-system -l app=crossplane-provider-proxmox"
+echo "  kubectl wait --for=condition=ready pod -n crossplane-system -l app=crossplane-provider-proxmox --timeout=120s"
+echo "  kubectl logs -n crossplane-system -l app=crossplane-provider-proxmox | grep '\[FIXED CODE\]'"
+

scripts/review-vm-deployments.sh

@@ -0,0 +1,112 @@
+#!/bin/bash
+# Review VM Deployment Details
+# Analyzes each VM configuration and deployment steps
+
+set -e
+
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+RED='\033[0;31m'
+NC='\033[0m'
+
+echo "=========================================="
+echo "VM Deployment Review"
+echo "=========================================="
+echo ""
+
+# Count VMs
+TOTAL_VMS=$(find examples/production -name "*.yaml" -type f | wc -l)
+echo -e "${BLUE}Total VMs to Deploy:${NC} $TOTAL_VMS"
+echo ""
+
+# Categorize VMs
+echo -e "${BLUE}VM Categories:${NC}"
+echo "  Core Infrastructure: $(ls -1 examples/production/*.yaml 2>/dev/null | wc -l)"
+echo "  Phoenix Infrastructure: $(ls -1 examples/production/phoenix/*.yaml 2>/dev/null | wc -l)"
+echo "  Blockchain Infrastructure: $(ls -1 examples/production/smom-dbis-138/*.yaml 2>/dev/null | wc -l)"
+echo ""
+
+# Review each VM
+echo "=========================================="
+echo "Detailed VM Configuration Review"
+echo "=========================================="
+echo ""
+
+VM_COUNT=0
+for file in $(find examples/production -name "*.yaml" -type f | sort); do
+    VM_COUNT=$((VM_COUNT + 1))
+    VM_NAME=$(basename "$file" .yaml)
+    CATEGORY=$(dirname "$file" | sed 's|examples/production/||' | sed 's|^\.$|core|')
+    
+    echo -e "${BLUE}[$VM_COUNT/$TOTAL_VMS] $VM_NAME${NC}"
+    echo "  Category: $CATEGORY"
+    echo "  File: $file"
+    
+    # Extract configuration
+    if kubectl apply --dry-run=client -f "$file" -o json 2>/dev/null > /tmp/vm_config.json; then
+        NODE=$(jq -r '.spec.forProvider.node // "N/A"' /tmp/vm_config.json)
+        SITE=$(jq -r '.spec.forProvider.site // "N/A"' /tmp/vm_config.json)
+        CPU=$(jq -r '.spec.forProvider.cpu // "N/A"' /tmp/vm_config.json)
+        MEMORY=$(jq -r '.spec.forProvider.memory // "N/A"' /tmp/vm_config.json)
+        DISK=$(jq -r '.spec.forProvider.disk // "N/A"' /tmp/vm_config.json)
+        STORAGE=$(jq -r '.spec.forProvider.storage // "N/A"' /tmp/vm_config.json)
+        NETWORK=$(jq -r '.spec.forProvider.network // "N/A"' /tmp/vm_config.json)
+        IMAGE=$(jq -r '.spec.forProvider.image // "N/A"' /tmp/vm_config.json)
+        USERDATA=$(jq -r '.spec.forProvider.userData // ""' /tmp/vm_config.json | wc -c)
+        
+        echo "  Configuration:"
+        echo "    Node: $NODE"
+        echo "    Site: $SITE"
+        echo "    CPU: $CPU cores"
+        echo "    Memory: $MEMORY"
+        echo "    Disk: $DISK"
+        echo "    Storage: $STORAGE"
+        echo "    Network: $NETWORK"
+        echo "    Image: $IMAGE"
+        if [ "$USERDATA" -gt 10 ]; then
+            echo "    Cloud-init: ✅ Configured"
+        else
+            echo "    Cloud-init: ⚠️  Not configured"
+        fi
+        
+        # Check current status
+        if kubectl get proxmoxvm "$VM_NAME" -o json 2>/dev/null > /tmp/vm_status.json; then
+            VMID=$(jq -r '.status.vmId // "pending"' /tmp/vm_status.json)
+            STATE=$(jq -r '.status.state // "creating"' /tmp/vm_status.json)
+            echo "  Status:"
+            echo "    VMID: $VMID"
+            echo "    State: $STATE"
+        else
+            echo "  Status: Not yet deployed"
+        fi
+        
+        echo ""
+    else
+        echo -e "${RED}  Error: Cannot read configuration${NC}"
+        echo ""
+    fi
+done
+
+# Summary
+echo "=========================================="
+echo "Deployment Summary"
+echo "=========================================="
+echo ""
+echo "Total VMs: $TOTAL_VMS"
+echo ""
+
+# Count by node
+echo "VMs by Node:"
+find examples/production -name "*.yaml" -type f -exec sh -c 'kubectl apply --dry-run=client -f "$1" -o json 2>/dev/null | jq -r ".spec.forProvider.node // \"unknown\""' _ {} \; | sort | uniq -c
+
+echo ""
+echo "VMs by Site:"
+find examples/production -name "*.yaml" -type f -exec sh -c 'kubectl apply --dry-run=client -f "$1" -o json 2>/dev/null | jq -r ".spec.forProvider.site // \"unknown\""' _ {} \; | sort | uniq -c
+
+echo ""
+echo "VMs by Storage:"
+find examples/production -name "*.yaml" -type f -exec sh -c 'kubectl apply --dry-run=client -f "$1" -o json 2>/dev/null | jq -r ".spec.forProvider.storage // \"unknown\""' _ {} \; | sort | uniq -c
+
+rm -f /tmp/vm_config.json /tmp/vm_status.json
+

scripts/run-ceph-osd-setup-remote.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Helper script to run Ceph OSD setup on r630-01 remotely
+# Usage: ./run-ceph-osd-setup-remote.sh [password] [--include-sdb]
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SETUP_SCRIPT="$SCRIPT_DIR/setup-ceph-osds-r630.sh"
+PROXMOX_HOST="192.168.11.11"
+PROXMOX_USER="root"
+
+# Get password from argument or environment
+if [ -n "$1" ] && [ "$1" != "--include-sdb" ]; then
+    PASSWORD="$1"
+    shift
+elif [ -n "$PROXMOX_PASSWORD" ]; then
+    PASSWORD="$PROXMOX_PASSWORD"
+fi
+
+# Check for --include-sdb flag
+INCLUDE_SDB="false"
+if [[ "$@" == *"--include-sdb"* ]]; then
+    INCLUDE_SDB="true"
+fi
+
+# Check if sshpass is installed
+if ! command -v sshpass &>/dev/null; then
+    echo "Error: sshpass not installed. Install with: sudo apt-get install sshpass"
+    exit 1
+fi
+
+if [ -z "$PASSWORD" ]; then
+    echo "Usage: $0 <password> [--include-sdb]"
+    echo "Or: PROXMOX_PASSWORD=yourpass $0 [--include-sdb]"
+    exit 1
+fi
+
+echo "=========================================="
+echo "Running Ceph OSD Setup on R630-01"
+echo "=========================================="
+echo ""
+
+# Copy script to remote server
+echo "Copying setup script to $PROXMOX_USER@$PROXMOX_HOST..."
+sshpass -p "$PASSWORD" scp -o StrictHostKeyChecking=no "$SETUP_SCRIPT" "$PROXMOX_USER@$PROXMOX_HOST:/tmp/setup-ceph-osds-r630.sh" || {
+    echo "Error: Failed to copy script"
+    exit 1
+}
+
+# Make it executable
+sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$PROXMOX_USER@$PROXMOX_HOST" "chmod +x /tmp/setup-ceph-osds-r630.sh"
+
+# Run the script
+echo ""
+echo "Running setup script..."
+echo ""
+
+if [ "$INCLUDE_SDB" = "true" ]; then
+    sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$PROXMOX_USER@$PROXMOX_HOST" "INCLUDE_SDB=true /tmp/setup-ceph-osds-r630.sh"
+else
+    sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$PROXMOX_USER@$PROXMOX_HOST" "/tmp/setup-ceph-osds-r630.sh"
+fi
+

scripts/run-disk-analysis-remote.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Helper script to run disk analysis on r630-01
+# This script copies the analysis script and runs it remotely
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+ANALYSIS_SCRIPT="$SCRIPT_DIR/analyze-r630-disk-layout.sh"
+
+# Check if analysis script exists
+if [ ! -f "$ANALYSIS_SCRIPT" ]; then
+    echo "Error: Analysis script not found at $ANALYSIS_SCRIPT"
+    exit 1
+fi
+
+echo "=========================================="
+echo "R630-01 Disk Analysis Runner"
+echo "=========================================="
+echo ""
+echo "This script will run the disk analysis on r630-01"
+echo ""
+echo "Option 1: Run via SSH (requires password)"
+echo "  ssh root@192.168.11.11 'bash -s' < $ANALYSIS_SCRIPT"
+echo ""
+echo "Option 2: Copy script and run"
+echo "  scp $ANALYSIS_SCRIPT root@192.168.11.11:/tmp/"
+echo "  ssh root@192.168.11.11 '/tmp/analyze-r630-disk-layout.sh'"
+echo ""
+echo "Option 3: Use existing SSH script"
+echo "  bash scripts/ssh-r630-01.sh 'bash -s' < $ANALYSIS_SCRIPT"
+echo ""
+echo "=========================================="
+echo "Attempting Option 3..."
+echo "=========================================="
+echo ""
+
+# Try to run via ssh-r630-01.sh
+if [ -f "$SCRIPT_DIR/ssh-r630-01.sh" ]; then
+    bash "$SCRIPT_DIR/ssh-r630-01.sh" 'bash -s' < "$ANALYSIS_SCRIPT"
+else
+    echo "ssh-r630-01.sh not found. Please run manually using one of the options above."
+    exit 1
+fi
+

scripts/run-package-install-remote.sh

@@ -0,0 +1,57 @@
+#!/bin/bash
+# Helper script to install packages on r630-01 remotely
+# Usage: ./run-package-install-remote.sh [password]
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INSTALL_SCRIPT="$SCRIPT_DIR/install-r630-packages.sh"
+PROXMOX_HOST="192.168.11.11"
+PROXMOX_USER="root"
+
+# Get password from argument or environment
+if [ -n "$1" ]; then
+    PASSWORD="$1"
+elif [ -n "$PROXMOX_PASSWORD" ]; then
+    PASSWORD="$PROXMOX_PASSWORD"
+fi
+
+# Check if sshpass is installed
+if ! command -v sshpass &>/dev/null; then
+    echo "Error: sshpass not installed. Install with: sudo apt-get install sshpass"
+    exit 1
+fi
+
+if [ -z "$PASSWORD" ]; then
+    echo "Usage: $0 <password>"
+    echo "Or: PROXMOX_PASSWORD=yourpass $0"
+    exit 1
+fi
+
+echo "=========================================="
+echo "Installing Packages on R630-01"
+echo "=========================================="
+echo ""
+
+# Copy script to remote server
+echo "Copying installation script to $PROXMOX_USER@$PROXMOX_HOST..."
+sshpass -p "$PASSWORD" scp -o StrictHostKeyChecking=no "$INSTALL_SCRIPT" "$PROXMOX_USER@$PROXMOX_HOST:/tmp/install-r630-packages.sh" || {
+    echo "Error: Failed to copy script"
+    exit 1
+}
+
+# Make it executable
+sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$PROXMOX_USER@$PROXMOX_HOST" "chmod +x /tmp/install-r630-packages.sh"
+
+# Run the script
+echo ""
+echo "Running installation script..."
+echo ""
+
+sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no -t "$PROXMOX_USER@$PROXMOX_HOST" "/tmp/install-r630-packages.sh"
+
+echo ""
+echo "=========================================="
+echo "Package installation complete!"
+echo "=========================================="
+

scripts/run-remote-commands.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Script to run commands on R630-01 using sshpass
+# Usage: ./run-remote-commands.sh [password]
+
+set -e
+
+PROXMOX_HOST="192.168.11.11"
+PROXMOX_USER="root"
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 <password>"
+    echo "Or set PROXMOX_PASSWORD environment variable"
+    exit 1
+fi
+
+PASSWORD="$1"
+
+# Check if sshpass is installed
+if ! command -v sshpass &> /dev/null; then
+    echo "sshpass not found. Installing..."
+    sudo apt-get update && sudo apt-get install -y sshpass
+fi
+
+echo "Running commands on $PROXMOX_USER@$PROXMOX_HOST..."
+echo ""
+
+# Run pvesm status
+echo "=== Proxmox Storage Status ==="
+sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$PROXMOX_USER@$PROXMOX_HOST" "pvesm status"
+echo ""
+
+# Run lsblk
+echo "=== Block Devices ==="
+sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$PROXMOX_USER@$PROXMOX_HOST" "lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL"
+echo ""
+
+# Check 250GB drives
+echo "=== 250GB Drives Status ==="
+sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$PROXMOX_USER@$PROXMOX_HOST" "
+for drive in sdc sdd sde sdf sdg sdh; do
+    if [ -b \"/dev/\$drive\" ]; then
+        echo \"\"
+        echo \"/dev/\$drive:\"
+        lsblk /dev/\$drive
+    fi
+done
+"
+

scripts/run-with-password.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Helper script to run commands on r630-01 with password
+# Usage: ./run-with-password.sh "command"
+
+PASSWORD="L@kers2010"
+HOST="root@192.168.11.11"
+
+if [ -z "$1" ]; then
+    echo "Usage: $0 'command'"
+    exit 1
+fi
+
+# Try sshpass first
+if command -v sshpass &> /dev/null; then
+    sshpass -p "$PASSWORD" ssh -o StrictHostKeyChecking=no "$HOST" "$1"
+# Try expect
+elif command -v expect &> /dev/null; then
+    expect << EOF
+spawn ssh -o StrictHostKeyChecking=no "$HOST" "$1"
+expect "password:"
+send "$PASSWORD\r"
+expect eof
+EOF
+else
+    echo "Error: sshpass or expect not found. Please install one:"
+    echo "  sudo apt install sshpass"
+    echo "  OR"
+    echo "  sudo apt install expect"
+    exit 1
+fi
+
+

scripts/scan-projects.ts

@@ -0,0 +1,951 @@
+#!/usr/bin/env tsx
+/**
+ * Project Scanner Script
+ * 
+ * Scans ~/projects/ directory for Git repos and monorepos,
+ * then adds them as Tenants (clients) in the Sankofa system.
+ * 
+ * Features:
+ * - Detects Git repositories and monorepos
+ * - Extracts project metadata (Git remote, branch, package manager, workspaces)
+ * - Creates Tenants via GraphQL API with proper authentication
+ * - Duplicate detection and validation
+ * - Progress reporting and structured logging
+ * 
+ * Usage:
+ *   tsx scripts/scan-projects.ts [options]
+ * 
+ * Options:
+ *   --dry-run              Show what would be created without actually creating
+ *   --skip-existing        Skip projects that already exist as tenants
+ *   --projects-dir=PATH    Directory to scan (default: ~/projects/)
+ *   --api-url=URL          GraphQL API URL (default: http://localhost:4000/graphql)
+ *   --email=EMAIL          Email for authentication (or set GRAPHQL_EMAIL env var)
+ *   --password=PASSWORD    Password for authentication (or set GRAPHQL_PASSWORD env var)
+ *   --auth-token=TOKEN     Direct authentication token (or set GRAPHQL_AUTH_TOKEN env var)
+ *   --output=json          Output results as JSON
+ *   --verbose              Enable verbose logging
+ *   --batch-size=N         Number of projects to process in parallel (default: 1)
+ */
+
+import { readdir, stat, readFile } from 'fs/promises'
+import { join, basename } from 'path'
+import { existsSync } from 'fs'
+import { execSync } from 'child_process'
+import { program } from 'commander'
+import cliProgress from 'cli-progress'
+import * as dotenv from 'dotenv'
+import { z } from 'zod'
+
+// Load environment variables
+dotenv.config()
+
+// Validation schemas
+const projectNameSchema = z.string().min(1).max(255).regex(/^[a-zA-Z0-9_-]+$/, {
+  message: 'Project name must contain only alphanumeric characters, hyphens, and underscores'
+})
+
+const gitUrlSchema = z.string().url().or(z.literal(''))
+
+interface ProjectInfo {
+  name: string
+  path: string
+  type: 'repo' | 'monorepo' | 'other'
+  gitRemote?: string
+  gitBranch?: string
+  packageManager?: 'npm' | 'pnpm' | 'yarn' | 'unknown'
+  workspaces?: string[]
+  metadata: Record<string, unknown>
+}
+
+interface GraphQLResponse<T = unknown> {
+  data?: T
+  errors?: Array<{ message: string; extensions?: { code?: string } }>
+}
+
+interface AuthPayload {
+  token: string
+  user: {
+    id: string
+    email: string
+    name: string
+    role: string
+  }
+}
+
+interface Tenant {
+  id: string
+  name: string
+  domain?: string | null
+}
+
+interface ScanOptions {
+  projectsDir: string
+  dryRun: boolean
+  skipExisting: boolean
+  apiUrl: string
+  email?: string
+  password?: string
+  authToken?: string
+  outputFormat: 'text' | 'json'
+  verbose: boolean
+  batchSize: number
+}
+
+interface ScanResult {
+  created: Array<{ name: string; id: string }>
+  skipped: Array<{ name: string; reason: string }>
+  errors: Array<{ name: string; error: string }>
+}
+
+class ProjectScanner {
+  private options: ScanOptions
+  private authToken?: string
+  private existingTenants: Map<string, Tenant> = new Map()
+
+  constructor(options: ScanOptions) {
+    this.options = options
+  }
+
+  /**
+   * Authenticate with the GraphQL API
+   */
+  async authenticate(): Promise<void> {
+    if (this.options.authToken) {
+      this.authToken = this.options.authToken
+      if (this.options.verbose) {
+        console.log('Using provided authentication token')
+      }
+      return
+    }
+
+    if (!this.options.email || !this.options.password) {
+      throw new Error(
+        'Authentication required. Provide --email and --password, or --auth-token, ' +
+        'or set GRAPHQL_EMAIL/GRAPHQL_PASSWORD or GRAPHQL_AUTH_TOKEN environment variables'
+      )
+    }
+
+    const mutation = `
+      mutation Login($email: String!, $password: String!) {
+        login(email: $email, password: $password) {
+          token
+          user {
+            id
+            email
+            name
+            role
+          }
+        }
+      }
+    `
+
+    try {
+      const response = await this.executeGraphQL<{ login: AuthPayload }>(
+        mutation,
+        { email: this.options.email, password: this.options.password },
+        'login'
+      )
+
+      if (!response) {
+        throw new Error('Login failed: No response received')
+      }
+
+      this.authToken = response.login.token
+
+      if (response.login.user.role !== 'ADMIN') {
+        throw new Error(
+          `Admin role required. Current role: ${response.login.user.role}. ` +
+          'Only ADMIN users can create tenants.'
+        )
+      }
+
+      if (this.options.verbose) {
+        console.log(`Authenticated as ${response.login.user.email} (${response.login.user.role})`)
+      }
+    } catch (error) {
+      throw new Error(`Authentication failed: ${error instanceof Error ? error.message : String(error)}`)
+    }
+  }
+
+  /**
+   * Load existing tenants for duplicate detection
+   */
+  async loadExistingTenants(): Promise<void> {
+    if (this.options.dryRun || !this.options.skipExisting) {
+      return
+    }
+
+    const query = `
+      query ListTenants {
+        tenants {
+          id
+          name
+          domain
+        }
+      }
+    `
+
+    try {
+      const response = await this.executeGraphQL<{ tenants: Tenant[] }>(
+        query,
+        {},
+        'tenants'
+      )
+
+      if (response) {
+        for (const tenant of response.tenants) {
+          this.existingTenants.set(tenant.name.toLowerCase(), tenant)
+          if (tenant.domain) {
+            this.existingTenants.set(tenant.domain.toLowerCase(), tenant)
+          }
+        }
+
+        if (this.options.verbose) {
+          console.log(`Loaded ${this.existingTenants.size} existing tenants for duplicate detection`)
+        }
+      }
+    } catch (error) {
+      if (this.options.verbose) {
+        console.warn(`Warning: Could not load existing tenants: ${error instanceof Error ? error.message : String(error)}`)
+      }
+    }
+  }
+
+  /**
+   * Check if a directory is a Git repository
+   */
+  private async isGitRepo(dirPath: string): Promise<boolean> {
+    return existsSync(join(dirPath, '.git'))
+  }
+
+  /**
+   * Get Git remote URL if available
+   */
+  private async getGitRemote(dirPath: string): Promise<string | undefined> {
+    try {
+      const result = execSync('git remote get-url origin', {
+        cwd: dirPath,
+        encoding: 'utf-8',
+        stdio: ['ignore', 'pipe', 'ignore'],
+      })
+      const url = result.trim()
+      // Validate URL
+      gitUrlSchema.parse(url)
+      return url
+    } catch {
+      return undefined
+    }
+  }
+
+  /**
+   * Get current Git branch
+   */
+  private async getGitBranch(dirPath: string): Promise<string | undefined> {
+    try {
+      const result = execSync('git rev-parse --abbrev-ref HEAD', {
+        cwd: dirPath,
+        encoding: 'utf-8',
+        stdio: ['ignore', 'pipe', 'ignore'],
+      })
+      return result.trim()
+    } catch {
+      return undefined
+    }
+  }
+
+  /**
+   * Extract domain from Git remote URL
+   */
+  private extractDomainFromGitRemote(remote?: string): string | undefined {
+    if (!remote) return undefined
+
+    try {
+      // Handle various Git URL formats
+      // https://github.com/user/repo.git -> github.com
+      // git@github.com:user/repo.git -> github.com
+      // https://gitlab.com/user/repo.git -> gitlab.com
+      // https://bitbucket.org/user/repo.git -> bitbucket.org
+
+      const patterns = [
+        /^https?:\/\/([^\/]+)\//,
+        /^git@([^:]+):/,
+        /^ssh:\/\/git@([^\/]+)\//,
+      ]
+
+      for (const pattern of patterns) {
+        const match = remote.match(pattern)
+        if (match && match[1]) {
+          const host = match[1].replace(/^www\./, '')
+          // Generate a safe domain name
+          return `${host.replace(/\./g, '-')}.dev`
+        }
+      }
+    } catch {
+      // Ignore errors
+    }
+
+    return undefined
+  }
+
+  /**
+   * Detect package manager from lock files
+   */
+  private detectPackageManager(dirPath: string): 'npm' | 'pnpm' | 'yarn' | 'unknown' {
+    if (existsSync(join(dirPath, 'pnpm-lock.yaml'))) return 'pnpm'
+    if (existsSync(join(dirPath, 'yarn.lock'))) return 'yarn'
+    if (existsSync(join(dirPath, 'package-lock.json'))) return 'npm'
+    return 'unknown'
+  }
+
+  /**
+   * Check if a directory is a monorepo
+   */
+  private async isMonorepo(dirPath: string): Promise<boolean> {
+    const packageJsonPath = join(dirPath, 'package.json')
+    if (!existsSync(packageJsonPath)) return false
+
+    try {
+      const content = await readFile(packageJsonPath, 'utf-8')
+      const pkg = JSON.parse(content)
+
+      // Check for workspaces (npm/yarn)
+      if (pkg.workspaces && Array.isArray(pkg.workspaces) && pkg.workspaces.length > 0) {
+        return true
+      }
+
+      // Check for pnpm workspace
+      if (existsSync(join(dirPath, 'pnpm-workspace.yaml'))) {
+        return true
+      }
+
+      // Check for lerna
+      if (existsSync(join(dirPath, 'lerna.json'))) {
+        return true
+      }
+
+      // Check for nx
+      if (existsSync(join(dirPath, 'nx.json'))) {
+        return true
+      }
+
+      // Check for turborepo
+      if (existsSync(join(dirPath, 'turbo.json'))) {
+        return true
+      }
+
+      // Check for multiple package.json files in subdirectories
+      const entries = await readdir(dirPath, { withFileTypes: true })
+      let packageJsonCount = 0
+      for (const entry of entries) {
+        if (entry.isDirectory() && !entry.name.startsWith('.')) {
+          if (existsSync(join(dirPath, entry.name, 'package.json'))) {
+            packageJsonCount++
+            if (packageJsonCount >= 2) return true
+          }
+        }
+      }
+    } catch {
+      // If we can't read/parse package.json, it's not a monorepo
+    }
+
+    return false
+  }
+
+  /**
+   * Get workspace packages for a monorepo
+   */
+  private async getWorkspaces(dirPath: string): Promise<string[]> {
+    const workspaces: string[] = []
+    const packageJsonPath = join(dirPath, 'package.json')
+
+    if (existsSync(packageJsonPath)) {
+      try {
+        const content = await readFile(packageJsonPath, 'utf-8')
+        const pkg = JSON.parse(content)
+
+        if (pkg.workspaces && Array.isArray(pkg.workspaces)) {
+          // Expand workspace patterns
+          for (const pattern of pkg.workspaces) {
+            const glob = pattern.replace('/*', '').replace('*', '')
+            const workspaceDir = join(dirPath, glob)
+            if (existsSync(workspaceDir)) {
+              const entries = await readdir(workspaceDir, { withFileTypes: true })
+              for (const entry of entries) {
+                if (entry.isDirectory()) {
+                  const workspacePath = join(workspaceDir, entry.name)
+                  if (existsSync(join(workspacePath, 'package.json'))) {
+                    workspaces.push(entry.name)
+                  }
+                }
+              }
+            }
+          }
+        }
+      } catch {
+        // Ignore errors
+      }
+    }
+
+    // Check pnpm-workspace.yaml
+    const pnpmWorkspacePath = join(dirPath, 'pnpm-workspace.yaml')
+    if (existsSync(pnpmWorkspacePath)) {
+      try {
+        const content = await readFile(pnpmWorkspacePath, 'utf-8')
+        // Simple parsing for pnpm workspace
+        const lines = content.split('\n')
+        for (const line of lines) {
+          const match = line.match(/^[\s-]+['"]?([^'"]+)['"]?/)
+          if (match) {
+            workspaces.push(match[1])
+          }
+        }
+      } catch {
+        // Ignore errors
+      }
+    }
+
+    return workspaces
+  }
+
+  /**
+   * Validate and sanitize project name
+   */
+  private validateProjectName(name: string): string {
+    // Remove invalid characters and replace with hyphens
+    let sanitized = name.replace(/[^a-zA-Z0-9_-]/g, '-')
+    // Remove consecutive hyphens
+    sanitized = sanitized.replace(/-+/g, '-')
+    // Remove leading/trailing hyphens
+    sanitized = sanitized.replace(/^-+|-+$/g, '')
+    // Ensure it's not empty
+    if (!sanitized) {
+      sanitized = 'project'
+    }
+    // Truncate to max length
+    if (sanitized.length > 255) {
+      sanitized = sanitized.substring(0, 255)
+    }
+
+    // Validate with schema
+    projectNameSchema.parse(sanitized)
+    return sanitized
+  }
+
+  /**
+   * Sanitize metadata to prevent injection and size issues
+   */
+  private sanitizeMetadata(metadata: Record<string, unknown>): Record<string, unknown> {
+    const sanitized: Record<string, unknown> = {}
+    const maxSize = 10 * 1024 // 10KB limit
+
+    for (const [key, value] of Object.entries(metadata)) {
+      // Only allow string, number, boolean, null, and arrays/objects of these types
+      if (
+        typeof value === 'string' ||
+        typeof value === 'number' ||
+        typeof value === 'boolean' ||
+        value === null
+      ) {
+        sanitized[key] = value
+      } else if (Array.isArray(value)) {
+        sanitized[key] = value.filter(
+          (v) => typeof v === 'string' || typeof v === 'number' || typeof v === 'boolean' || v === null
+        )
+      } else if (typeof value === 'object') {
+        // Recursively sanitize nested objects
+        sanitized[key] = this.sanitizeMetadata(value as Record<string, unknown>)
+      }
+    }
+
+    // Check size
+    const jsonString = JSON.stringify(sanitized)
+    if (jsonString.length > maxSize) {
+      // Truncate by removing less important fields
+      delete sanitized.description
+      delete sanitized.keywords
+      const truncated = JSON.stringify(sanitized)
+      if (truncated.length > maxSize) {
+        throw new Error(`Metadata too large (${jsonString.length} bytes, max ${maxSize} bytes)`)
+      }
+    }
+
+    return sanitized
+  }
+
+  /**
+   * Scan a directory and collect project information
+   */
+  private async scanDirectory(dirPath: string): Promise<ProjectInfo | null> {
+    const name = basename(dirPath)
+    const stats = await stat(dirPath)
+
+    if (!stats.isDirectory()) {
+      return null
+    }
+
+    // Skip hidden directories and common non-project directories
+    if (name.startsWith('.') || ['node_modules', 'dist', 'build', '.git'].includes(name)) {
+      return null
+    }
+
+    const isGit = await this.isGitRepo(dirPath)
+    const isMono = await this.isMonorepo(dirPath)
+
+    // Only process Git repos or directories with package.json
+    if (!isGit && !existsSync(join(dirPath, 'package.json'))) {
+      return null
+    }
+
+    // Validate and sanitize project name
+    const sanitizedName = this.validateProjectName(name)
+
+    const projectInfo: ProjectInfo = {
+      name: sanitizedName,
+      path: dirPath,
+      type: isMono ? 'monorepo' : isGit ? 'repo' : 'other',
+      metadata: {},
+    }
+
+    if (isGit) {
+      projectInfo.gitRemote = await this.getGitRemote(dirPath)
+      projectInfo.gitBranch = await this.getGitBranch(dirPath)
+    }
+
+    if (existsSync(join(dirPath, 'package.json'))) {
+      projectInfo.packageManager = this.detectPackageManager(dirPath)
+
+      if (isMono) {
+        projectInfo.workspaces = await this.getWorkspaces(dirPath)
+      }
+
+      // Read package.json for additional metadata
+      try {
+        const content = await readFile(join(dirPath, 'package.json'), 'utf-8')
+        const pkg = JSON.parse(content)
+        projectInfo.metadata = {
+          description: pkg.description,
+          version: pkg.version,
+          author: typeof pkg.author === 'string' ? pkg.author : pkg.author?.name,
+          license: pkg.license,
+          keywords: Array.isArray(pkg.keywords) ? pkg.keywords.slice(0, 10) : undefined,
+        }
+      } catch {
+        // Ignore errors
+      }
+    }
+
+    // Sanitize metadata
+    projectInfo.metadata = this.sanitizeMetadata(projectInfo.metadata)
+
+    return projectInfo
+  }
+
+  /**
+   * Scan all projects in the projects directory
+   */
+  async scanAllProjects(): Promise<ProjectInfo[]> {
+    const projects: ProjectInfo[] = []
+
+    if (!existsSync(this.options.projectsDir)) {
+      throw new Error(`Projects directory does not exist: ${this.options.projectsDir}`)
+    }
+
+    if (this.options.verbose) {
+      console.log(`Scanning projects in: ${this.options.projectsDir}`)
+    }
+
+    const entries = await readdir(this.options.projectsDir, { withFileTypes: true })
+
+    for (const entry of entries) {
+      if (entry.isDirectory()) {
+        const projectPath = join(this.options.projectsDir, entry.name)
+        try {
+          const projectInfo = await this.scanDirectory(projectPath)
+
+          if (projectInfo) {
+            projects.push(projectInfo)
+          }
+        } catch (error) {
+          if (this.options.verbose) {
+            console.warn(`Warning: Failed to scan ${entry.name}: ${error instanceof Error ? error.message : String(error)}`)
+          }
+        }
+      }
+    }
+
+    return projects
+  }
+
+  /**
+   * Check if tenant already exists
+   */
+  private isDuplicate(project: ProjectInfo): Tenant | null {
+    const nameKey = project.name.toLowerCase()
+    if (this.existingTenants.has(nameKey)) {
+      return this.existingTenants.get(nameKey)!
+    }
+
+    if (project.gitRemote) {
+      const domain = this.extractDomainFromGitRemote(project.gitRemote)
+      if (domain) {
+        const domainKey = domain.toLowerCase()
+        if (this.existingTenants.has(domainKey)) {
+          return this.existingTenants.get(domainKey)!
+        }
+      }
+    }
+
+    return null
+  }
+
+  /**
+   * Create a Tenant via GraphQL
+   */
+  private async createTenant(project: ProjectInfo): Promise<string | null> {
+    const mutation = `
+      mutation CreateTenant($input: CreateTenantInput!) {
+        createTenant(input: $input) {
+          id
+          name
+          domain
+          status
+          tier
+        }
+      }
+    `
+
+    const domain = this.extractDomainFromGitRemote(project.gitRemote)
+
+    const variables = {
+      input: {
+        name: project.name,
+        domain: domain || undefined,
+        tier: 'STANDARD' as const,
+        metadata: {
+          type: project.type,
+          path: project.path,
+          gitRemote: project.gitRemote,
+          gitBranch: project.gitBranch,
+          packageManager: project.packageManager,
+          workspaces: project.workspaces,
+          ...project.metadata,
+        },
+      },
+    }
+
+    return this.executeGraphQLWithRetry(mutation, variables, 'createTenant', (data) => data?.createTenant?.id)
+  }
+
+  /**
+   * Execute a GraphQL query/mutation with retry logic
+   */
+  private async executeGraphQLWithRetry<T>(
+    query: string,
+    variables: Record<string, unknown>,
+    operationName: string,
+    extractId?: (data: T) => string | null | undefined
+  ): Promise<string | null> {
+    const maxRetries = 3
+    let lastError: Error | null = null
+
+    for (let attempt = 0; attempt <= maxRetries; attempt++) {
+      if (attempt > 0) {
+        const delay = Math.min(1000 * Math.pow(2, attempt - 1), 10000) // Exponential backoff, max 10s
+        if (this.options.verbose) {
+          console.log(`Retrying ${operationName} (attempt ${attempt + 1}/${maxRetries + 1}) after ${delay}ms...`)
+        }
+        await new Promise((resolve) => setTimeout(resolve, delay))
+      }
+
+      try {
+        const response = await this.executeGraphQL<T>(query, variables, operationName)
+
+        if (response) {
+          if (extractId) {
+            const id = extractId(response)
+            return id || null
+          }
+          return (response as { id?: string })?.id || null
+        }
+
+        return null
+      } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error))
+
+        // Don't retry on validation or authentication errors
+        if (error instanceof Error) {
+          if (error.message.includes('UNAUTHENTICATED') || error.message.includes('FORBIDDEN')) {
+            throw error
+          }
+          if (error.message.includes('VALIDATION_ERROR') || error.message.includes('BAD_USER_INPUT')) {
+            throw error
+          }
+        }
+
+        if (attempt < maxRetries) {
+          continue
+        }
+      }
+    }
+
+    throw lastError || new Error(`Failed to execute ${operationName} after ${maxRetries + 1} attempts`)
+  }
+
+  /**
+   * Execute a GraphQL query/mutation
+   */
+  private async executeGraphQL<T>(
+    query: string,
+    variables: Record<string, unknown>,
+    operationName: string
+  ): Promise<T | null> {
+    if (this.options.dryRun) {
+      if (this.options.verbose) {
+        console.log(`[DRY RUN] Would execute: ${operationName}`)
+        console.log(`  Variables:`, JSON.stringify(variables, null, 2))
+      }
+      return null
+    }
+
+    const headers: Record<string, string> = {
+      'Content-Type': 'application/json',
+    }
+
+    if (this.authToken) {
+      headers['Authorization'] = `Bearer ${this.authToken}`
+    }
+
+    try {
+      const response = await fetch(this.options.apiUrl, {
+        method: 'POST',
+        headers,
+        body: JSON.stringify({
+          query,
+          variables,
+          operationName,
+        }),
+      })
+
+      if (!response.ok) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`)
+      }
+
+      const result: GraphQLResponse<T> = await response.json()
+
+      if (result.errors) {
+        const errorMessages = result.errors.map((e) => e.message).join(', ')
+        const errorCodes = result.errors.map((e) => e.extensions?.code).filter(Boolean).join(', ')
+        throw new Error(`${errorCodes ? `[${errorCodes}] ` : ''}${errorMessages}`)
+      }
+
+      return result.data || null
+    } catch (error) {
+      if (error instanceof Error) {
+        throw error
+      }
+      throw new Error(`Failed to execute ${operationName}: ${String(error)}`)
+    }
+  }
+
+  /**
+   * Import all scanned projects
+   */
+  async importProjects(projects: ProjectInfo[]): Promise<ScanResult> {
+    const result: ScanResult = {
+      created: [],
+      skipped: [],
+      errors: [],
+    }
+
+    if (projects.length === 0) {
+      console.log('No projects found to import.')
+      return result
+    }
+
+    if (!this.options.outputFormat || this.options.outputFormat === 'text') {
+      console.log(`\nFound ${projects.length} projects to import:`)
+      projects.forEach((p) => {
+        console.log(`  - ${p.name} (${p.type})`)
+      })
+    }
+
+    if (this.options.dryRun) {
+      if (!this.options.outputFormat || this.options.outputFormat === 'text') {
+        console.log('\n[DRY RUN] Would create the following tenants:')
+      }
+    } else {
+      if (!this.options.outputFormat || this.options.outputFormat === 'text') {
+        console.log(`\nCreating tenants...`)
+      }
+    }
+
+    // Create progress bar
+    const progressBar = this.options.outputFormat !== 'json' && !this.options.verbose
+      ? new cliProgress.SingleBar({
+          format: 'Progress |{bar}| {percentage}% | {value}/{total} projects | ETA: {eta}s',
+          barCompleteChar: '\u2588',
+          barIncompleteChar: '\u2591',
+          hideCursor: true,
+        })
+      : null
+
+    if (progressBar) {
+      progressBar.start(projects.length, 0)
+    }
+
+    // Process projects in batches
+    const batchSize = this.options.batchSize || 1
+    for (let i = 0; i < projects.length; i += batchSize) {
+      const batch = projects.slice(i, i + batchSize)
+      const batchPromises = batch.map(async (project) => {
+        try {
+          // Check for duplicates
+          if (this.options.skipExisting) {
+            const existing = this.isDuplicate(project)
+            if (existing) {
+              result.skipped.push({
+                name: project.name,
+                reason: `Already exists as tenant: ${existing.name} (ID: ${existing.id})`,
+              })
+              if (progressBar) progressBar.increment()
+              return
+            }
+          }
+
+          const id = await this.createTenant(project)
+
+          if (id) {
+            result.created.push({ name: project.name, id })
+            if (this.options.verbose && this.options.outputFormat !== 'json') {
+              console.log(`✓ Created tenant for ${project.name} (ID: ${id})`)
+            }
+          } else if (!this.options.dryRun) {
+            result.errors.push({
+              name: project.name,
+              error: 'Failed to create tenant (no ID returned)',
+            })
+          }
+        } catch (error) {
+          const errorMessage = error instanceof Error ? error.message : String(error)
+          result.errors.push({
+            name: project.name,
+            error: errorMessage,
+          })
+          if (this.options.verbose && this.options.outputFormat !== 'json') {
+            console.error(`✗ Error processing ${project.name}: ${errorMessage}`)
+          }
+        } finally {
+          if (progressBar) progressBar.increment()
+        }
+      })
+
+      await Promise.all(batchPromises)
+    }
+
+    if (progressBar) {
+      progressBar.stop()
+    }
+
+    // Print summary
+    if (this.options.outputFormat === 'json') {
+      console.log(JSON.stringify(result, null, 2))
+    } else {
+      console.log(`\nSummary:`)
+      console.log(`  Created: ${result.created.length}`)
+      console.log(`  Skipped: ${result.skipped.length}`)
+      console.log(`  Errors: ${result.errors.length}`)
+      console.log(`  Total: ${projects.length}`)
+
+      if (result.errors.length > 0 && this.options.verbose) {
+        console.log(`\nErrors:`)
+        result.errors.forEach((e) => {
+          console.log(`  - ${e.name}: ${e.error}`)
+        })
+      }
+    }
+
+    return result
+  }
+}
+
+// Main execution
+async function main() {
+  program
+    .name('scan-projects')
+    .description('Scan projects directory and create Tenants in Sankofa system')
+    .option('--dry-run', 'Show what would be created without actually creating', false)
+    .option('--skip-existing', 'Skip projects that already exist as tenants', false)
+    .option('--projects-dir <path>', 'Directory to scan', process.env.PROJECTS_DIR || join(process.env.HOME || '~', 'projects'))
+    .option('--api-url <url>', 'GraphQL API URL', process.env.GRAPHQL_API_URL || 'http://localhost:4000/graphql')
+    .option('--email <email>', 'Email for authentication', process.env.GRAPHQL_EMAIL)
+    .option('--password <password>', 'Password for authentication', process.env.GRAPHQL_PASSWORD)
+    .option('--auth-token <token>', 'Direct authentication token', process.env.GRAPHQL_AUTH_TOKEN)
+    .option('--output <format>', 'Output format: text or json', 'text')
+    .option('--verbose', 'Enable verbose logging', false)
+    .option('--batch-size <number>', 'Number of projects to process in parallel', '1')
+    .parse(process.argv)
+
+  const rawOptions = program.opts<Record<string, unknown>>()
+
+  // Map commander options to ScanOptions
+  const options: ScanOptions = {
+    projectsDir: (rawOptions.projectsDir as string) || process.env.PROJECTS_DIR || join(process.env.HOME || '~', 'projects'),
+    dryRun: rawOptions.dryRun === true,
+    skipExisting: rawOptions.skipExisting === true,
+    apiUrl: (rawOptions.apiUrl as string) || process.env.GRAPHQL_API_URL || 'http://localhost:4000/graphql',
+    email: (rawOptions.email as string) || process.env.GRAPHQL_EMAIL,
+    password: (rawOptions.password as string) || process.env.GRAPHQL_PASSWORD,
+    authToken: (rawOptions.authToken as string) || process.env.GRAPHQL_AUTH_TOKEN,
+    outputFormat: ((rawOptions.output as string) || 'text') as 'text' | 'json',
+    verbose: rawOptions.verbose === true,
+    batchSize: 1,
+  }
+
+  // Validate batch size
+  const batchSize = parseInt((rawOptions.batchSize as string) || '1', 10)
+  if (isNaN(batchSize) || batchSize < 1) {
+    console.error('Error: --batch-size must be a positive integer')
+    process.exit(1)
+  }
+  options.batchSize = batchSize
+
+  // Validate output format
+  if (options.outputFormat !== 'text' && options.outputFormat !== 'json') {
+    console.error('Error: --output must be "text" or "json"')
+    process.exit(1)
+  }
+
+  const scanner = new ProjectScanner(options)
+
+  try {
+    // Authenticate (skip in dry-run mode)
+    if (!options.dryRun) {
+      await scanner.authenticate()
+      // Load existing tenants for duplicate detection
+      await scanner.loadExistingTenants()
+    } else if (options.verbose) {
+      console.log('[DRY RUN] Skipping authentication')
+    }
+
+    // Scan projects
+    const projects = await scanner.scanAllProjects()
+
+    // Import projects
+    await scanner.importProjects(projects)
+  } catch (error) {
+    const errorMessage = error instanceof Error ? error.message : String(error)
+    console.error('Fatal error:', errorMessage)
+    if (options.verbose && error instanceof Error && error.stack) {
+      console.error(error.stack)
+    }
+    process.exit(1)
+  }
+}
+
+if (require.main === module) {
+  main()
+}
+
+export { ProjectScanner, ProjectInfo, ScanResult, ScanOptions }
+

scripts/setup-ceph-complete.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+# Complete Ceph setup script - run on each node
+set -e
+
+HOSTNAME=$(hostname -s)
+PUBLIC_NETWORK="192.168.11.0/24"
+CLUSTER_NETWORK="192.168.11.0/24"
+
+echo "=== Ceph Setup on $HOSTNAME ==="
+
+# Step 1: Install Ceph
+echo "Installing Ceph packages..."
+apt-get update
+apt-get install -y ceph ceph-common ceph-base
+
+# Step 2: Initialize cluster (only on ml110-01)
+if [ "$HOSTNAME" = "ml110-01" ]; then
+    if [ ! -f "/etc/pve/ceph.conf" ]; then
+        echo "Initializing Ceph cluster..."
+        pveceph init --network $PUBLIC_NETWORK --cluster-network $CLUSTER_NETWORK
+    else
+        echo "Ceph already initialized"
+    fi
+fi
+
+# Step 3: Create monitor
+echo "Creating monitor..."
+pveceph mon create 2>&1 || echo "Monitor may already exist"
+
+# Step 4: Start monitor
+echo "Starting monitor service..."
+systemctl enable ceph-mon@$HOSTNAME
+systemctl start ceph-mon@$HOSTNAME
+sleep 5
+
+# Step 5: Verify
+echo "Checking status..."
+systemctl status ceph-mon@$HOSTNAME --no-pager | head -15
+
+echo ""
+echo "=== Setup Complete ==="
+echo "Run 'ceph quorum_status' to verify quorum"
+echo "Run 'ceph -s' to check cluster status"
+

scripts/setup-ceph-osds-r630.sh

@@ -0,0 +1,272 @@
+#!/bin/bash
+# Setup Ceph OSDs on R630-01 available disks
+# Creates OSDs on sdc-sdh (6x 232.9G SSDs) and optionally sdb (279.4G)
+# Run on R630-01 as root
+
+set +e  # Don't exit on error - we want to continue even if some OSDs fail
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+CYAN='\033[0;36m'
+NC='\033[0m'
+
+# Configuration
+PRIMARY_DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")  # 6x SSDs
+SECONDARY_DRIVE="sdb"  # Optional 279.4G disk
+INCLUDE_SDB="${INCLUDE_SDB:-false}"  # Set to "true" to include sdb
+
+echo "=========================================="
+echo -e "${CYAN}Ceph OSD Setup for R630-01${NC}"
+echo "=========================================="
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+# Check if Ceph is installed
+echo -e "${BLUE}=== Step 1: Prerequisites Check ===${NC}"
+echo "-----------------------------------"
+
+if ! command -v ceph &>/dev/null; then
+    echo -e "${RED}ERROR: Ceph is not installed${NC}"
+    echo "Please install Ceph first"
+    exit 1
+fi
+echo -e "${GREEN}✓ Ceph is installed${NC}"
+
+if ! command -v ceph-volume &>/dev/null; then
+    echo -e "${RED}ERROR: ceph-volume is not installed${NC}"
+    exit 1
+fi
+echo -e "${GREEN}✓ ceph-volume is installed${NC}"
+
+# Check cluster connectivity
+echo ""
+echo "Checking Ceph cluster connectivity..."
+if ceph quorum_status &>/dev/null 2>&1; then
+    echo -e "${GREEN}✓ Ceph cluster is accessible${NC}"
+    CLUSTER_ACCESSIBLE=true
+else
+    echo -e "${YELLOW}⚠ Warning: Cannot access Ceph cluster${NC}"
+    echo "  This might be normal if this is the first OSD"
+    CLUSTER_ACCESSIBLE=false
+fi
+
+# Show current OSD status
+if [ "$CLUSTER_ACCESSIBLE" = true ]; then
+    echo ""
+    echo "Current OSD status:"
+    ceph osd tree 2>/dev/null || echo -e "${YELLOW}  Cannot get OSD tree${NC}"
+    echo ""
+    CURRENT_OSD_COUNT=$(ceph osd tree 2>/dev/null | grep -c "osd\." || echo "0")
+    echo "Current OSD count: $CURRENT_OSD_COUNT"
+fi
+echo ""
+
+# Verify disks are available
+echo -e "${BLUE}=== Step 2: Disk Verification ===${NC}"
+echo "-----------------------------------"
+
+available_disks=()
+unavailable_disks=()
+
+# Check primary drives (sdc-sdh)
+echo "Checking primary drives (sdc-sdh)..."
+for disk in "${PRIMARY_DRIVES[@]}"; do
+    if [ ! -b "/dev/$disk" ]; then
+        echo -e "  ${RED}✗ /dev/$disk not found${NC}"
+        unavailable_disks+=("$disk")
+        continue
+    fi
+    
+    # Check if already has OSD
+    if ceph-volume lvm list /dev/$disk 2>/dev/null | grep -q "osd\."; then
+        echo -e "  ${YELLOW}⚠ /dev/$disk already has an OSD, skipping${NC}"
+        unavailable_disks+=("$disk")
+        continue
+    fi
+    
+    # Check if has partitions or is mounted
+    partitions=$(lsblk -n /dev/$disk 2>/dev/null | grep -c "part" || echo "0")
+    partitions=$(echo "$partitions" | tr -d '\n\r ' | head -1)
+    mountpoint=$(lsblk -d -n -o MOUNTPOINT /dev/$disk 2>/dev/null | tr -d '\n' || echo "")
+    
+    if [ "${partitions:-0}" -eq 0 ] && [ -z "$mountpoint" ]; then
+        size=$(lsblk -d -n -o SIZE /dev/$disk 2>/dev/null || echo "unknown")
+        echo -e "  ${GREEN}✓ /dev/$disk ($size) - Available${NC}"
+        available_disks+=("$disk")
+    else
+        echo -e "  ${YELLOW}⚠ /dev/$disk has partitions or is mounted, skipping${NC}"
+        unavailable_disks+=("$disk")
+    fi
+done
+
+# Check secondary drive (sdb) if requested
+if [ "$INCLUDE_SDB" = "true" ]; then
+    echo ""
+    echo "Checking secondary drive (sdb)..."
+    if [ ! -b "/dev/$SECONDARY_DRIVE" ]; then
+        echo -e "  ${RED}✗ /dev/$SECONDARY_DRIVE not found${NC}"
+    elif ceph-volume lvm list /dev/$SECONDARY_DRIVE 2>/dev/null | grep -q "osd\."; then
+        echo -e "  ${YELLOW}⚠ /dev/$SECONDARY_DRIVE already has an OSD, skipping${NC}"
+    else
+        # Check if it has LVM signature (needs wiping)
+        if pvs 2>/dev/null | grep -q "/dev/$SECONDARY_DRIVE"; then
+            echo -e "  ${YELLOW}⚠ /dev/$SECONDARY_DRIVE has LVM signature (will be wiped)${NC}"
+        fi
+        size=$(lsblk -d -n -o SIZE /dev/$SECONDARY_DRIVE 2>/dev/null || echo "unknown")
+        echo -e "  ${GREEN}✓ /dev/$SECONDARY_DRIVE ($size) - Will be prepared${NC}"
+        available_disks+=("$SECONDARY_DRIVE")
+    fi
+fi
+
+echo ""
+echo "Summary:"
+echo "  Available disks: ${#available_disks[@]}"
+echo "  Unavailable/skipped: ${#unavailable_disks[@]}"
+
+if [ ${#available_disks[@]} -eq 0 ]; then
+    echo -e "${RED}ERROR: No available disks to create OSDs on${NC}"
+    exit 1
+fi
+
+echo ""
+read -p "Continue with OSD creation on ${#available_disks[@]} disk(s)? (yes/no): " confirm
+if [ "$confirm" != "yes" ]; then
+    echo "Aborted by user"
+    exit 0
+fi
+echo ""
+
+# Prepare disks
+echo -e "${BLUE}=== Step 3: Prepare Disks ===${NC}"
+echo "-----------------------------------"
+
+for disk in "${available_disks[@]}"; do
+    echo "Preparing /dev/$disk..."
+    
+    # Unmount any partitions
+    umount /dev/${disk}* 2>/dev/null || true
+    
+    # For sdb, remove LVM signature if present
+    if [ "$disk" = "$SECONDARY_DRIVE" ]; then
+        if pvs 2>/dev/null | grep -q "/dev/$disk"; then
+            echo "  Removing LVM signature..."
+            pvremove -f /dev/$disk 2>/dev/null || true
+        fi
+    fi
+    
+    # Wipe filesystem signatures (but keep partition table if needed)
+    echo "  Wiping filesystem signatures..."
+    wipefs -a /dev/$disk 2>/dev/null || true
+    
+    echo -e "  ${GREEN}✓ /dev/$disk prepared${NC}"
+done
+echo ""
+
+# Create OSDs
+echo -e "${BLUE}=== Step 4: Create Ceph OSDs ===${NC}"
+echo "-----------------------------------"
+
+osd_count=0
+failed_disks=()
+successful_osds=()
+
+for disk in "${available_disks[@]}"; do
+    echo -e "${CYAN}Creating OSD on /dev/$disk...${NC}"
+    
+    # Create OSD with logging
+    log_file="/tmp/ceph-osd-${disk}.log"
+    if ceph-volume lvm create --data /dev/$disk 2>&1 | tee "$log_file"; then
+        # Try to extract OSD ID
+        osd_id=$(grep -oP "osd\.\K\d+" "$log_file" 2>/dev/null | head -1)
+        if [ -n "$osd_id" ]; then
+            echo -e "${GREEN}  ✓ OSD $osd_id created on /dev/$disk${NC}"
+            successful_osds+=("osd.$osd_id")
+            
+            # Enable and start service
+            if systemctl enable ceph-osd@$osd_id 2>/dev/null; then
+                if systemctl start ceph-osd@$osd_id 2>/dev/null; then
+                    echo -e "${GREEN}  ✓ OSD $osd_id service started${NC}"
+                else
+                    echo -e "${YELLOW}  ⚠ OSD $osd_id service start had issues${NC}"
+                fi
+            fi
+        else
+            echo -e "${GREEN}  ✓ OSD created on /dev/$disk (ID not extracted)${NC}"
+        fi
+        osd_count=$((osd_count + 1))
+    else
+        echo -e "${RED}  ✗ Failed to create OSD on /dev/$disk${NC}"
+        failed_disks+=("$disk")
+        echo "  Check $log_file for details"
+    fi
+    echo ""
+done
+
+# Summary
+echo "=========================================="
+echo -e "${CYAN}OSD Creation Summary${NC}"
+echo "=========================================="
+echo ""
+echo "Successfully created: $osd_count OSD(s)"
+echo "Successful OSDs: ${successful_osds[@]}"
+
+if [ ${#failed_disks[@]} -gt 0 ]; then
+    echo -e "${YELLOW}Failed disks: ${failed_disks[@]}${NC}"
+fi
+echo ""
+
+# Verify Ceph status
+echo -e "${BLUE}=== Step 5: Verify Ceph Status ===${NC}"
+echo "-----------------------------------"
+
+if [ "$CLUSTER_ACCESSIBLE" = true ] || ceph quorum_status &>/dev/null 2>&1; then
+    echo "Ceph Health:"
+    ceph health 2>/dev/null || echo -e "${YELLOW}Cannot get health status${NC}"
+    echo ""
+    
+    echo "OSD Tree:"
+    ceph osd tree 2>/dev/null || echo -e "${YELLOW}Cannot get OSD tree${NC}"
+    echo ""
+    
+    echo "OSD Details:"
+    ceph osd df 2>/dev/null || echo -e "${YELLOW}Cannot get OSD details${NC}"
+    echo ""
+    
+    NEW_OSD_COUNT=$(ceph osd tree 2>/dev/null | grep -c "osd\." || echo "0")
+    echo "Total OSD count: $NEW_OSD_COUNT"
+    
+    if [ "$NEW_OSD_COUNT" -ge 3 ]; then
+        echo -e "${GREEN}  ✓ SUCCESS: Have 3+ OSDs (should resolve TOO_FEW_OSDS if present)${NC}"
+    elif [ "$NEW_OSD_COUNT" -ge 1 ]; then
+        echo -e "${YELLOW}  ⚠ Have $NEW_OSD_COUNT OSD(s). Need 3+ for production use.${NC}"
+    fi
+else
+    echo -e "${YELLOW}Ceph cluster not accessible. OSDs may need time to join cluster.${NC}"
+fi
+echo ""
+
+# Check OSD services
+echo -e "${BLUE}=== Step 6: OSD Service Status ===${NC}"
+echo "-----------------------------------"
+systemctl list-units --type=service | grep ceph-osd | head -10 || echo "No OSD services found"
+echo ""
+
+echo "=========================================="
+echo -e "${GREEN}Setup Complete!${NC}"
+echo "=========================================="
+echo ""
+echo "Next steps:"
+echo "1. Monitor Ceph health: ceph health"
+echo "2. Check OSD status: ceph osd tree"
+echo "3. Verify OSD services: systemctl status ceph-osd@<id>"
+echo "4. If needed, create more OSDs on other nodes"
+echo ""
+

scripts/ssh-ml110-01.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# SSH to ML110-01 using password from .env
+
+set -e
+
+# Get the directory where this script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Get the project root (one level up from scripts directory)
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+# Locate .env file in project root
+ENV_FILE="$PROJECT_ROOT/.env"
+
+# Verify .env file exists
+if [[ ! -f "$ENV_FILE" ]]; then
+    echo "Error: .env file not found at $ENV_FILE"
+    echo "Current directory: $(pwd)"
+    echo "Script directory: $SCRIPT_DIR"
+    echo "Project root: $PROJECT_ROOT"
+    exit 1
+fi
+
+if [[ -f "$ENV_FILE" ]]; then
+    # Extract password from .env (lines 45-46)
+    PROXMOX_PASSWORD=$(sed -n '45,46p' "$ENV_FILE" | grep "PROXMOX_ROOT_PASS" | head -1 | cut -d'=' -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+    # Remove quotes if present
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD#\"}"
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD%\"}"
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD#\'}"
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD%\'}"
+else
+    echo "Error: .env file not found"
+    exit 1
+fi
+
+if ! command -v sshpass &> /dev/null; then
+    echo "Error: sshpass not installed. Install with: sudo apt-get install sshpass"
+    exit 1
+fi
+
+echo "Connecting to ML110-01 (192.168.11.10)..."
+sshpass -p "$PROXMOX_PASSWORD" ssh -o StrictHostKeyChecking=no root@192.168.11.10 "$@"
+

scripts/ssh-proxmox-nodes.sh

@@ -0,0 +1,146 @@
+#!/bin/bash
+# SSH Commands for Proxmox Nodes
+# Provides easy access to both Proxmox machines
+
+set -e
+
+# Load password from .env file
+# Get the directory where this script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Get the project root (one level up from scripts directory)
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+# Locate .env file in project root
+ENV_FILE="$PROJECT_ROOT/.env"
+
+if [[ -f "$ENV_FILE" ]]; then
+    # Source the password from .env (lines 45-46)
+    export PROXMOX_PASSWORD=$(sed -n '45,46p' "$ENV_FILE" | grep -E "PROXMOX_ROOT_PASS|^[^#]" | head -1 | cut -d'=' -f2 | tr -d '"' | tr -d "'")
+else
+    echo "Warning: .env file not found. Password will need to be entered manually."
+    PROXMOX_PASSWORD=""
+fi
+
+# Colors
+GREEN='\033[0;32m'
+BLUE='\033[0;34m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+# Configuration
+ML110_01_IP="192.168.11.10"
+R630_01_IP="192.168.11.11"
+ML110_01_HOSTNAME="ml110-01"
+R630_01_HOSTNAME="r630-01"
+DEFAULT_USER="root"
+
+echo "=========================================="
+echo "Proxmox Node SSH Access"
+echo "=========================================="
+echo ""
+
+# Function to display SSH command
+show_ssh_command() {
+    local node_name=$1
+    local ip=$2
+    local hostname=$3
+    local user=$4
+    
+    echo -e "${BLUE}${node_name}${NC}"
+    echo "  IP Address: $ip"
+    echo "  Hostname: $hostname"
+    echo "  User: $user"
+    echo ""
+    echo "  SSH Command:"
+    echo -e "  ${GREEN}ssh ${user}@${ip}${NC}"
+    echo ""
+    if [[ -n "$hostname" ]]; then
+        echo "  Or via hostname (if DNS configured):"
+        echo -e "  ${GREEN}ssh ${user}@${hostname}${NC}"
+    fi
+    echo ""
+}
+
+# Display commands for both nodes
+show_ssh_command "Site-1 (ML110-01)" "$ML110_01_IP" "$ML110_01_HOSTNAME" "$DEFAULT_USER"
+show_ssh_command "Site-2 (R630-01)" "$R630_01_IP" "$R630_01_HOSTNAME" "$DEFAULT_USER"
+
+echo "=========================================="
+echo "Quick Access Commands"
+echo "=========================================="
+echo ""
+echo "SSH to ML110-01:"
+if command -v sshpass &> /dev/null && [[ -n "$PROXMOX_PASSWORD" ]]; then
+    echo -e "  ${GREEN}sshpass -p '***' ssh root@${ML110_01_IP}${NC}"
+    echo "  (Password loaded from .env)"
+else
+    echo -e "  ${GREEN}ssh root@${ML110_01_IP}${NC}"
+fi
+echo ""
+echo "SSH to R630-01:"
+if command -v sshpass &> /dev/null && [[ -n "$PROXMOX_PASSWORD" ]]; then
+    echo -e "  ${GREEN}sshpass -p '***' ssh root@${R630_01_IP}${NC}"
+    echo "  (Password loaded from .env)"
+else
+    echo -e "  ${GREEN}ssh root@${R630_01_IP}${NC}"
+fi
+echo ""
+
+echo "=========================================="
+echo "Verification Commands"
+echo "=========================================="
+echo ""
+echo "Test connectivity to ML110-01:"
+echo -e "  ${GREEN}ping -c 3 ${ML110_01_IP}${NC}"
+echo ""
+echo "Test connectivity to R630-01:"
+echo -e "  ${GREEN}ping -c 3 ${R630_01_IP}${NC}"
+echo ""
+echo "Test SSH to ML110-01:"
+if command -v sshpass &> /dev/null && [[ -n "$PROXMOX_PASSWORD" ]]; then
+    echo -e "  ${GREEN}sshpass -p '***' ssh -o ConnectTimeout=5 root@${ML110_01_IP} 'hostname'${NC}"
+else
+    echo -e "  ${GREEN}ssh -o ConnectTimeout=5 root@${ML110_01_IP} 'hostname'${NC}"
+fi
+echo ""
+echo "Test SSH to R630-01:"
+if command -v sshpass &> /dev/null && [[ -n "$PROXMOX_PASSWORD" ]]; then
+    echo -e "  ${GREEN}sshpass -p '***' ssh -o ConnectTimeout=5 root@${R630_01_IP} 'hostname'${NC}"
+else
+    echo -e "  ${GREEN}ssh -o ConnectTimeout=5 root@${R630_01_IP} 'hostname'${NC}"
+fi
+echo ""
+
+echo "=========================================="
+echo "Proxmox API Verification"
+echo "=========================================="
+echo ""
+echo "Check Proxmox API on ML110-01:"
+echo -e "  ${GREEN}curl -k https://${ML110_01_IP}:8006/api2/json/version${NC}"
+echo ""
+echo "Check Proxmox API on R630-01:"
+echo -e "  ${GREEN}curl -k https://${R630_01_IP}:8006/api2/json/version${NC}"
+echo ""
+
+echo "=========================================="
+echo "Useful Commands After SSH"
+echo "=========================================="
+echo ""
+echo "Check Proxmox status:"
+echo "  pveversion"
+echo "  systemctl status pve-cluster"
+echo ""
+echo "List VMs:"
+echo "  qm list"
+echo ""
+echo "Check storage:"
+echo "  pvesm status"
+echo ""
+echo "Check network:"
+echo "  ip addr show"
+echo "  cat /etc/network/interfaces"
+echo ""
+echo "Check Ceph (if configured):"
+echo "  ceph status"
+echo "  ceph df"
+echo ""
+

scripts/ssh-r630-01.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# SSH to R630-01 using password from .env
+
+set -e
+
+# Get the directory where this script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Get the project root (one level up from scripts directory)
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+# Locate .env file in project root
+ENV_FILE="$PROJECT_ROOT/.env"
+
+# Verify .env file exists
+if [[ ! -f "$ENV_FILE" ]]; then
+    echo "Error: .env file not found at $ENV_FILE"
+    echo "Current directory: $(pwd)"
+    echo "Script directory: $SCRIPT_DIR"
+    echo "Project root: $PROJECT_ROOT"
+    exit 1
+fi
+
+if [[ -f "$ENV_FILE" ]]; then
+    # Extract password from .env (lines 45-46)
+    PROXMOX_PASSWORD=$(sed -n '45,46p' "$ENV_FILE" | grep "PROXMOX_ROOT_PASS" | head -1 | cut -d'=' -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+    # Remove quotes if present
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD#\"}"
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD%\"}"
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD#\'}"
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD%\'}"
+else
+    echo "Error: .env file not found"
+    exit 1
+fi
+
+if ! command -v sshpass &> /dev/null; then
+    echo "Error: sshpass not installed. Install with: sudo apt-get install sshpass"
+    exit 1
+fi
+
+echo "Connecting to R630-01 (192.168.11.11)..."
+sshpass -p "$PROXMOX_PASSWORD" ssh -o StrictHostKeyChecking=no root@192.168.11.11 "$@"
+

scripts/test-scan-projects.sh

@@ -0,0 +1,155 @@
+#!/bin/bash
+# Integration test script for scan-projects.ts
+# Tests the script with a mock projects directory structure
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+TEST_DIR="/tmp/scan-projects-test-$$"
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+cleanup() {
+    echo -e "${YELLOW}Cleaning up test directory: $TEST_DIR${NC}"
+    rm -rf "$TEST_DIR"
+}
+
+trap cleanup EXIT
+
+echo -e "${GREEN}Setting up test environment...${NC}"
+
+# Create test directory structure
+mkdir -p "$TEST_DIR"
+
+# Create test projects
+mkdir -p "$TEST_DIR/git-repo"
+cd "$TEST_DIR/git-repo"
+git init --quiet
+git remote add origin "https://github.com/testuser/git-repo.git" 2>/dev/null || true
+echo "# Test Project" > README.md
+git add README.md
+git commit -m "Initial commit" --quiet 2>/dev/null || true
+
+mkdir -p "$TEST_DIR/monorepo"
+cd "$TEST_DIR/monorepo"
+cat > package.json <<EOF
+{
+  "name": "monorepo",
+  "version": "1.0.0",
+  "workspaces": ["packages/*"]
+}
+EOF
+mkdir -p packages/pkg1 packages/pkg2
+echo '{"name": "pkg1"}' > packages/pkg1/package.json
+echo '{"name": "pkg2"}' > packages/pkg2/package.json
+echo 'packages:' > pnpm-workspace.yaml
+
+mkdir -p "$TEST_DIR/npm-project"
+cd "$TEST_DIR/npm-project"
+cat > package.json <<EOF
+{
+  "name": "npm-project",
+  "version": "1.0.0"
+}
+EOF
+echo '{}' > package-lock.json
+
+mkdir -p "$TEST_DIR/yarn-project"
+cd "$TEST_DIR/yarn-project"
+cat > package.json <<EOF
+{
+  "name": "yarn-project",
+  "version": "1.0.0"
+}
+EOF
+echo '{}' > yarn.lock
+
+mkdir -p "$TEST_DIR/invalid-name project"
+cd "$TEST_DIR/invalid-name project"
+cat > package.json <<EOF
+{
+  "name": "invalid-name project",
+  "version": "1.0.0"
+}
+EOF
+
+echo -e "${GREEN}Test environment created at: $TEST_DIR${NC}"
+echo ""
+echo "Test projects:"
+echo "  - git-repo (Git repository)"
+echo "  - monorepo (Monorepo with workspaces)"
+echo "  - npm-project (npm project)"
+echo "  - yarn-project (yarn project)"
+echo "  - invalid-name project (invalid name for testing)"
+echo ""
+
+# Check if API is available
+if [ -z "${GRAPHQL_API_URL:-}" ]; then
+    GRAPHQL_API_URL="http://localhost:4000/graphql"
+fi
+
+echo -e "${YELLOW}Testing with API: $GRAPHQL_API_URL${NC}"
+echo ""
+
+# Test 1: Dry run
+echo -e "${GREEN}Test 1: Dry run${NC}"
+cd "$PROJECT_ROOT"
+if pnpm tsx scripts/scan-projects.ts \
+    --dry-run \
+    --projects-dir "$TEST_DIR" \
+    --api-url "$GRAPHQL_API_URL" \
+    --verbose 2>&1 | grep -q "DRY RUN"; then
+    echo -e "${GREEN}✓ Dry run test passed${NC}"
+else
+    echo -e "${RED}✗ Dry run test failed${NC}"
+    exit 1
+fi
+
+echo ""
+
+# Test 2: Validation (should fail on invalid project name)
+echo -e "${GREEN}Test 2: Validation${NC}"
+if pnpm tsx scripts/scan-projects.ts \
+    --dry-run \
+    --projects-dir "$TEST_DIR" \
+    --api-url "$GRAPHQL_API_URL" \
+    --verbose 2>&1 | grep -q "invalid-name project"; then
+    echo -e "${GREEN}✓ Validation test passed (invalid name detected)${NC}"
+else
+    echo -e "${YELLOW}⚠ Validation test: Invalid name may have been sanitized${NC}"
+fi
+
+echo ""
+
+# Test 3: JSON output
+echo -e "${GREEN}Test 3: JSON output${NC}"
+if pnpm tsx scripts/scan-projects.ts \
+    --dry-run \
+    --projects-dir "$TEST_DIR" \
+    --api-url "$GRAPHQL_API_URL" \
+    --output json 2>&1 | grep -q '"created"'; then
+    echo -e "${GREEN}✓ JSON output test passed${NC}"
+else
+    echo -e "${RED}✗ JSON output test failed${NC}"
+    exit 1
+fi
+
+echo ""
+
+# Test 4: Help
+echo -e "${GREEN}Test 4: Help command${NC}"
+if pnpm tsx scripts/scan-projects.ts --help 2>&1 | grep -q "Scan projects directory"; then
+    echo -e "${GREEN}✓ Help command test passed${NC}"
+else
+    echo -e "${RED}✗ Help command test failed${NC}"
+    exit 1
+fi
+
+echo ""
+echo -e "${GREEN}All integration tests passed!${NC}"
+

scripts/test-ssh-password.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Test SSH password from .env file
+
+set -e
+
+# Get the directory where this script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Get the project root (one level up from scripts directory)
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+# Locate .env file in project root
+ENV_FILE="$PROJECT_ROOT/.env"
+
+# Verify .env file exists
+if [[ ! -f "$ENV_FILE" ]]; then
+    echo "Error: .env file not found at $ENV_FILE"
+    echo "Current directory: $(pwd)"
+    echo "Script directory: $SCRIPT_DIR"
+    echo "Project root: $PROJECT_ROOT"
+    exit 1
+fi
+
+if [[ -f "$ENV_FILE" ]]; then
+    # Extract password from .env (lines 45-46)
+    PROXMOX_PASSWORD=$(sed -n '45,46p' "$ENV_FILE" | grep "PROXMOX_ROOT_PASS" | head -1 | cut -d'=' -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+    # Remove quotes if present
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD#\"}"
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD%\"}"
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD#\'}"
+    PROXMOX_PASSWORD="${PROXMOX_PASSWORD%\'}"
+    
+    echo "Password extracted from .env:"
+    echo "  Length: ${#PROXMOX_PASSWORD} characters"
+    echo "  First 3 chars: ${PROXMOX_PASSWORD:0:3}..."
+    echo ""
+    
+    if ! command -v sshpass &> /dev/null; then
+        echo "Error: sshpass not installed"
+        exit 1
+    fi
+    
+    echo "Testing ML110-01 (192.168.11.10)..."
+    if sshpass -p "$PROXMOX_PASSWORD" ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@192.168.11.10 'hostname && pveversion' 2>&1; then
+        echo "✅ ML110-01: Connection successful!"
+    else
+        echo "❌ ML110-01: Connection failed"
+        echo ""
+        echo "Note: If password is incorrect, update .env file line 45-46"
+    fi
+    
+    echo ""
+    echo "Testing R630-01 (192.168.11.11)..."
+    if sshpass -p "$PROXMOX_PASSWORD" ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no root@192.168.11.11 'hostname && pveversion' 2>&1; then
+        echo "✅ R630-01: Connection successful!"
+    else
+        echo "❌ R630-01: Connection failed"
+        echo ""
+        echo "Note: If password is incorrect, update .env file line 45-46"
+    fi
+else
+    echo "Error: .env file not found at $ENV_FILE"
+    exit 1
+fi
+

scripts/update-all-vm-templates.sh

@@ -0,0 +1,160 @@
+#!/bin/bash
+# Update all VM template files to match current Kubernetes configurations
+
+set -euo pipefail
+
+TEMPLATE_DIR="examples/production"
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  UPDATE ALL VM TEMPLATE FILES"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+# Function to create/update a VM template file
+update_vm_template() {
+    local vm_name=$1
+    local namespace=$2
+    local node=$3
+    local cpu=$4
+    local memory=$5
+    local disk=$6
+    local storage=$7
+    local image=$8
+    local network=$9
+    local site=${10}
+    local file_path=$11
+    
+    # Ensure directory exists
+    mkdir -p "$(dirname "$file_path")"
+    
+    # Create/update the template file
+    cat > "$file_path" <<EOF
+apiVersion: proxmox.sankofa.nexus/v1alpha1
+kind: ProxmoxVM
+metadata:
+  name: $vm_name
+  namespace: $namespace
+spec:
+  forProvider:
+    node: $node
+    name: $vm_name
+    cpu: $cpu
+    memory: $memory
+    disk: $disk
+    storage: $storage
+    network: $network
+    image: $image
+    site: $site
+EOF
+    
+    echo "  ✅ Updated: $file_path"
+}
+
+# Get all VM configurations from Kubernetes
+echo "Step 1: Fetching VM configurations from Kubernetes..."
+VM_CONFIGS=$(kubectl get proxmoxvms -A -o json 2>/dev/null | jq -r '.items[] | 
+    select(.spec.forProvider.node == "r630-01" or .spec.forProvider.node == "ml110-01") | 
+    "\(.metadata.namespace)|\(.metadata.name)|\(.spec.forProvider.node)|\(.spec.forProvider.cpu)|\(.spec.forProvider.memory)|\(.spec.forProvider.disk)|\(.spec.forProvider.storage)|\(.spec.forProvider.image)|\(.spec.forProvider.network)|\(.spec.forProvider.site)"' | sort)
+
+echo "Found $(echo "$VM_CONFIGS" | wc -l) VMs to update"
+echo ""
+
+# Update SMOM-DBIS-138 VMs
+echo "Step 2: Updating SMOM-DBIS-138 VM templates..."
+while IFS='|' read -r namespace name node cpu memory disk storage image network site; do
+    if [[ "$name" =~ ^(smom-validator|smom-sentry|rpc-node|smom-services|smom-blockscout|monitoring|management)$ ]]; then
+        # Map Kubernetes names to template file names
+        case "$name" in
+            smom-validator-*)
+                file_name="validator-${name##smom-validator-}.yaml"
+                ;;
+            smom-sentry-*)
+                file_name="sentry-${name##smom-sentry-}.yaml"
+                ;;
+            rpc-node-*)
+                file_name="rpc-node-${name##rpc-node-}.yaml"
+                ;;
+            smom-services)
+                file_name="services.yaml"
+                ;;
+            smom-blockscout)
+                file_name="blockscout.yaml"
+                ;;
+            monitoring)
+                file_name="monitoring.yaml"
+                ;;
+            management)
+                file_name="management.yaml"
+                ;;
+            *)
+                continue
+                ;;
+        esac
+        
+        file_path="$TEMPLATE_DIR/smom-dbis-138/$file_name"
+        update_vm_template "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site" "$file_path"
+    fi
+done <<< "$VM_CONFIGS"
+echo ""
+
+# Update Phoenix VMs
+echo "Step 3: Updating Phoenix VM templates..."
+while IFS='|' read -r namespace name node cpu memory disk storage image network site; do
+    if [[ "$name" =~ ^phoenix- ]]; then
+        # Map Kubernetes names to template file names
+        file_name="${name##phoenix-}.yaml"
+        file_path="$TEMPLATE_DIR/phoenix/$file_name"
+        update_vm_template "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site" "$file_path"
+    fi
+done <<< "$VM_CONFIGS"
+echo ""
+
+# Update other production VMs
+echo "Step 4: Updating other production VM templates..."
+while IFS='|' read -r namespace name node cpu memory disk storage image network site; do
+    # Skip SMOM and Phoenix VMs (already handled)
+    if [[ "$name" =~ ^(smom-|phoenix-|rpc-node|monitoring|management)$ ]]; then
+        continue
+    fi
+    
+    # Skip example/template VMs that aren't meant for production
+    if [[ "$name" =~ ^(basic-vm-001|large-vm-001|medium-vm-001|vm-100)$ ]]; then
+        continue
+    fi
+    
+    # Map to template file names
+    case "$name" in
+        nginx-proxy-vm)
+            file_name="nginx-proxy-vm.yaml"
+            ;;
+        cloudflare-tunnel-vm)
+            file_name="cloudflare-tunnel-vm.yaml"
+            ;;
+        *)
+            # Use name as-is for other VMs
+            file_name="${name}.yaml"
+            ;;
+    esac
+    
+    file_path="$TEMPLATE_DIR/$file_name"
+    update_vm_template "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site" "$file_path"
+done <<< "$VM_CONFIGS"
+echo ""
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  SUMMARY"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+echo "✅ All VM template files updated!"
+echo ""
+echo "📝 Updated templates:"
+echo "  - SMOM-DBIS-138 VMs: $(find $TEMPLATE_DIR/smom-dbis-138 -name "*.yaml" | wc -l) files"
+echo "  - Phoenix VMs: $(find $TEMPLATE_DIR/phoenix -name "*.yaml" | wc -l) files"
+echo "  - Other production VMs: $(find $TEMPLATE_DIR -maxdepth 1 -name "*.yaml" | wc -l) files"
+echo ""
+echo "All templates now use:"
+echo "  - Storage: local-lvm"
+echo "  - Image: 9000 (template) or local:iso/ubuntu-22.04-cloud.img"
+echo "  - Network: vmbr0"
+echo "  - Current CPU, RAM, and Disk specifications"
+

scripts/update-smom-vm-specs.sh

@@ -0,0 +1,126 @@
+#!/bin/bash
+# Update SMOM-DBIS-138 VMs to match specifications
+
+set -euo pipefail
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  UPDATE SMOM-DBIS-138 VM SPECIFICATIONS"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+# Update Validators: 2 CPU → 6 CPU each (24 total)
+echo "Step 1: Updating Validators (CPU: 2 → 6)..."
+for i in 01 02 03 04; do
+    kubectl patch proxmoxvm -n default smom-validator-$i --type='json' \
+        -p='[{"op": "replace", "path": "/spec/forProvider/cpu", "value": 6}]' 2>/dev/null && \
+        echo "  ✅ smom-validator-$i: CPU updated to 6" || \
+        echo "  ❌ smom-validator-$i: Failed"
+done
+echo ""
+
+# Update Sentries: 2 CPU → 4, 4 GiB RAM → 8, 20 GiB disk → 15
+echo "Step 2: Updating Sentries (CPU: 2→4, RAM: 4→8, Disk: 20→15)..."
+for i in 01 02 03 04; do
+    kubectl patch proxmoxvm -n default smom-sentry-$i --type='json' \
+        -p='[
+            {"op": "replace", "path": "/spec/forProvider/cpu", "value": 4},
+            {"op": "replace", "path": "/spec/forProvider/memory", "value": "8Gi"},
+            {"op": "replace", "path": "/spec/forProvider/disk", "value": "15Gi"}
+        ]' 2>/dev/null && \
+        echo "  ✅ smom-sentry-$i: CPU=4, RAM=8Gi, Disk=15Gi" || \
+        echo "  ❌ smom-sentry-$i: Failed"
+done
+echo ""
+
+# Update RPC Nodes: 2 CPU → 4, 4 GiB RAM → 8, 20 GiB disk → 10
+echo "Step 3: Updating RPC Nodes (CPU: 2→4, RAM: 4→8, Disk: 20→10)..."
+for i in 01 02 03 04; do
+    kubectl patch proxmoxvm -n default rpc-node-$i --type='json' \
+        -p='[
+            {"op": "replace", "path": "/spec/forProvider/cpu", "value": 4},
+            {"op": "replace", "path": "/spec/forProvider/memory", "value": "8Gi"},
+            {"op": "replace", "path": "/spec/forProvider/disk", "value": "10Gi"}
+        ]' 2>/dev/null && \
+        echo "  ✅ rpc-node-$i: CPU=4, RAM=8Gi, Disk=10Gi" || \
+        echo "  ❌ rpc-node-$i: Failed"
+done
+echo ""
+
+# Update Services: 2 CPU → 4, 4 GiB RAM → 8, 20 GiB disk → 35
+echo "Step 4: Updating Services (CPU: 2→4, RAM: 4→8, Disk: 20→35)..."
+kubectl patch proxmoxvm -n default smom-services --type='json' \
+    -p='[
+        {"op": "replace", "path": "/spec/forProvider/cpu", "value": 4},
+        {"op": "replace", "path": "/spec/forProvider/memory", "value": "8Gi"},
+        {"op": "replace", "path": "/spec/forProvider/disk", "value": "35Gi"}
+    ]' 2>/dev/null && \
+    echo "  ✅ smom-services: CPU=4, RAM=8Gi, Disk=35Gi" || \
+    echo "  ❌ smom-services: Failed"
+echo ""
+
+# Update Blockscout: 2 CPU → 4, 4 GiB RAM → 8, 20 GiB disk → 12
+echo "Step 5: Updating Blockscout (CPU: 2→4, RAM: 4→8, Disk: 20→12)..."
+kubectl patch proxmoxvm -n default smom-blockscout --type='json' \
+    -p='[
+        {"op": "replace", "path": "/spec/forProvider/cpu", "value": 4},
+        {"op": "replace", "path": "/spec/forProvider/memory", "value": "8Gi"},
+        {"op": "replace", "path": "/spec/forProvider/disk", "value": "12Gi"}
+    ]' 2>/dev/null && \
+    echo "  ✅ smom-blockscout: CPU=4, RAM=8Gi, Disk=12Gi" || \
+    echo "  ❌ smom-blockscout: Failed"
+echo ""
+
+# Update Monitoring: 2 CPU → 4, 4 GiB RAM → 8, 20 GiB disk → 9
+echo "Step 6: Updating Monitoring (CPU: 2→4, RAM: 4→8, Disk: 20→9)..."
+kubectl patch proxmoxvm -n default monitoring --type='json' \
+    -p='[
+        {"op": "replace", "path": "/spec/forProvider/cpu", "value": 4},
+        {"op": "replace", "path": "/spec/forProvider/memory", "value": "8Gi"},
+        {"op": "replace", "path": "/spec/forProvider/disk", "value": "9Gi"}
+    ]' 2>/dev/null && \
+    echo "  ✅ monitoring: CPU=4, RAM=8Gi, Disk=9Gi" || \
+    echo "  ❌ monitoring: Failed"
+echo ""
+
+# Update Management: 20 GiB disk → 2
+echo "Step 7: Updating Management (Disk: 20→2)..."
+kubectl patch proxmoxvm -n default management --type='json' \
+    -p='[{"op": "replace", "path": "/spec/forProvider/disk", "value": "2Gi"}]' 2>/dev/null && \
+    echo "  ✅ management: Disk=2Gi" || \
+    echo "  ❌ management: Failed"
+echo ""
+
+# Update all images to use template 9000
+echo "Step 8: Updating all VMs to use template 9000..."
+for vm in smom-validator-01 smom-validator-02 smom-validator-03 smom-validator-04 \
+          smom-sentry-01 smom-sentry-02 smom-sentry-03 smom-sentry-04 \
+          rpc-node-01 rpc-node-02 rpc-node-03 rpc-node-04 \
+          smom-services smom-blockscout monitoring; do
+    kubectl patch proxmoxvm -n default $vm --type='json' \
+        -p='[{"op": "replace", "path": "/spec/forProvider/image", "value": "9000"}]' 2>/dev/null && \
+        echo "  ✅ $vm: image=9000" || \
+        echo "  ❌ $vm: Failed"
+done
+echo ""
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  VERIFICATION"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+echo "Calculating totals..."
+VALIDATORS_CPU=$(kubectl get proxmoxvms -n default -o json 2>/dev/null | jq -r '[.items[] | select(.metadata.name | startswith("smom-validator")) | .spec.forProvider.cpu] | add // 0')
+SENTRIES_CPU=$(kubectl get proxmoxvms -n default -o json 2>/dev/null | jq -r '[.items[] | select(.metadata.name | startswith("smom-sentry")) | .spec.forProvider.cpu] | add // 0')
+RPC_CPU=$(kubectl get proxmoxvms -n default -o json 2>/dev/null | jq -r '[.items[] | select(.metadata.name | startswith("rpc-node")) | .spec.forProvider.cpu] | add // 0')
+SERVICES_CPU=$(kubectl get proxmoxvm -n default smom-services -o jsonpath='{.spec.forProvider.cpu}' 2>/dev/null || echo "0")
+BLOCKSCOUT_CPU=$(kubectl get proxmoxvm -n default smom-blockscout -o jsonpath='{.spec.forProvider.cpu}' 2>/dev/null || echo "0")
+MONITORING_CPU=$(kubectl get proxmoxvm -n default monitoring -o jsonpath='{.spec.forProvider.cpu}' 2>/dev/null || echo "0")
+MANAGEMENT_CPU=$(kubectl get proxmoxvm -n default management -o jsonpath='{.spec.forProvider.cpu}' 2>/dev/null || echo "0")
+
+TOTAL_CPU=$((VALIDATORS_CPU + SENTRIES_CPU + RPC_CPU + SERVICES_CPU + BLOCKSCOUT_CPU + MONITORING_CPU + MANAGEMENT_CPU))
+
+echo "Total CPU: $TOTAL_CPU (Expected: 68)"
+echo ""
+echo "✅ Update complete! All VMs configured to match specifications."
+echo ""
+echo "📝 Next: Monitor deployment with: kubectl get proxmoxvms -A -w"
+

scripts/update-vm-template-files.sh

@@ -0,0 +1,227 @@
+#!/bin/bash
+# Update VM template files with current Kubernetes configurations
+# Preserves metadata, labels, and userData sections
+
+set -euo pipefail
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  UPDATE VM TEMPLATE FILES"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+# Function to update a VM template file
+update_template_file() {
+    local file_path=$1
+    local vm_name=$2
+    local namespace=$3
+    local node=$4
+    local cpu=$5
+    local memory=$6
+    local disk=$7
+    local storage=$8
+    local image=$9
+    local network=${10}
+    local site=${11}
+    
+    # Read existing file to preserve metadata and userData
+    if [ -f "$file_path" ]; then
+        # Extract metadata section (including labels)
+        metadata_section=$(awk '/^metadata:/,/^spec:/ {print}' "$file_path" | head -n -1)
+        # Extract userData section if it exists
+        userdata_section=$(awk '/userData:/,/providerConfigRef:/ {print}' "$file_path" | head -n -1 || echo "")
+        # Extract providerConfigRef if it exists
+        provider_config=$(grep -A 2 "providerConfigRef:" "$file_path" || echo "")
+    else
+        metadata_section="metadata:
+  name: $vm_name
+  namespace: $namespace"
+        userdata_section=""
+        provider_config="  providerConfigRef:
+    name: proxmox-provider-config"
+    fi
+    
+    # Create updated file
+    {
+        echo "apiVersion: proxmox.sankofa.nexus/v1alpha1"
+        echo "kind: ProxmoxVM"
+        echo "$metadata_section"
+        echo "spec:"
+        echo "  forProvider:"
+        echo "    node: \"$node\""
+        echo "    name: \"$vm_name\""
+        echo "    cpu: $cpu"
+        echo "    memory: \"$memory\""
+        echo "    disk: \"$disk\""
+        echo "    storage: \"$storage\""
+        echo "    network: \"$network\""
+        echo "    image: \"$image\""
+        echo "    site: \"$site\""
+        if [ -n "$userdata_section" ]; then
+            echo "$userdata_section"
+        fi
+        if [ -n "$provider_config" ]; then
+            echo "$provider_config"
+        fi
+    } > "$file_path.tmp" && mv "$file_path.tmp" "$file_path"
+    
+    echo "  ✅ Updated: $file_path"
+}
+
+# Get VM configurations from Kubernetes
+echo "Step 1: Fetching VM configurations from Kubernetes..."
+VM_CONFIGS=$(kubectl get proxmoxvms -A -o json 2>/dev/null | jq -r '.items[] | 
+    select(.spec.forProvider.node == "r630-01" or .spec.forProvider.node == "ml110-01") | 
+    "\(.metadata.namespace)|\(.metadata.name)|\(.spec.forProvider.node)|\(.spec.forProvider.cpu)|\(.spec.forProvider.memory)|\(.spec.forProvider.disk)|\(.spec.forProvider.storage)|\(.spec.forProvider.image)|\(.spec.forProvider.network)|\(.spec.forProvider.site)"' | sort)
+
+UPDATED_COUNT=0
+
+# Update SMOM-DBIS-138 templates
+echo ""
+echo "Step 2: Updating SMOM-DBIS-138 VM templates..."
+while IFS='|' read -r namespace name node cpu memory disk storage image network site; do
+    case "$name" in
+        smom-validator-01)
+            update_template_file "examples/production/smom-dbis-138/validator-01.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        smom-validator-02)
+            update_template_file "examples/production/smom-dbis-138/validator-02.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        smom-validator-03)
+            update_template_file "examples/production/smom-dbis-138/validator-03.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        smom-validator-04)
+            update_template_file "examples/production/smom-dbis-138/validator-04.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        smom-sentry-01)
+            update_template_file "examples/production/smom-dbis-138/sentry-01.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        smom-sentry-02)
+            update_template_file "examples/production/smom-dbis-138/sentry-02.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        smom-sentry-03)
+            update_template_file "examples/production/smom-dbis-138/sentry-03.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        smom-sentry-04)
+            update_template_file "examples/production/smom-dbis-138/sentry-04.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        rpc-node-01)
+            update_template_file "examples/production/smom-dbis-138/rpc-node-01.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        rpc-node-02)
+            update_template_file "examples/production/smom-dbis-138/rpc-node-02.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        rpc-node-03)
+            update_template_file "examples/production/smom-dbis-138/rpc-node-03.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        rpc-node-04)
+            update_template_file "examples/production/smom-dbis-138/rpc-node-04.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        smom-services)
+            update_template_file "examples/production/smom-dbis-138/services.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        smom-blockscout)
+            update_template_file "examples/production/smom-dbis-138/blockscout.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        monitoring)
+            update_template_file "examples/production/smom-dbis-138/monitoring.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        management)
+            update_template_file "examples/production/smom-dbis-138/management.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+    esac
+done <<< "$VM_CONFIGS"
+
+# Update Phoenix templates
+echo ""
+echo "Step 3: Updating Phoenix VM templates..."
+while IFS='|' read -r namespace name node cpu memory disk storage image network site; do
+    case "$name" in
+        phoenix-as4-gateway)
+            update_template_file "examples/production/phoenix/as4-gateway.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        phoenix-business-integration-gateway)
+            update_template_file "examples/production/phoenix/business-integration-gateway.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        phoenix-codespaces-ide)
+            update_template_file "examples/production/phoenix/codespaces-ide.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        phoenix-devops-runner)
+            update_template_file "examples/production/phoenix/devops-runner.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        phoenix-dns-primary)
+            update_template_file "examples/production/phoenix/dns-primary.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        phoenix-email-server)
+            update_template_file "examples/production/phoenix/email-server.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        phoenix-financial-messaging-gateway)
+            update_template_file "examples/production/phoenix/financial-messaging-gateway.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        phoenix-git-server)
+            update_template_file "examples/production/phoenix/git-server.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+    esac
+done <<< "$VM_CONFIGS"
+
+# Update other production VMs
+echo ""
+echo "Step 4: Updating other production VM templates..."
+while IFS='|' read -r namespace name node cpu memory disk storage image network site; do
+    # Skip SMOM and Phoenix (already handled)
+    if [[ "$name" =~ ^(smom-|phoenix-|rpc-node|monitoring|management)$ ]]; then
+        continue
+    fi
+    # Skip example VMs
+    if [[ "$name" =~ ^(basic-vm-001|large-vm-001|medium-vm-001|vm-100)$ ]]; then
+        continue
+    fi
+    
+    case "$name" in
+        nginx-proxy-vm)
+            update_template_file "examples/production/nginx-proxy-vm.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+        cloudflare-tunnel-vm)
+            update_template_file "examples/production/cloudflare-tunnel-vm.yaml" "$name" "$namespace" "$node" "$cpu" "$memory" "$disk" "$storage" "$image" "$network" "$site"
+            ((UPDATED_COUNT++))
+            ;;
+    esac
+done <<< "$VM_CONFIGS"
+
+echo ""
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  SUMMARY"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+echo "✅ Updated $UPDATED_COUNT VM template files"
+echo ""
+echo "All templates now reflect current Kubernetes configurations:"
+echo "  - Storage: local-lvm (or as configured)"
+echo "  - Image: 9000 (template) or local:iso/ubuntu-22.04-cloud.img"
+echo "  - Current CPU, RAM, and Disk specifications"
+echo "  - Metadata, labels, and userData preserved"
+

scripts/update-vms-to-ceph-rbd.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+# Update all VMs configured for r630-01 to use ceph-rbd instead of ceph-fs
+
+set -e
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  UPDATE VMs TO USE ceph-rbd STORAGE"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+cd "$(dirname "$0")/.." || exit 1
+
+echo "Step 1: Finding VMs configured for r630-01 with ceph-fs storage..."
+VMS=$(kubectl get proxmoxvms -A -o json 2>/dev/null | \
+    jq -r '.items[] | 
+        select(.spec.forProvider.node == "r630-01") | 
+        select(.spec.forProvider.storage == "ceph-fs") | 
+        select(.metadata.name != "basic-vm-001" and .metadata.name != "large-vm-001" and .metadata.name != "medium-vm-001" and .metadata.name != "vm-100") |
+        "\(.metadata.namespace)/\(.metadata.name)"' 2>/dev/null)
+
+if [ -z "$VMS" ]; then
+    echo "✅ No VMs found with ceph-fs storage on r630-01"
+    exit 0
+fi
+
+VM_COUNT=$(echo "$VMS" | wc -l)
+echo "Found $VM_COUNT VMs to update:"
+echo "$VMS" | while read -r vm; do
+    echo "  • $vm"
+done
+
+echo ""
+read -p "Update these VMs to use ceph-rbd? (y/N) " -n 1 -r
+echo
+if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+    echo "Cancelled."
+    exit 0
+fi
+
+echo ""
+echo "Step 2: Updating VMs..."
+UPDATED=0
+FAILED=0
+
+echo "$VMS" | while read -r vm; do
+    NAMESPACE=$(echo "$vm" | cut -d'/' -f1)
+    NAME=$(echo "$vm" | cut -d'/' -f2)
+    
+    echo "  Updating $vm..."
+    if kubectl patch proxmoxvm "$NAME" -n "$NAMESPACE" --type='json' \
+        -p='[{"op": "replace", "path": "/spec/forProvider/storage", "value": "ceph-rbd"}]' 2>/dev/null; then
+        echo "    ✅ Updated $vm"
+        ((UPDATED++)) || true
+    else
+        echo "    ❌ Failed to update $vm"
+        ((FAILED++)) || true
+    fi
+done
+
+echo ""
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  SUMMARY"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+echo "VMs updated: $UPDATED"
+echo "VMs failed: $FAILED"
+echo ""
+echo "✅ Update complete! VMs will now use ceph-rbd storage."
+echo ""
+echo "Note: Make sure ceph-rbd storage is configured in Proxmox first:"
+echo "  ssh root@192.168.11.11 'bash -s' < scripts/create-ceph-rbd-storage-r630.sh"
+

scripts/update-vms-to-local-lvm.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# Update VMs to use local-lvm storage instead of ceph-rbd
+
+set -euo pipefail
+
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  UPDATE VMs TO USE local-lvm STORAGE"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo ""
+
+# Find VMs configured for r630-01 with ceph-rbd storage
+echo "Step 1: Finding VMs configured for r630-01 with ceph-rbd storage..."
+VMS=$(kubectl get proxmoxvms -A -o json 2>/dev/null | \
+    jq -r '[.items[] | 
+        select(.spec.forProvider.node == "r630-01") | 
+        select(.spec.forProvider.storage == "ceph-rbd") |
+        select(.metadata.name != "basic-vm-001" and 
+               .metadata.name != "large-vm-001" and 
+               .metadata.name != "medium-vm-001" and 
+               .metadata.name != "vm-100") |
+        "\(.metadata.namespace)/\(.metadata.name)"
+    ] | .[]')
+
+if [ -z "$VMS" ]; then
+    echo "✅ No VMs found to update (already using local-lvm or different storage)"
+    exit 0
+fi
+
+VM_COUNT=$(echo "$VMS" | wc -l)
+echo "Found $VM_COUNT VMs to update:"
+echo "$VMS" | sed 's/^/  • /'
+echo ""
+
+# Update VMs
+echo "Step 2: Updating VMs..."
+UPDATED=0
+FAILED=0
+
+for VM in $VMS; do
+    NAMESPACE=$(echo "$VM" | cut -d'/' -f1)
+    NAME=$(echo "$VM" | cut -d'/' -f2)
+    
+    echo -n "Updating $VM... "
+    if kubectl patch proxmoxvm -n "$NAMESPACE" "$NAME" \
+        --type='json' \
+        -p='[{"op": "replace", "path": "/spec/forProvider/storage", "value": "local-lvm"}]' 2>/dev/null; then
+        echo "✅"
+        UPDATED=$((UPDATED + 1))
+    else
+        echo "❌"
+        FAILED=$((FAILED + 1))
+    fi
+done
+
+echo ""
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "  SUMMARY"
+echo "═══════════════════════════════════════════════════════════════════════════════"
+echo "VMs updated: $UPDATED"
+echo "VMs failed: $FAILED"
+echo ""
+echo "✅ Update complete! VMs will now use local-lvm storage."
+echo ""
+echo "📝 Next Steps:"
+echo "   1. Copy cloud image to local-lvm storage"
+echo "   2. Monitor VM deployment: kubectl get proxmoxvms -A -w"
+

scripts/verify-disks-via-api.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# Alternative verification using Proxmox API
+# This uses the Proxmox API token to query disk information
+
+set -e
+
+# Configuration
+PROXMOX_HOST="192.168.11.11"
+PROXMOX_PORT="8006"
+PROXMOX_USER="root@pam"
+
+# Get token from Kubernetes secret
+echo "Attempting to get Proxmox credentials from Kubernetes..."
+TOKEN=$(kubectl get secret -n crossplane-system proxmox-credentials -o jsonpath='{.data.token}' 2>/dev/null | base64 -d)
+
+if [ -z "$TOKEN" ]; then
+    echo "ERROR: Could not retrieve Proxmox token from Kubernetes"
+    echo "Please run the verification script directly on R630-01 instead"
+    exit 1
+fi
+
+echo "Token retrieved (first 20 chars): ${TOKEN:0:20}..."
+echo ""
+
+# Try to query Proxmox API for storage information
+echo "Querying Proxmox API for storage information..."
+echo ""
+
+# Note: Proxmox API endpoints for storage/disk info
+# This is a basic attempt - may need adjustment based on actual API
+
+echo "=== Storage Information via Proxmox API ==="
+echo ""
+echo "Note: Direct disk enumeration via API is limited."
+echo "For detailed disk information, please run the verification script"
+echo "directly on R630-01:"
+echo ""
+echo "  ssh root@192.168.11.11"
+echo "  bash /tmp/verify-r630-250gb-drives.sh"
+echo ""
+echo "Or run these commands manually on R630-01:"
+echo ""
+echo "  lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL,SERIAL"
+echo "  fdisk -l | grep -E '^Disk /dev/sd'"
+echo "  ceph osd tree"
+echo ""
+

scripts/verify-proxmox-certs.sh

@@ -0,0 +1,254 @@
+#!/bin/bash
+set -euo pipefail
+
+# Proxmox Certificate Verification Script
+# Verifies ACME certificate installation and validity on Proxmox nodes
+
+# Color codes for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Configuration
+NODES=(
+    "ml110-01.sankofa.nexus:192.168.11.10"
+    "r630-01.sankofa.nexus:192.168.11.11"
+)
+PORT=8006
+WARN_DAYS=30  # Warn if certificate expires in less than 30 days
+
+log() {
+    echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $*" >&2
+}
+
+info() {
+    echo -e "${GREEN}✓${NC} $*"
+}
+
+warn() {
+    echo -e "${YELLOW}⚠${NC} $*" >&2
+}
+
+error() {
+    echo -e "${RED}✗${NC} $*" >&2
+}
+
+# Check if openssl is available
+check_openssl() {
+    if ! command -v openssl &> /dev/null; then
+        error "openssl is not installed. Please install openssl to use this script."
+        exit 1
+    fi
+}
+
+# Get certificate expiration date
+get_cert_expiry() {
+    local hostname=$1
+    local ip=$2
+    local port=$3
+    
+    # Connect and extract expiration date
+    local expiry=$(echo | openssl s_client -connect "${ip}:${port}" -servername "${hostname}" 2>/dev/null | \
+                   openssl x509 -noout -enddate 2>/dev/null | \
+                   cut -d= -f2)
+    
+    if [ -z "${expiry}" ]; then
+        return 1
+    fi
+    
+    echo "${expiry}"
+}
+
+# Calculate days until expiration
+days_until_expiry() {
+    local expiry_date=$1
+    
+    # Convert expiry date to epoch timestamp
+    # Try Linux date format first, then macOS format
+    local expiry_epoch
+    if expiry_epoch=$(date -d "${expiry_date}" +%s 2>/dev/null); then
+        :
+    elif expiry_epoch=$(date -j -f "%b %d %H:%M:%S %Y %Z" "${expiry_date}" +%s 2>/dev/null); then
+        :
+    else
+        return 1
+    fi
+    
+    local now_epoch=$(date +%s)
+    local diff=$((expiry_epoch - now_epoch))
+    local days=$((diff / 86400))
+    
+    echo "${days}"
+}
+
+# Get certificate issuer
+get_cert_issuer() {
+    local hostname=$1
+    local ip=$2
+    local port=$3
+    
+    local issuer=$(echo | openssl s_client -connect "${ip}:${port}" -servername "${hostname}" 2>/dev/null | \
+                   openssl x509 -noout -issuer 2>/dev/null | \
+                   sed 's/issuer=//')
+    
+    echo "${issuer}"
+}
+
+# Get certificate subject
+get_cert_subject() {
+    local hostname=$1
+    local ip=$2
+    local port=$3
+    
+    local subject=$(echo | openssl s_client -connect "${ip}:${port}" -servername "${hostname}" 2>/dev/null | \
+                    openssl x509 -noout -subject 2>/dev/null | \
+                    sed 's/subject=//')
+    
+    echo "${subject}"
+}
+
+# Get certificate SANs (Subject Alternative Names)
+get_cert_sans() {
+    local hostname=$1
+    local ip=$2
+    local port=$3
+    
+    local sans=$(echo | openssl s_client -connect "${ip}:${port}" -servername "${hostname}" 2>/dev/null | \
+                 openssl x509 -noout -text 2>/dev/null | \
+                 grep -A1 "Subject Alternative Name" | \
+                 sed 's/.*DNS://g' | \
+                 tr ',' '\n' | \
+                 sed 's/^ *//' | \
+                 grep -v "^$")
+    
+    echo "${sans}"
+}
+
+# Test SSL connection
+test_ssl_connection() {
+    local hostname=$1
+    local ip=$2
+    local port=$3
+    
+    # Test connection and get certificate
+    local output=$(echo | openssl s_client -connect "${ip}:${port}" -servername "${hostname}" 2>&1)
+    
+    if echo "${output}" | grep -q "Verify return code: 0"; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# Verify certificate for a node
+verify_node_cert() {
+    local node_info=$1
+    local hostname=$(echo "${node_info}" | cut -d: -f1)
+    local ip=$(echo "${node_info}" | cut -d: -f2)
+    
+    log "Checking certificate for ${hostname} (${ip}:${PORT})..."
+    
+    # Test SSL connection
+    if ! test_ssl_connection "${hostname}" "${ip}" "${PORT}"; then
+        error "SSL connection failed for ${hostname}"
+        return 1
+    fi
+    
+    info "SSL connection successful"
+    
+    # Get certificate details
+    local issuer=$(get_cert_issuer "${hostname}" "${ip}" "${PORT}")
+    local subject=$(get_cert_subject "${hostname}" "${ip}" "${PORT}")
+    local expiry=$(get_cert_expiry "${hostname}" "${ip}" "${PORT}")
+    
+    if [ -z "${expiry}" ]; then
+        error "Could not retrieve certificate expiration for ${hostname}"
+        return 1
+    fi
+    
+    # Check if certificate is from Let's Encrypt
+    if echo "${issuer}" | grep -qi "lets encrypt\|let's encrypt\|letsencrypt"; then
+        info "Certificate issuer: Let's Encrypt"
+    else
+        warn "Certificate issuer: ${issuer}"
+        warn "Certificate may not be from Let's Encrypt"
+    fi
+    
+    # Display certificate subject
+    info "Certificate subject: ${subject}"
+    
+    # Check expiration
+    local days=$(days_until_expiry "${expiry}")
+    
+    if [ "${days}" -lt 0 ]; then
+        error "Certificate expired ${days#-} days ago!"
+        error "Expiration date: ${expiry}"
+        return 1
+    elif [ "${days}" -lt "${WARN_DAYS}" ]; then
+        warn "Certificate expires in ${days} days (${expiry})"
+        warn "Renewal should occur automatically, but please monitor"
+    else
+        info "Certificate expires in ${days} days (${expiry})"
+    fi
+    
+    # Get and display SANs
+    local sans=$(get_cert_sans "${hostname}" "${ip}" "${PORT}")
+    if [ -n "${sans}" ]; then
+        info "Certificate SANs:"
+        echo "${sans}" | while read -r san; do
+            echo "    - ${san}"
+        done
+    fi
+    
+    # Verify hostname matches
+    if echo "${subject}" | grep -qi "${hostname}" || echo "${sans}" | grep -qi "${hostname}"; then
+        info "Hostname ${hostname} matches certificate"
+    else
+        warn "Hostname ${hostname} may not match certificate"
+        warn "Subject: ${subject}"
+    fi
+    
+    echo ""
+    return 0
+}
+
+# Main verification function
+main() {
+    log "Proxmox Certificate Verification Script"
+    log "========================================"
+    echo ""
+    
+    check_openssl
+    
+    local total_nodes=${#NODES[@]}
+    local passed=0
+    local failed=0
+    
+    for node_info in "${NODES[@]}"; do
+        if verify_node_cert "${node_info}"; then
+            ((passed++))
+        else
+            ((failed++))
+        fi
+    done
+    
+    echo ""
+    log "========================================"
+    log "Verification Summary"
+    log "========================================"
+    info "Total nodes checked: ${total_nodes}"
+    info "Passed: ${passed}"
+    if [ "${failed}" -gt 0 ]; then
+        error "Failed: ${failed}"
+        exit 1
+    else
+        info "All certificates are valid"
+        exit 0
+    fi
+}
+
+# Run main function
+main "$@"
+

scripts/verify-r630-250gb-drives.sh

@@ -0,0 +1,193 @@
+#!/bin/bash
+# Comprehensive script to verify 250GB drives on R630-01
+# Run this script on R630-01 (192.168.11.11) as root
+
+set -e
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+echo "=========================================="
+echo "R630-01 250GB Drive Verification"
+echo "=========================================="
+echo "Date: $(date)"
+echo "Hostname: $(hostname)"
+echo ""
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}ERROR: Please run as root${NC}"
+    exit 1
+fi
+
+echo -e "${BLUE}=== Step 1: All Block Devices ===${NC}"
+echo "-----------------------------------"
+echo "Listing all block devices with details:"
+lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL,SERIAL,STATE | grep -E "NAME|sd"
+echo ""
+
+echo -e "${BLUE}=== Step 2: All Physical Disks ===${NC}"
+echo "-----------------------------------"
+echo "All disks detected by system:"
+fdisk -l 2>/dev/null | grep -E "^Disk /dev/sd" | sort -k2
+echo ""
+
+echo -e "${BLUE}=== Step 3: Identifying 250GB Drives ===${NC}"
+echo "-----------------------------------"
+echo "Drives around 250GB size (232-260 GB):"
+echo ""
+found_250gb=false
+for disk in /dev/sd[a-z]; do
+    if [ -b "$disk" ]; then
+        size_info=$(fdisk -l "$disk" 2>/dev/null | grep "^Disk $disk" | awk '{print $3, $4}')
+        if [ ! -z "$size_info" ]; then
+            size_num=$(echo "$size_info" | grep -oE '[0-9]+' | head -1)
+            size_unit=$(echo "$size_info" | grep -oE '[A-Za-z]+' | head -1)
+            
+            # Check if size is around 250GB (232-260 GB range)
+            if [ "$size_unit" = "GiB" ] || [ "$size_unit" = "GB" ]; then
+                if [ "$size_num" -ge 232 ] && [ "$size_num" -le 260 ]; then
+                    echo -e "${GREEN}  ✓ $disk: ${size_info}${NC}"
+                    found_250gb=true
+                    
+                    # Check if it's mounted or in use
+                    mount_point=$(lsblk -n -o MOUNTPOINT "$disk" 2>/dev/null | head -1)
+                    if [ ! -z "$mount_point" ]; then
+                        echo -e "    ${YELLOW}  WARNING: Mounted at $mount_point${NC}"
+                    fi
+                    
+                    # Check if it has partitions
+                    partitions=$(lsblk -n -o NAME "$disk" 2>/dev/null | grep -v "^$(basename $disk)$" | wc -l)
+                    if [ "$partitions" -gt 0 ]; then
+                        echo -e "    ${YELLOW}  WARNING: Has $partitions partition(s)${NC}"
+                        lsblk "$disk"
+                    else
+                        echo -e "    ${GREEN}  ✓ No partitions (available for OSD)${NC}"
+                    fi
+                fi
+            fi
+        fi
+    fi
+done
+
+if [ "$found_250gb" = false ]; then
+    echo -e "${YELLOW}  No 250GB drives found in expected size range${NC}"
+    echo "  Checking all drives for reference:"
+    fdisk -l 2>/dev/null | grep -E "^Disk /dev/sd" | awk '{print "    " $0}'
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 4: Current Ceph OSD Status ===${NC}"
+echo "-----------------------------------"
+if command -v ceph &> /dev/null; then
+    echo "Current OSD tree:"
+    ceph osd tree 2>/dev/null || echo -e "${YELLOW}Ceph command failed or not accessible${NC}"
+    echo ""
+    echo "OSD details:"
+    ceph osd df 2>/dev/null || echo -e "${YELLOW}Ceph command failed${NC}"
+    echo ""
+    echo "Ceph health:"
+    ceph health 2>/dev/null || echo -e "${YELLOW}Ceph command failed${NC}"
+else
+    echo -e "${YELLOW}Ceph command not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 5: Storage Pools ===${NC}"
+echo "-----------------------------------"
+if command -v pvesm &> /dev/null; then
+    echo "Proxmox storage pools:"
+    pvesm status 2>/dev/null || echo -e "${YELLOW}pvesm command failed${NC}"
+else
+    echo -e "${YELLOW}pvesm command not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 6: Volume Groups ===${NC}"
+echo "-----------------------------------"
+if command -v vgs &> /dev/null; then
+    echo "LVM Volume Groups:"
+    vgs 2>/dev/null || echo -e "${YELLOW}vgs command failed${NC}"
+    echo ""
+    echo "Physical Volumes:"
+    pvs 2>/dev/null || echo -e "${YELLOW}pvs command failed${NC}"
+else
+    echo -e "${YELLOW}LVM commands not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 7: Unused/Available Disks ===${NC}"
+echo "-----------------------------------"
+echo "Checking for disks that could be used for Ceph OSD:"
+echo ""
+available_count=0
+for disk in /dev/sd[a-z]; do
+    if [ -b "$disk" ]; then
+        # Skip if it's the boot disk (sda) or current OSD (sdb)
+        if [ "$disk" = "/dev/sda" ] || [ "$disk" = "/dev/sdb" ]; then
+            continue
+        fi
+        
+        # Check if disk has mount point
+        mount_point=$(lsblk -n -o MOUNTPOINT "$disk" 2>/dev/null | head -1)
+        if [ -z "$mount_point" ]; then
+            # Check if disk is in a volume group
+            in_vg=$(pvs 2>/dev/null | grep "$disk" | wc -l)
+            if [ "$in_vg" -eq 0 ]; then
+                size_info=$(fdisk -l "$disk" 2>/dev/null | grep "^Disk $disk" | awk '{print $3, $4}')
+                if [ ! -z "$size_info" ]; then
+                    echo -e "${GREEN}  ✓ $disk: ${size_info} - Potentially available${NC}"
+                    available_count=$((available_count + 1))
+                    
+                    # Show disk details
+                    echo "    Details:"
+                    lsblk "$disk" | sed 's/^/      /'
+                fi
+            fi
+        fi
+    fi
+done
+
+if [ "$available_count" -eq 0 ]; then
+    echo -e "${YELLOW}  No obviously available disks found${NC}"
+    echo "  All disks may be in use or need further investigation"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 8: PERC Controller Status (if available) ===${NC}"
+echo "-----------------------------------"
+if command -v omreport &> /dev/null; then
+    echo "PERC controller virtual disks:"
+    omreport storage vdisk 2>/dev/null || echo -e "${YELLOW}omreport not available or no PERC controller${NC}"
+    echo ""
+    echo "PERC controller physical disks:"
+    omreport storage pdisk 2>/dev/null || echo -e "${YELLOW}omreport not available${NC}"
+else
+    echo -e "${YELLOW}Dell OpenManage tools not installed${NC}"
+    echo "  To install: apt-get install srvadmin-all"
+fi
+echo ""
+
+echo "=========================================="
+echo -e "${GREEN}Verification Complete${NC}"
+echo "=========================================="
+echo ""
+echo "Summary:"
+echo "--------"
+echo "1. Review the 250GB drives identified above"
+echo "2. Check if any are available (no partitions, not mounted, not in VG)"
+echo "3. If available, you can create a Ceph OSD with:"
+echo ""
+echo -e "   ${YELLOW}ceph-volume lvm create --data /dev/sdX${NC}"
+echo ""
+echo "   (Replace sdX with the actual device, e.g., sdc, sdd, etc.)"
+echo ""
+echo "4. After creating OSD, verify with:"
+echo "   ceph osd tree"
+echo "   ceph health"
+echo ""
+

scripts/verify-r630-disks.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+# Script to verify and configure 250GB drives on R630-01 for Ceph OSD
+# Run this script on R630-01 (192.168.11.11)
+
+set -e
+
+echo "=========================================="
+echo "R630-01 Disk Verification and Ceph OSD Setup"
+echo "=========================================="
+echo ""
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Check if running as root
+if [ "$EUID" -ne 0 ]; then 
+    echo -e "${RED}Please run as root${NC}"
+    exit 1
+fi
+
+echo "Step 1: Listing all block devices..."
+echo "-----------------------------------"
+lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL,SERIAL | grep -E "NAME|sd"
+echo ""
+
+echo "Step 2: Checking all disks..."
+echo "-----------------------------------"
+fdisk -l 2>/dev/null | grep -E "^Disk /dev/sd" | sort
+echo ""
+
+echo "Step 3: Identifying 250GB drives..."
+echo "-----------------------------------"
+echo "Looking for drives around 250GB size..."
+fdisk -l 2>/dev/null | grep -E "^Disk /dev/sd" | grep -iE "232|233|234|235|236|237|238|239|240|241|242|243|244|245|246|247|248|249|250|251|252|253|254|255|256|257|258|259|260" || echo "No 250GB drives found in fdisk output"
+echo ""
+
+echo "Step 4: Checking current Ceph OSD status..."
+echo "-----------------------------------"
+if command -v ceph &> /dev/null; then
+    echo "Current OSD tree:"
+    ceph osd tree 2>/dev/null || echo "Ceph not accessible or not configured"
+    echo ""
+    echo "Current OSD details:"
+    ceph osd df 2>/dev/null || echo "Ceph not accessible"
+    echo ""
+    echo "Ceph health:"
+    ceph health 2>/dev/null || echo "Ceph not accessible"
+else
+    echo "Ceph command not found"
+fi
+echo ""
+
+echo "Step 5: Checking storage pools..."
+echo "-----------------------------------"
+if command -v pvesm &> /dev/null; then
+    pvesm status 2>/dev/null || echo "pvesm not accessible"
+else
+    echo "pvesm command not found"
+fi
+echo ""
+
+echo "Step 6: Finding unused disks (potential for OSD)..."
+echo "-----------------------------------"
+echo "Checking for disks without partitions or filesystems..."
+for disk in /dev/sd[a-z]; do
+    if [ -b "$disk" ]; then
+        # Skip if it's the boot disk or has partitions
+        if ! lsblk -n -o MOUNTPOINT "$disk" 2>/dev/null | grep -q "/"; then
+            size=$(fdisk -l "$disk" 2>/dev/null | grep "^Disk $disk" | awk '{print $3 $4}')
+            echo "  $disk: $size (potentially available)"
+        fi
+    fi
+done
+echo ""
+
+echo "=========================================="
+echo "Summary and Recommendations"
+echo "=========================================="
+echo ""
+echo "1. Review the output above to identify the 6x 250GB drives"
+echo "2. Check if any drives are uninitialized (no partitions)"
+echo "3. If a drive is available, you can create a Ceph OSD with:"
+echo ""
+echo "   # WARNING: This will destroy all data on the drive!"
+echo "   ceph-volume lvm create --data /dev/sdX"
+echo ""
+echo "   Replace sdX with the actual device (e.g., sdc, sdd, etc.)"
+echo ""
+echo "4. After creating OSD, verify with:"
+echo "   ceph osd tree"
+echo "   ceph health"
+echo ""
+echo "5. If Ceph health improves, the third OSD is working!"
+echo ""
+

scripts/wipe-all-250gb-drives.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+# Script to wipe all 6x 250GB drives (sdc, sdd, sde, sdf, sdg, sdh)
+# Run on R630-01 as root
+# WARNING: This will destroy all data on these drives!
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+# Drives to wipe
+DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")
+
+echo "=========================================="
+echo "Wipe All 250GB Drives"
+echo "=========================================="
+echo ""
+echo -e "${RED}WARNING: This will destroy ALL data on these drives!${NC}"
+echo "Drives to wipe: ${DRIVES[@]}"
+echo ""
+read -p "Are you sure you want to continue? (type 'yes' to confirm): " confirm
+
+if [ "$confirm" != "yes" ]; then
+    echo "Aborted"
+    exit 0
+fi
+
+echo ""
+echo -e "${BLUE}=== Step 1: Removing from ubuntu-vg ===${NC}"
+echo "-----------------------------------"
+
+# Check if ubuntu-vg exists and has mounted volumes
+if vgs ubuntu-vg &>/dev/null; then
+    echo "Checking ubuntu-vg status..."
+    
+    # Unmount any mounted logical volumes
+    echo "  Unmounting logical volumes..."
+    umount /dev/ubuntu-vg/* 2>/dev/null || true
+    
+    # Deactivate volume group
+    echo "  Deactivating ubuntu-vg..."
+    vgchange -a n ubuntu-vg 2>/dev/null || true
+    
+    # Remove physical volumes from ubuntu-vg
+    echo "  Removing physical volumes from ubuntu-vg..."
+    for drive in "${DRIVES[@]}"; do
+        if pvs 2>/dev/null | grep -q "/dev/${drive}3.*ubuntu-vg"; then
+            echo "    Removing /dev/${drive}3 from ubuntu-vg..."
+            pvremove "/dev/${drive}3" -y -ff 2>/dev/null || echo "    (Already removed or not in VG)"
+        fi
+    done
+    
+    echo -e "${GREEN}  ✓ Removed from ubuntu-vg${NC}"
+else
+    echo -e "${YELLOW}  ubuntu-vg not found or already removed${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 2: Wiping All Drives ===${NC}"
+echo "-----------------------------------"
+
+for drive in "${DRIVES[@]}"; do
+    if [ ! -b "/dev/$drive" ]; then
+        echo -e "${YELLOW}  /dev/$drive not found, skipping...${NC}"
+        continue
+    fi
+    
+    echo -e "${BLUE}  Wiping /dev/$drive...${NC}"
+    
+    # Unmount any partitions
+    umount /dev/${drive}* 2>/dev/null || true
+    
+    # Wipe filesystem signatures
+    echo "    Removing filesystem signatures..."
+    wipefs -a "/dev/$drive" 2>/dev/null || true
+    
+    # Zero out first 100MB
+    echo "    Zeroing first 100MB..."
+    dd if=/dev/zero of="/dev/$drive" bs=1M count=100 status=progress 2>/dev/null || true
+    
+    echo -e "${GREEN}    ✓ /dev/$drive wiped${NC}"
+    echo ""
+done
+
+echo "=========================================="
+echo -e "${GREEN}All Drives Wiped Successfully!${NC}"
+echo "=========================================="
+echo ""
+echo "Next steps:"
+echo "1. Create Ceph OSD on one drive (for third OSD):"
+echo "   ceph-volume lvm create --data /dev/sdc"
+echo ""
+echo "2. Or create OSDs on multiple drives (for better performance):"
+echo "   ceph-volume lvm create --data /dev/sdc"
+echo "   ceph-volume lvm create --data /dev/sdd"
+echo "   ceph-volume lvm create --data /dev/sde"
+echo "   # ... etc"
+echo ""
+echo "3. Verify:"
+echo "   ceph osd tree"
+echo "   ceph health"
+echo ""
+

scripts/wipe-and-create-osds.sh

@@ -0,0 +1,133 @@
+#!/bin/bash
+# Script to wipe all 6x 250GB drives AND create Ceph OSDs on them
+# Run on R630-01 as root
+# WARNING: This will destroy all data on these drives!
+
+set -e
+
+# Colors
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m'
+
+# Drives to process
+DRIVES=("sdc" "sdd" "sde" "sdf" "sdg" "sdh")
+
+echo "=========================================="
+echo "Wipe All 250GB Drives and Create Ceph OSDs"
+echo "=========================================="
+echo ""
+echo -e "${RED}WARNING: This will destroy ALL data on these drives!${NC}"
+echo "Drives to process: ${DRIVES[@]}"
+echo ""
+echo "This script will:"
+echo "  1. Remove drives from ubuntu-vg"
+echo "  2. Wipe all 6 drives"
+echo "  3. Create Ceph OSDs on all 6 drives"
+echo ""
+read -p "Are you sure you want to continue? (type 'yes' to confirm): " confirm
+
+if [ "$confirm" != "yes" ]; then
+    echo "Aborted"
+    exit 0
+fi
+
+echo ""
+echo -e "${BLUE}=== Step 1: Removing from ubuntu-vg ===${NC}"
+echo "-----------------------------------"
+
+# Check if ubuntu-vg exists
+if vgs ubuntu-vg &>/dev/null; then
+    echo "Unmounting logical volumes..."
+    umount /dev/ubuntu-vg/* 2>/dev/null || true
+    
+    echo "Deactivating ubuntu-vg..."
+    vgchange -a n ubuntu-vg 2>/dev/null || true
+    
+    echo "Removing physical volumes from ubuntu-vg..."
+    for drive in "${DRIVES[@]}"; do
+        if pvs 2>/dev/null | grep -q "/dev/${drive}3.*ubuntu-vg"; then
+            echo "  Removing /dev/${drive}3..."
+            pvremove "/dev/${drive}3" -y -ff 2>/dev/null || true
+        fi
+    done
+    echo -e "${GREEN}  ✓ Removed from ubuntu-vg${NC}"
+else
+    echo -e "${YELLOW}  ubuntu-vg not found${NC}"
+fi
+echo ""
+
+echo -e "${BLUE}=== Step 2: Wiping All Drives ===${NC}"
+echo "-----------------------------------"
+
+for drive in "${DRIVES[@]}"; do
+    if [ ! -b "/dev/$drive" ]; then
+        echo -e "${YELLOW}  /dev/$drive not found, skipping...${NC}"
+        continue
+    fi
+    
+    echo -e "${BLUE}  Wiping /dev/$drive...${NC}"
+    
+    # Unmount any partitions
+    umount /dev/${drive}* 2>/dev/null || true
+    
+    # Wipe filesystem signatures
+    wipefs -a "/dev/$drive" 2>/dev/null || true
+    
+    # Zero out first 100MB
+    dd if=/dev/zero of="/dev/$drive" bs=1M count=100 status=progress 2>/dev/null || true
+    
+    echo -e "${GREEN}    ✓ /dev/$drive wiped${NC}"
+done
+echo ""
+
+echo -e "${BLUE}=== Step 3: Creating Ceph OSDs ===${NC}"
+echo "-----------------------------------"
+
+osd_count=0
+for drive in "${DRIVES[@]}"; do
+    if [ ! -b "/dev/$drive" ]; then
+        continue
+    fi
+    
+    echo -e "${BLUE}  Creating OSD on /dev/$drive...${NC}"
+    
+    if ceph-volume lvm create --data "/dev/$drive" 2>&1 | tee /tmp/ceph-osd-${drive}.log; then
+        echo -e "${GREEN}    ✓ OSD created on /dev/$drive${NC}"
+        osd_count=$((osd_count + 1))
+    else
+        echo -e "${RED}    ✗ Failed to create OSD on /dev/$drive${NC}"
+        echo "    Check /tmp/ceph-osd-${drive}.log for details"
+    fi
+    echo ""
+done
+
+echo "=========================================="
+echo -e "${GREEN}Process Complete!${NC}"
+echo "=========================================="
+echo ""
+echo "Created $osd_count OSD(s) out of ${#DRIVES[@]} drives"
+echo ""
+
+echo "Verifying Ceph status..."
+echo "OSD Tree:"
+ceph osd tree
+echo ""
+echo "Ceph Health:"
+ceph health
+echo ""
+echo "OSD Details:"
+ceph osd df
+echo ""
+
+if [ "$osd_count" -gt 0 ]; then
+    echo -e "${GREEN}✓ Successfully created $osd_count OSD(s)!${NC}"
+    echo ""
+    echo "You should now have at least 3 OSDs, which fixes the TOO_FEW_OSDS error."
+else
+    echo -e "${RED}✗ No OSDs were created. Check logs above for errors.${NC}"
+fi
+echo ""
+