331 lines
11 KiB
Bash
331 lines
11 KiB
Bash
|
#!/bin/bash
|
|||
|
|
# Comprehensive Infrastructure Testing Script for Phase 1
|
|||
|
|
# Tests all deployed resources and connectivity
|
|||
|
|
|
|||
|
|
set -euo pipefail
|
|||
|
|
|
|||
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|||
|
|
PHASE1_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|||
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
|
|||
|
|
|
|||
|
|
# Colors for output
|
|||
|
|
RED='\033[0;31m'
|
|||
|
|
GREEN='\033[0;32m'
|
|||
|
|
YELLOW='\033[1;33m'
|
|||
|
|
BLUE='\033[0;34m'
|
|||
|
|
NC='\033[0m' # No Color
|
|||
|
|
|
|||
|
|
# Test results
|
|||
|
|
TESTS_PASSED=0
|
|||
|
|
TESTS_FAILED=0
|
|||
|
|
TESTS_SKIPPED=0
|
|||
|
|
|
|||
|
|
# Function to print test result
|
|||
|
|
print_test() {
|
|||
|
|
local status=$1
|
|||
|
|
local message=$2
|
|||
|
|
case $status in
|
|||
|
|
PASS)
|
|||
|
|
echo -e "${GREEN}✓${NC} $message"
|
|||
|
|
((TESTS_PASSED++))
|
|||
|
|
;;
|
|||
|
|
FAIL)
|
|||
|
|
echo -e "${RED}✗${NC} $message"
|
|||
|
|
((TESTS_FAILED++))
|
|||
|
|
;;
|
|||
|
|
SKIP)
|
|||
|
|
echo -e "${YELLOW}⊘${NC} $message"
|
|||
|
|
((TESTS_SKIPPED++))
|
|||
|
|
;;
|
|||
|
|
INFO)
|
|||
|
|
echo -e "${BLUE}ℹ${NC} $message"
|
|||
|
|
;;
|
|||
|
|
esac
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
echo "=========================================="
|
|||
|
|
echo "Phase 1 Infrastructure Testing"
|
|||
|
|
echo "=========================================="
|
|||
|
|
echo ""
|
|||
|
|
|
|||
|
|
# Test 1: Terraform State
|
|||
|
|
echo "1. Testing Terraform State..."
|
|||
|
|
cd "$PHASE1_DIR"
|
|||
|
|
if terraform state list > /dev/null 2>&1; then
|
|||
|
|
RESOURCE_COUNT=$(terraform state list 2>/dev/null | wc -l)
|
|||
|
|
if [ "$RESOURCE_COUNT" -ge 100 ]; then
|
|||
|
|
print_test PASS "Terraform state accessible ($RESOURCE_COUNT resources)"
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Terraform state has fewer resources than expected ($RESOURCE_COUNT)"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Terraform state not accessible"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 2: Terraform Outputs
|
|||
|
|
echo ""
|
|||
|
|
echo "2. Testing Terraform Outputs..."
|
|||
|
|
if terraform output -json > /dev/null 2>&1; then
|
|||
|
|
OUTPUTS=$(terraform output -json 2>/dev/null)
|
|||
|
|
if echo "$OUTPUTS" | jq -e '.nginx_proxy' > /dev/null 2>&1; then
|
|||
|
|
print_test PASS "Terraform outputs accessible"
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Terraform outputs incomplete"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Terraform outputs not accessible"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 3: Nginx Proxy VM
|
|||
|
|
echo ""
|
|||
|
|
echo "3. Testing Nginx Proxy VM..."
|
|||
|
|
NGINX_IP=$(terraform output -json 2>/dev/null | jq -r '.nginx_proxy.value.public_ip // empty' || echo "")
|
|||
|
|
if [ -n "$NGINX_IP" ] && [ "$NGINX_IP" != "null" ]; then
|
|||
|
|
print_test PASS "Nginx proxy IP: $NGINX_IP"
|
|||
|
|
|
|||
|
|
# Test SSH connectivity (non-blocking)
|
|||
|
|
if timeout 5 bash -c "echo > /dev/tcp/$NGINX_IP/22" 2>/dev/null; then
|
|||
|
|
print_test PASS "Nginx proxy SSH port (22) accessible"
|
|||
|
|
else
|
|||
|
|
print_test SKIP "Nginx proxy SSH port test (may require VPN/Bastion)"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test HTTP/HTTPS ports
|
|||
|
|
if timeout 5 bash -c "echo > /dev/tcp/$NGINX_IP/80" 2>/dev/null; then
|
|||
|
|
print_test PASS "Nginx proxy HTTP port (80) accessible"
|
|||
|
|
else
|
|||
|
|
print_test INFO "Nginx proxy HTTP port (80) not yet accessible (Nginx may not be configured)"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
if timeout 5 bash -c "echo > /dev/tcp/$NGINX_IP/443" 2>/dev/null; then
|
|||
|
|
print_test PASS "Nginx proxy HTTPS port (443) accessible"
|
|||
|
|
else
|
|||
|
|
print_test INFO "Nginx proxy HTTPS port (443) not yet accessible (SSL not configured)"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Nginx proxy IP not found"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 4: Backend VMs
|
|||
|
|
echo ""
|
|||
|
|
echo "4. Testing Backend VMs..."
|
|||
|
|
BACKEND_VMS=$(terraform output -json 2>/dev/null | jq -r '.phase1_us_regions.value | to_entries[] | "\(.key):\(.value.private_ips[0])"' || echo "")
|
|||
|
|
BACKEND_COUNT=0
|
|||
|
|
if [ -n "$BACKEND_VMS" ]; then
|
|||
|
|
while IFS=: read -r region ip; do
|
|||
|
|
if [ -n "$ip" ] && [ "$ip" != "null" ]; then
|
|||
|
|
print_test PASS "Backend VM $region: $ip"
|
|||
|
|
((BACKEND_COUNT++))
|
|||
|
|
|
|||
|
|
# Test SSH connectivity (non-blocking, will likely fail without VPN)
|
|||
|
|
if timeout 3 bash -c "echo > /dev/tcp/$ip/22" 2>/dev/null; then
|
|||
|
|
print_test PASS " → SSH port (22) accessible"
|
|||
|
|
else
|
|||
|
|
print_test SKIP " → SSH port (22) requires VPN/Bastion (expected for private IPs)"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test RPC ports (non-blocking)
|
|||
|
|
if timeout 3 bash -c "echo > /dev/tcp/$ip/8545" 2>/dev/null; then
|
|||
|
|
print_test PASS " → RPC HTTP port (8545) accessible"
|
|||
|
|
else
|
|||
|
|
print_test INFO " → RPC HTTP port (8545) not accessible (Besu may not be running)"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
if timeout 3 bash -c "echo > /dev/tcp/$ip/8546" 2>/dev/null; then
|
|||
|
|
print_test PASS " → RPC WebSocket port (8546) accessible"
|
|||
|
|
else
|
|||
|
|
print_test INFO " → RPC WebSocket port (8546) not accessible (Besu may not be running)"
|
|||
|
|
fi
|
|||
|
|
fi
|
|||
|
|
done <<< "$BACKEND_VMS"
|
|||
|
|
|
|||
|
|
if [ "$BACKEND_COUNT" -eq 5 ]; then
|
|||
|
|
print_test PASS "All 5 backend VMs found"
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Expected 5 backend VMs, found $BACKEND_COUNT"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Backend VMs not found in outputs"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 5: Azure Resource Groups
|
|||
|
|
echo ""
|
|||
|
|
echo "5. Testing Azure Resource Groups..."
|
|||
|
|
if command -v az > /dev/null 2>&1; then
|
|||
|
|
RGS=$(az group list --query "[?contains(name, 'az-p-') && contains(name, '-rg-comp-001')].name" -o tsv 2>/dev/null || echo "")
|
|||
|
|
RG_COUNT=$(echo "$RGS" | grep -c . || echo "0")
|
|||
|
|
if [ "$RG_COUNT" -ge 6 ]; then
|
|||
|
|
print_test PASS "Resource groups found: $RG_COUNT"
|
|||
|
|
echo "$RGS" | while read -r rg; do
|
|||
|
|
if [ -n "$rg" ]; then
|
|||
|
|
print_test INFO " → $rg"
|
|||
|
|
fi
|
|||
|
|
done
|
|||
|
|
else
|
|||
|
|
print_test SKIP "Resource groups check (may require Azure CLI authentication)"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test SKIP "Azure CLI not available"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 6: Storage Accounts
|
|||
|
|
echo ""
|
|||
|
|
echo "6. Testing Storage Accounts..."
|
|||
|
|
STORAGE_OUTPUT=$(terraform output -json 2>/dev/null | jq -r '.storage_accounts.value.boot_diagnostics // {}' || echo "{}")
|
|||
|
|
if [ "$STORAGE_OUTPUT" != "{}" ]; then
|
|||
|
|
STORAGE_COUNT=$(echo "$STORAGE_OUTPUT" | jq -r 'length' || echo "0")
|
|||
|
|
if [ "$STORAGE_COUNT" -ge 5 ]; then
|
|||
|
|
print_test PASS "Boot diagnostics storage accounts: $STORAGE_COUNT"
|
|||
|
|
else
|
|||
|
|
print_test INFO "Storage accounts found: $STORAGE_COUNT"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test SKIP "Storage accounts check"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 7: Key Vault
|
|||
|
|
echo ""
|
|||
|
|
echo "7. Testing Key Vault..."
|
|||
|
|
KEY_VAULT=$(terraform output -json 2>/dev/null | jq -r '.key_vault_name.value // empty' || echo "")
|
|||
|
|
if [ -n "$KEY_VAULT" ] && [ "$KEY_VAULT" != "null" ]; then
|
|||
|
|
print_test PASS "Key Vault: $KEY_VAULT"
|
|||
|
|
|
|||
|
|
if command -v az > /dev/null 2>&1; then
|
|||
|
|
if az keyvault show --name "$KEY_VAULT" > /dev/null 2>&1; then
|
|||
|
|
print_test PASS "Key Vault accessible via Azure CLI"
|
|||
|
|
else
|
|||
|
|
print_test SKIP "Key Vault access check (may require authentication)"
|
|||
|
|
fi
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Key Vault not found"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 8: Monitoring Resources
|
|||
|
|
echo ""
|
|||
|
|
echo "8. Testing Monitoring Resources..."
|
|||
|
|
MONITORING=$(terraform output -json 2>/dev/null | jq -r '.monitoring.value // {}' || echo "{}")
|
|||
|
|
if [ "$MONITORING" != "{}" ]; then
|
|||
|
|
MONITORING_COUNT=$(echo "$MONITORING" | jq -r 'length' || echo "0")
|
|||
|
|
if [ "$MONITORING_COUNT" -ge 5 ]; then
|
|||
|
|
print_test PASS "Log Analytics Workspaces: $MONITORING_COUNT"
|
|||
|
|
else
|
|||
|
|
print_test INFO "Monitoring resources found: $MONITORING_COUNT"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test SKIP "Monitoring resources check"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 9: Backup Resources
|
|||
|
|
echo ""
|
|||
|
|
echo "9. Testing Backup Resources..."
|
|||
|
|
BACKUPS=$(terraform output -json 2>/dev/null | jq -r '.backups.value // {}' || echo "{}")
|
|||
|
|
if [ "$BACKUPS" != "{}" ]; then
|
|||
|
|
BACKUP_COUNT=$(echo "$BACKUPS" | jq -r 'length' || echo "0")
|
|||
|
|
if [ "$BACKUP_COUNT" -ge 5 ]; then
|
|||
|
|
print_test PASS "Recovery Services Vaults: $BACKUP_COUNT"
|
|||
|
|
else
|
|||
|
|
print_test INFO "Backup resources found: $BACKUP_COUNT"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test SKIP "Backup resources check"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 10: Network Connectivity (if accessible)
|
|||
|
|
echo ""
|
|||
|
|
echo "10. Testing Network Connectivity..."
|
|||
|
|
if [ -n "$NGINX_IP" ] && [ "$NGINX_IP" != "null" ]; then
|
|||
|
|
# Test internet connectivity from Nginx proxy (if we can reach it)
|
|||
|
|
print_test INFO "Network connectivity tests require SSH access to VMs"
|
|||
|
|
print_test INFO "Run these tests after SSH access is configured:"
|
|||
|
|
print_test INFO " - ping tests between VMs"
|
|||
|
|
print_test INFO " - DNS resolution tests"
|
|||
|
|
print_test INFO " - Service connectivity tests"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 11: Cloudflare Credentials
|
|||
|
|
echo ""
|
|||
|
|
echo "11. Testing Cloudflare Credentials..."
|
|||
|
|
ENV_FILE="$PROJECT_ROOT/.env"
|
|||
|
|
if [ -f "$ENV_FILE" ]; then
|
|||
|
|
if grep -q "CLOUDFLARE_ZONE_ID" "$ENV_FILE" && \
|
|||
|
|
grep -q "CLOUDFLARE_ACCOUNT_ID" "$ENV_FILE" && \
|
|||
|
|
grep -q "CLOUDFLARE_API_TOKEN" "$ENV_FILE"; then
|
|||
|
|
print_test PASS "Cloudflare credentials found in .env"
|
|||
|
|
|
|||
|
|
# Test API token (if curl available)
|
|||
|
|
if command -v curl > /dev/null 2>&1; then
|
|||
|
|
source <(grep -v '^#' "$ENV_FILE" | grep CLOUDFLARE_ | xargs)
|
|||
|
|
if [ -n "${CLOUDFLARE_API_TOKEN:-}" ] && [ -n "${CLOUDFLARE_ZONE_ID:-}" ]; then
|
|||
|
|
API_TEST=$(curl -s -X GET "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}" \
|
|||
|
|
-H "Authorization: Bearer ${CLOUDFLARE_API_TOKEN}" \
|
|||
|
|
-H "Content-Type: application/json" 2>/dev/null | jq -r '.success // false' || echo "false")
|
|||
|
|
if [ "$API_TEST" = "true" ]; then
|
|||
|
|
print_test PASS "Cloudflare API token valid"
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Cloudflare API token invalid or insufficient permissions"
|
|||
|
|
fi
|
|||
|
|
fi
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Cloudflare credentials incomplete in .env"
|
|||
|
|
fi
|
|||
|
|
else
|
|||
|
|
print_test SKIP "Cloudflare credentials (.env file not found)"
|
|||
|
|
fi
|
|||
|
|
|
|||
|
|
# Test 12: Scripts Availability
|
|||
|
|
echo ""
|
|||
|
|
echo "12. Testing Scripts Availability..."
|
|||
|
|
SCRIPTS=(
|
|||
|
|
"setup-cloudflare-tunnel.sh"
|
|||
|
|
"setup-besu-node.sh"
|
|||
|
|
"update-nginx-backends.sh"
|
|||
|
|
"setup-cloudflare-tunnel-backend.sh"
|
|||
|
|
"automated-cloudflare-setup.sh"
|
|||
|
|
)
|
|||
|
|
for script in "${SCRIPTS[@]}"; do
|
|||
|
|
if [ -f "$SCRIPT_DIR/$script" ] && [ -x "$SCRIPT_DIR/$script" ]; then
|
|||
|
|
print_test PASS "Script available and executable: $script"
|
|||
|
|
else
|
|||
|
|
print_test FAIL "Script missing or not executable: $script"
|
|||
|
|
fi
|
|||
|
|
done
|
|||
|
|
|
|||
|
|
# Test 13: Documentation
|
|||
|
|
echo ""
|
|||
|
|
echo "13. Testing Documentation..."
|
|||
|
|
DOCS=(
|
|||
|
|
"NEXT_STEPS_COMPLETE.md"
|
|||
|
|
"DEPLOYMENT_VERIFICATION.md"
|
|||
|
|
"CLOUDFLARE_CREDENTIALS.md"
|
|||
|
|
"CLOUDFLARE_AUTOMATION.md"
|
|||
|
|
"SETUP_COMPLETE_FINAL.md"
|
|||
|
|
)
|
|||
|
|
for doc in "${DOCS[@]}"; do
|
|||
|
|
if [ -f "$PHASE1_DIR/$doc" ]; then
|
|||
|
|
print_test PASS "Documentation available: $doc"
|
|||
|
|
else
|
|||
|
|
print_test INFO "Documentation missing: $doc"
|
|||
|
|
fi
|
|||
|
|
done
|
|||
|
|
|
|||
|
|
# Summary
|
|||
|
|
echo ""
|
|||
|
|
echo "=========================================="
|
|||
|
|
echo "Test Summary"
|
|||
|
|
echo "=========================================="
|
|||
|
|
echo -e "${GREEN}Passed:${NC} $TESTS_PASSED"
|
|||
|
|
echo -e "${RED}Failed:${NC} $TESTS_FAILED"
|
|||
|
|
echo -e "${YELLOW}Skipped:${NC} $TESTS_SKIPPED"
|
|||
|
|
echo ""
|
|||
|
|
|
|||
|
|
if [ $TESTS_FAILED -eq 0 ]; then
|
|||
|
|
echo -e "${GREEN}✓ All critical tests passed!${NC}"
|
|||
|
|
exit 0
|
|||
|
|
else
|
|||
|
|
echo -e "${RED}✗ Some tests failed. Review output above.${NC}"
|
|||
|
|
exit 1
|
|||
|
|
fi
|
|||
|
|
|