Align E2E profile workflow across scripts and runbooks

2026-03-06 08:46:55 -08:00
parent e4c9dda0fd
commit d38174dc25
18 changed files with 345 additions and 53 deletions
--- a/scripts/verify/run-full-connection-and-fastly-tests.sh
+++ b/scripts/verify/run-full-connection-and-fastly-tests.sh
@@ -91,7 +91,7 @@ echo ""
 # 5) End-to-end routing (full domain list: DNS, SSL, HTTPS, RPC where applicable)
 # When only RPC fails (edge blocks POST), treat as success so full run passes
 info "5. End-to-end routing (all domains)"
-if E2E_SUCCESS_IF_ONLY_RPC_BLOCKED=1 bash scripts/verify/verify-end-to-end-routing.sh 2>&1; then
+if E2E_SUCCESS_IF_ONLY_RPC_BLOCKED=1 bash scripts/verify/verify-end-to-end-routing.sh --profile=public 2>&1; then
  ok "E2E routing completed"
 else
  warn "E2E routing had failures (see above)"
--- a/scripts/verify/run-full-verification.sh
+++ b/scripts/verify/run-full-verification.sh
@@ -102,7 +102,7 @@ log_info "Progress: 5/$TOTAL_STEPS steps"
 log_info ""
 log_info "Step 5/$TOTAL_STEPS: End-to-End Routing Verification"
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
-if bash "$SCRIPT_DIR/verify-end-to-end-routing.sh"; then
+if bash "$SCRIPT_DIR/verify-end-to-end-routing.sh" --profile=public; then
    log_success "E2E verification complete"
 else
    log_warn "E2E verification completed with warnings"
--- a/scripts/verify/verify-end-to-end-routing.sh
+++ b/scripts/verify/verify-end-to-end-routing.sh
@@ -32,6 +32,12 @@ PUBLIC_IP="${PUBLIC_IP:-76.53.10.36}"
 PUBLIC_IP_FOURTH="${PUBLIC_IP_FOURTH:-76.53.10.40}"
 # Set ACCEPT_ANY_DNS=1 to pass DNS if domain resolves to any IP (e.g. Fastly CNAME or Cloudflare Tunnel)
 ACCEPT_ANY_DNS="${ACCEPT_ANY_DNS:-0}"
+# Use system resolver (e.g. /etc/hosts) instead of dig @8.8.8.8 — set when running from LAN with generate-e2e-hosts.sh entries
+E2E_USE_SYSTEM_RESOLVER="${E2E_USE_SYSTEM_RESOLVER:-0}"
+if [ "$E2E_USE_SYSTEM_RESOLVER" = "1" ]; then
+    ACCEPT_ANY_DNS=1
+    log_info "E2E_USE_SYSTEM_RESOLVER=1: using getent (respects /etc/hosts); ACCEPT_ANY_DNS=1"
+fi
 # When using Option B (RPC via Cloudflare Tunnel), RPC hostnames resolve to Cloudflare IPs; auto-enable if tunnel ID set
 if [ "$ACCEPT_ANY_DNS" = "0" ] && [ -n "${CLOUDFLARE_TUNNEL_ID:-}" ]; then
    ACCEPT_ANY_DNS=1
@@ -46,8 +52,8 @@ if [ "$ACCEPT_ANY_DNS" = "0" ] && [ -f "$PROJECT_ROOT/.env" ]; then
    fi
 fi

-# Expected domains and their types (all Cloudflare/DNS-facing public endpoints)
-declare -A DOMAIN_TYPES=(
+# Expected domains and their types (full combined inventory)
+declare -A DOMAIN_TYPES_ALL=(
    ["explorer.d-bis.org"]="web"
    ["rpc-http-pub.d-bis.org"]="rpc-http"
    ["rpc-ws-pub.d-bis.org"]="rpc-ws"
@@ -94,10 +100,77 @@ declare -A DOMAIN_TYPES=(
    ["dev.d-bis.org"]="web"
    ["codespaces.d-bis.org"]="web"
 )
+# Private/admin profile domains (private RPC + Fireblocks RPC only).
+declare -a PRIVATE_PROFILE_DOMAINS=(
+    "rpc-http-prv.d-bis.org"
+    "rpc-ws-prv.d-bis.org"
+    "rpc-fireblocks.d-bis.org"
+    "ws.rpc-fireblocks.d-bis.org"
+)
+
+PRIVATE_PROFILE_SET=" ${PRIVATE_PROFILE_DOMAINS[*]} "
+PROFILE="${E2E_PROFILE:-public}"
+LIST_ENDPOINTS=0
+
+for arg in "$@"; do
+    case "$arg" in
+        --list-endpoints) LIST_ENDPOINTS=1 ;;
+        --profile=*) PROFILE="${arg#*=}" ;;
+        --profile-public) PROFILE="public" ;;
+        --profile-private) PROFILE="private" ;;
+        --profile-all) PROFILE="all" ;;
+        *)
+            if [[ "$arg" != "--list-endpoints" ]]; then
+                echo "Unknown argument: $arg" >&2
+                echo "Usage: $0 [--list-endpoints] [--profile=public|private|all]" >&2
+                exit 2
+            fi
+            ;;
+    esac
+done
+
+declare -A DOMAIN_TYPES=()
+for domain in "${!DOMAIN_TYPES_ALL[@]}"; do
+    is_private=0
+    [[ "$PRIVATE_PROFILE_SET" == *" $domain "* ]] && is_private=1
+    case "$PROFILE" in
+        public)
+            [[ "$is_private" -eq 0 ]] && DOMAIN_TYPES["$domain"]="${DOMAIN_TYPES_ALL[$domain]}"
+            ;;
+        private)
+            [[ "$is_private" -eq 1 ]] && DOMAIN_TYPES["$domain"]="${DOMAIN_TYPES_ALL[$domain]}"
+            ;;
+        all)
+            DOMAIN_TYPES["$domain"]="${DOMAIN_TYPES_ALL[$domain]}"
+            ;;
+        *)
+            echo "Invalid profile: $PROFILE (expected public|private|all)" >&2
+            exit 2
+            ;;
+    esac
+done
+
 # Domains that are optional (not yet configured); no DNS = skip instead of fail. Space-separated.
-E2E_OPTIONAL_DOMAINS="${E2E_OPTIONAL_DOMAINS:-dapp.d-bis.org}"
-# Domains that are optional when any test fails (off-LAN, 502, unreachable); fail → skip so run passes. Set to empty for strict.
-E2E_OPTIONAL_WHEN_FAIL="${E2E_OPTIONAL_WHEN_FAIL:-dapp.d-bis.org mifos.d-bis.org explorer.d-bis.org dbis-admin.d-bis.org dbis-api.d-bis.org dbis-api-2.d-bis.org secure.d-bis.org sankofa.nexus www.sankofa.nexus phoenix.sankofa.nexus www.phoenix.sankofa.nexus the-order.sankofa.nexus studio.sankofa.nexus mim4u.org www.mim4u.org secure.mim4u.org training.mim4u.org rpc-http-pub.d-bis.org rpc.d-bis.org rpc2.d-bis.org rpc-http-prv.d-bis.org rpc-fireblocks.d-bis.org ws.rpc-fireblocks.d-bis.org rpc.public-0138.defi-oracle.io rpc.defi-oracle.io ws.rpc.d-bis.org rpc-ws-prv.d-bis.org ws.rpc2.d-bis.org}"
+if [[ -z "${E2E_OPTIONAL_DOMAINS:-}" ]]; then
+    if [[ "$PROFILE" == "private" ]]; then
+        E2E_OPTIONAL_DOMAINS=""
+    else
+        E2E_OPTIONAL_DOMAINS="dapp.d-bis.org"
+    fi
+else
+    E2E_OPTIONAL_DOMAINS="${E2E_OPTIONAL_DOMAINS}"
+fi
+
+# Domains that are optional when any test fails (off-LAN, 502, unreachable); fail → skip so run passes.
+if [[ -z "${E2E_OPTIONAL_WHEN_FAIL:-}" ]]; then
+    if [[ "$PROFILE" == "private" ]]; then
+        E2E_OPTIONAL_WHEN_FAIL="rpc-http-prv.d-bis.org rpc-ws-prv.d-bis.org rpc-fireblocks.d-bis.org ws.rpc-fireblocks.d-bis.org"
+    else
+        E2E_OPTIONAL_WHEN_FAIL="dapp.d-bis.org mifos.d-bis.org explorer.d-bis.org dbis-admin.d-bis.org dbis-api.d-bis.org dbis-api-2.d-bis.org secure.d-bis.org sankofa.nexus www.sankofa.nexus phoenix.sankofa.nexus www.phoenix.sankofa.nexus the-order.sankofa.nexus studio.sankofa.nexus mim4u.org www.mim4u.org secure.mim4u.org training.mim4u.org rpc-http-pub.d-bis.org rpc.d-bis.org rpc2.d-bis.org rpc.public-0138.defi-oracle.io rpc.defi-oracle.io ws.rpc.d-bis.org ws.rpc2.d-bis.org"
+    fi
+else
+    E2E_OPTIONAL_WHEN_FAIL="${E2E_OPTIONAL_WHEN_FAIL}"
+fi

 # Per-domain expected DNS IP (optional). Unset = use PUBLIC_IP.
 declare -A EXPECTED_IP=(
@@ -106,11 +179,34 @@ declare -A EXPECTED_IP=(
    ["codespaces.d-bis.org"]="$PUBLIC_IP_FOURTH"
 )

+# --list-endpoints: print selected profile endpoints and exit (no tests)
+if [[ "$LIST_ENDPOINTS" == "1" ]]; then
+    echo ""
+    echo "E2E endpoints (${#DOMAIN_TYPES[@]} total, profile: $PROFILE) — verify-end-to-end-routing.sh"
+    echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
+    echo ""
+    printf "%-40s %-12s %s\n" "Domain" "Type" "URL"
+    printf "%-40s %-12s %s\n" "------" "----" "---"
+    for domain in $(echo "${!DOMAIN_TYPES[@]}" | tr ' ' '\n' | sort); do
+        dtype="${DOMAIN_TYPES[$domain]:-unknown}"
+        if [[ "$dtype" == "rpc-http" || "$dtype" == "rpc-ws" ]]; then
+            url="https://$domain (RPC)"
+        else
+            url="https://$domain"
+        fi
+        printf "%-40s %-12s %s\n" "$domain" "$dtype" "$url"
+    done
+    echo ""
+    exit 0
+fi
+
 echo ""
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
 echo "🔍 End-to-End Routing Verification"
 echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
 echo ""
+echo "Profile: $PROFILE"
+echo ""

 E2E_RESULTS=()

@@ -126,7 +222,11 @@ test_domain() {
    
    # Test 1: DNS Resolution
    log_info "Test 1: DNS Resolution"
-    dns_result=$(dig +short "$domain" @8.8.8.8 2>/dev/null | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | head -1 || echo "")
+    if [ "${E2E_USE_SYSTEM_RESOLVER:-0}" = "1" ]; then
+        dns_result=$(getent hosts "$domain" 2>/dev/null | awk '{print $1}' | head -1 || echo "")
+    else
+        dns_result=$(dig +short "$domain" @8.8.8.8 2>/dev/null | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' | head -1 || echo "")
+    fi
    expected_ip="${EXPECTED_IP[$domain]:-$PUBLIC_IP}"
    
    if [ "$dns_result" = "$expected_ip" ]; then
@@ -372,8 +472,21 @@ cat > "$REPORT_FILE" <<EOF

 **Date**: $(date -Iseconds)
 **Public IP**: $PUBLIC_IP
+**Profile**: $PROFILE
 **Verifier**: $(whoami)

+## All endpoints ($TOTAL_TESTS)
+
+| Domain | Type | URL |
+|--------|------|-----|
+EOF
+for domain in $(echo "${!DOMAIN_TYPES[@]}" | tr ' ' '\n' | sort); do
+    dtype="${DOMAIN_TYPES[$domain]:-unknown}"
+    echo "| $domain | $dtype | https://$domain |" >> "$REPORT_FILE"
+done
+
+cat >> "$REPORT_FILE" <<EOF
+
 ## Summary

 - **Total domains tested**: $TOTAL_TESTS
@@ -383,7 +496,25 @@ cat > "$REPORT_FILE" <<EOF
 - **Skipped / optional (not configured or unreachable)**: $SKIPPED_OPTIONAL
 - **Average response time**: ${AVG_RESPONSE_TIME}s

-## Test Results by Domain
+## Results overview
+
+| Domain | Type | DNS | SSL | HTTPS | RPC |
+|--------|------|-----|-----|-------|-----|
+EOF
+
+for result in "${E2E_RESULTS[@]}"; do
+    domain=$(echo "$result" | jq -r '.domain' 2>/dev/null || echo "")
+    domain_type=$(echo "$result" | jq -r '.domain_type' 2>/dev/null || echo "")
+    dns_status=$(echo "$result" | jq -r '.tests.dns.status // "-"' 2>/dev/null || echo "-")
+    ssl_status=$(echo "$result" | jq -r '.tests.ssl.status // "-"' 2>/dev/null || echo "-")
+    https_status=$(echo "$result" | jq -r '.tests.https.status // "-"' 2>/dev/null || echo "-")
+    rpc_status=$(echo "$result" | jq -r '.tests.rpc_http.status // "-"' 2>/dev/null || echo "-")
+    echo "| $domain | $domain_type | $dns_status | $ssl_status | $https_status | $rpc_status |" >> "$REPORT_FILE"
+done
+
+cat >> "$REPORT_FILE" <<EOF
+
+## Test Results by Domain (detail)

 EOF