bea1903ac9
Some checks failed
Deploy to Phoenix / deploy (push) Has been cancelled
Co-authored-by: Cursor <cursoragent@cursor.com>
123 lines
4.5 KiB
Bash
Executable File
123 lines
4.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Run all steps to address remaining E2E 502s: backends, NPMplus proxy refresh, RPC diagnostics, optional Besu mass-fix and E2E.
|
|
# Run from project root. From LAN (or host that can reach NPMplus and Proxmox).
|
|
#
|
|
# Usage:
|
|
# ./scripts/maintenance/address-all-remaining-502s.sh
|
|
# ./scripts/maintenance/address-all-remaining-502s.sh --no-npm
|
|
# ./scripts/maintenance/address-all-remaining-502s.sh --run-besu-fix
|
|
# ./scripts/maintenance/address-all-remaining-502s.sh --e2e
|
|
# ./scripts/maintenance/address-all-remaining-502s.sh --dry-run # print steps only, no SSH/NPM
|
|
#
|
|
# Requires: SSH to r630-01. For NPMplus update: NPM_PASSWORD in .env or env.
|
|
# See: docs/00-meta/502_DEEP_DIVE_ROOT_CAUSES_AND_FIXES.md
|
|
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
|
cd "$PROJECT_ROOT"
|
|
|
|
[[ -f "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" ]] && source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true
|
|
[[ -f "${PROJECT_ROOT}/config/ip-addresses.conf" ]] && source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
|
|
|
|
SKIP_NPM=false
|
|
RUN_BESU_FIX=false
|
|
RUN_E2E=false
|
|
DRY_RUN=false
|
|
for arg in "${@:-}"; do
|
|
[[ "$arg" == "--no-npm" ]] && SKIP_NPM=true
|
|
[[ "$arg" == "--run-besu-fix" ]] && RUN_BESU_FIX=true
|
|
[[ "$arg" == "--e2e" ]] && RUN_E2E=true
|
|
[[ "$arg" == "--dry-run" ]] && DRY_RUN=true
|
|
done
|
|
|
|
echo ""
|
|
echo "=== Address all remaining 502s ==="
|
|
echo " --no-npm=$SKIP_NPM --run-besu-fix=$RUN_BESU_FIX --e2e=$RUN_E2E --dry-run=$DRY_RUN"
|
|
echo ""
|
|
|
|
if $DRY_RUN; then
|
|
echo "Would run in order:"
|
|
echo " 1. fix-all-502s-comprehensive.sh --dry-run"
|
|
$SKIP_NPM || echo " 2. update-npmplus-proxy-hosts-api.sh (if NPM_PASSWORD set)"
|
|
echo " 3. diagnose-rpc-502s.sh (save report)"
|
|
$RUN_BESU_FIX && echo " 4. fix-all-besu-nodes.sh"
|
|
$RUN_E2E && echo " 5. verify-end-to-end-routing.sh"
|
|
echo ""
|
|
exit 0
|
|
fi
|
|
|
|
# 1. Comprehensive backend fix
|
|
echo "--- Step 1/5: Fix all 502 backends (comprehensive) ---"
|
|
if bash "${SCRIPT_DIR}/fix-all-502s-comprehensive.sh" 2>/dev/null; then
|
|
echo " Backends fix completed."
|
|
else
|
|
echo " Backends fix had warnings (check output above)."
|
|
fi
|
|
echo ""
|
|
|
|
# 2. NPMplus proxy update
|
|
if ! $SKIP_NPM; then
|
|
if [ -n "${NPM_PASSWORD:-}" ]; then
|
|
echo "--- Step 2/5: Update NPMplus proxy hosts (DBIS, RPC, etc.) ---"
|
|
if bash "${PROJECT_ROOT}/scripts/nginx-proxy-manager/update-npmplus-proxy-hosts-api.sh" 2>/dev/null; then
|
|
echo " NPMplus proxy hosts updated."
|
|
else
|
|
echo " NPMplus update failed (check output). Ensure LAN and NPM_PASSWORD correct."
|
|
fi
|
|
else
|
|
echo "--- Step 2/5: NPMplus update skipped (NPM_PASSWORD not set) ---"
|
|
echo " To fix dbis-admin/secure/dbis-api 502 when backends are up: set NPM_PASSWORD in .env and run:"
|
|
echo " ./scripts/nginx-proxy-manager/update-npmplus-proxy-hosts-api.sh"
|
|
fi
|
|
else
|
|
echo "--- Step 2/5: NPMplus update skipped (--no-npm) ---"
|
|
fi
|
|
echo ""
|
|
|
|
# 3. RPC diagnostics
|
|
echo "--- Step 3/5: RPC diagnostics (2101, 2500-2505) ---"
|
|
if [ -f "${SCRIPT_DIR}/diagnose-rpc-502s.sh" ]; then
|
|
REPORT_DIR="${PROJECT_ROOT}/docs/04-configuration/verification-evidence"
|
|
mkdir -p "$REPORT_DIR"
|
|
REPORT="${REPORT_DIR}/rpc-502-diagnostics-$(date +%Y%m%d-%H%M%S).txt"
|
|
if bash "${SCRIPT_DIR}/diagnose-rpc-502s.sh" 2>&1 | tee "$REPORT"; then
|
|
echo " Diagnostics saved to: $REPORT"
|
|
fi
|
|
else
|
|
echo " (diagnose-rpc-502s.sh not found; skipping)"
|
|
fi
|
|
echo ""
|
|
|
|
# 4. Optional Besu mass-fix
|
|
if $RUN_BESU_FIX; then
|
|
echo "--- Step 4/5: Fix all Besu nodes (config + restart) ---"
|
|
if bash "${PROJECT_ROOT}/scripts/besu/fix-all-besu-nodes.sh" 2>/dev/null; then
|
|
echo " Besu fix completed. Wait 60-90s then re-run E2E."
|
|
else
|
|
echo " Besu fix had issues (check output above)."
|
|
fi
|
|
echo ""
|
|
else
|
|
echo "--- Step 4/5: Besu mass-fix skipped ---"
|
|
echo " If RPC 502 persists: ./scripts/maintenance/address-all-remaining-502s.sh --run-besu-fix"
|
|
echo " Or: ./scripts/besu/fix-all-besu-nodes.sh"
|
|
echo ""
|
|
fi
|
|
|
|
# 5. Optional E2E
|
|
if $RUN_E2E; then
|
|
echo "--- Step 5/5: E2E verification ---"
|
|
if [ -f "${PROJECT_ROOT}/scripts/verify/verify-end-to-end-routing.sh" ]; then
|
|
E2E_ACCEPT_502_INTERNAL=1 bash "${PROJECT_ROOT}/scripts/verify/verify-end-to-end-routing.sh" 2>/dev/null || true
|
|
fi
|
|
echo ""
|
|
fi
|
|
|
|
echo "=== Done ==="
|
|
echo " Next: ./scripts/verify/verify-end-to-end-routing.sh"
|
|
echo " If 502s remain: check report in docs/04-configuration/verification-evidence/rpc-502-diagnostics-*.txt"
|
|
echo " DBIS/RPC proxy: NPM_PASSWORD=xxx ./scripts/nginx-proxy-manager/update-npmplus-proxy-hosts-api.sh"
|
|
echo ""
|