Add Sankofa consolidated hub operator tooling

2026-04-13 21:41:14 -07:00
parent 49740f1a59
commit b7eebb87b3
42 changed files with 2635 additions and 14 deletions
--- a/scripts/deployment/ensure-dbis-api-trust-proxy-on-ct.sh
+++ b/scripts/deployment/ensure-dbis-api-trust-proxy-on-ct.sh
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Ensure TRUST_PROXY=1 exists in dbis_core API CT so req.ip / rate limits use X-Forwarded-For
+# when traffic arrives via NPM or the Phoenix API hub nginx.
+#
+# Usage:
+#   bash scripts/deployment/ensure-dbis-api-trust-proxy-on-ct.sh --dry-run --vmid 10150
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=10150 bash scripts/deployment/ensure-dbis-api-trust-proxy-on-ct.sh --apply --vmid 10150
+#
+# Mutations: appends lines to /opt/dbis-core/.env (backup first), restarts dbis-api.service.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+ENV_PATH="${DBIS_API_ENV_PATH:-/opt/dbis-core/.env}"
+APPLY=false
+DRY_RUN=false
+VMID="${VMID_DBIS_API:-10150}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-dbis-api-trust-proxy-on-ct ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}  env=${ENV_PATH}"
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  echo "[DRY-RUN] Would check ${ENV_PATH} on CT ${VMID}; append TRUST_PROXY=1 if missing; restart dbis-api."
+  # shellcheck disable=SC2029
+  ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc \"if [[ ! -f '${ENV_PATH}' ]]; then echo '(missing ${ENV_PATH})'; exit 0; fi; if grep -qE '^(TRUST_PROXY|TRUST_PROXY_HOPS)=' '${ENV_PATH}' 2>/dev/null; then grep -E '^(TRUST_PROXY|TRUST_PROXY_HOPS)=' '${ENV_PATH}' | sed 's/=.*/=<set>/'; else echo '(no TRUST_PROXY / TRUST_PROXY_HOPS lines yet)'; fi\""
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+REMOTE_SH="${WORKDIR}/remote.sh"
+{
+  printf 'ENV_PATH=%q\n' "$ENV_PATH"
+  cat <<'EOS'
+set -euo pipefail
+if [[ ! -f "$ENV_PATH" ]]; then
+  echo "ERROR: missing $ENV_PATH" >&2
+  exit 2
+fi
+if grep -qE '^[[:space:]]*TRUST_PROXY[[:space:]]*=[[:space:]]*(1|true|yes)' "$ENV_PATH"; then
+  echo "OK: TRUST_PROXY already enabled"
+  exit 0
+fi
+cp -a "$ENV_PATH" "${ENV_PATH}.bak.ensure-trust-proxy-$(date +%Y%m%d%H%M%S)"
+{
+  echo ""
+  echo "# Added by ensure-dbis-api-trust-proxy-on-ct.sh — NPM / Phoenix API hub"
+  echo "TRUST_PROXY=1"
+} >>"$ENV_PATH"
+systemctl restart dbis-api.service
+systemctl is-active dbis-api.service
+echo "OK: appended TRUST_PROXY=1 and restarted dbis-api"
+EOS
+} >"$REMOTE_SH"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -s" <"$REMOTE_SH"
--- a/scripts/deployment/ensure-sankofa-phoenix-7800-nft-dport-4000-guard.sh
+++ b/scripts/deployment/ensure-sankofa-phoenix-7800-nft-dport-4000-guard.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# Optional: load nftables guard on Phoenix CT (7800) — reject TCP dport 4000 from non-loopback.
+# See config/nftables/sankofa-phoenix-7800-guard-dport-4000.nft
+#
+# Usage:
+#   bash scripts/deployment/ensure-sankofa-phoenix-7800-nft-dport-4000-guard.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 bash scripts/deployment/ensure-sankofa-phoenix-7800-nft-dport-4000-guard.sh --apply --vmid 7800
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+NFT_SRC="${PROJECT_ROOT}/config/nftables/sankofa-phoenix-7800-guard-dport-4000.nft"
+REMOTE_NFT="/tmp/sankofa-phoenix-7800-guard-dport-4000.nft"
+APPLY=false
+DRY_RUN=false
+VMID="${SANKOFA_PHOENIX_VMID:-7800}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+[[ -f "$NFT_SRC" ]] || { echo "ERROR: missing $NFT_SRC" >&2; exit 2; }
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-sankofa-phoenix-7800-nft-dport-4000-guard ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}"
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  echo "[DRY-RUN] Ruleset: ${NFT_SRC}"
+  if command -v nft >/dev/null 2>&1; then
+    nft -c -f "$NFT_SRC" && echo "OK: nft -c syntax (operator host)"
+  else
+    echo "SKIP: install nftables on this workstation for local syntax check, or rely on PVE after --apply."
+  fi
+  echo "On --apply: scp ruleset to PVE → pct push into CT → ${REMOTE_NFT}; pct exec nft -f (idempotent if table exists)."
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+# pct push reads the source file from the *PVE host* filesystem, not the operator workstation.
+PVE_STAGING="/tmp/sankofa-phoenix-7800-guard-dport-4000-from-operator.nft"
+scp $SSH_OPTS "$NFT_SRC" "root@${PROXMOX_HOST}:${PVE_STAGING}"
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct push ${VMID} ${PVE_STAGING} ${REMOTE_NFT} && rm -f ${PVE_STAGING}"
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc '
+  set -euo pipefail
+  if nft list table inet sankofa_phoenix_guard >/dev/null 2>&1; then
+    echo OK: table inet sankofa_phoenix_guard already present \(skip load\)
+    nft list table inet sankofa_phoenix_guard
+    exit 0
+  fi
+  nft -f ${REMOTE_NFT}
+  echo OK: nft rules loaded
+  nft list table inet sankofa_phoenix_guard
+'"
+
+echo ""
+echo "Remove guard: pct exec ${VMID} -- nft delete table inet sankofa_phoenix_guard"
--- a/scripts/deployment/ensure-sankofa-phoenix-api-db-migrate-up-7800.sh
+++ b/scripts/deployment/ensure-sankofa-phoenix-api-db-migrate-up-7800.sh
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Run sankofa-api DB migrations (pnpm db:migrate:up) on CT 7800 against DB_* from /opt/sankofa-api/.env.
+# Uses Python to load .env so JWT_SECRET and other values with shell metacharacters do not break `source`.
+#
+# Prerequisites: DB_HOST/DB_USER/DB_PASSWORD/DB_NAME correct; Postgres reachable (e.g. VMID 7803).
+#
+# Usage:
+#   bash scripts/deployment/ensure-sankofa-phoenix-api-db-migrate-up-7800.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 bash scripts/deployment/ensure-sankofa-phoenix-api-db-migrate-up-7800.sh --apply --vmid 7800
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+API_ROOT="${SANKOFA_PHOENIX_API_ROOT:-/opt/sankofa-api}"
+APPLY=false
+DRY_RUN=false
+VMID="${SANKOFA_PHOENIX_VMID:-7800}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-sankofa-phoenix-api-db-migrate-up-7800 ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}  dir=${API_ROOT}"
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  # shellcheck disable=SC2029
+  ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc \"
+    [[ -d '${API_ROOT}' ]] || { echo 'missing ${API_ROOT}'; exit 0; }
+    test -f '${API_ROOT}/package.json' && grep -E 'db:migrate' '${API_ROOT}/package.json' | head -3
+  \""
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+REMOTE_SH="${WORKDIR}/remote.sh"
+{
+  printf 'export API_ROOT=%q\n' "$API_ROOT"
+  cat <<'EOS'
+set -euo pipefail
+cd "$API_ROOT"
+python3 <<'PY'
+import os, subprocess
+from pathlib import Path
+env = dict(os.environ)
+p = Path(".env")
+if not p.is_file():
+    raise SystemExit("ERROR: missing .env")
+for line in p.read_text().splitlines():
+    line = line.strip()
+    if not line or line.startswith("#"):
+        continue
+    if "=" not in line:
+        continue
+    k, v = line.split("=", 1)
+    env[k] = v
+r = subprocess.run(["pnpm", "run", "db:migrate:up"], cwd=str(Path.cwd()), env=env)
+raise SystemExit(r.returncode)
+PY
+echo "OK: db:migrate:up finished"
+EOS
+} >"$REMOTE_SH"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -s" <"$REMOTE_SH"
+
+echo ""
+echo "Optional: systemctl restart sankofa-api (if schema changed compatibility)."
--- a/scripts/deployment/ensure-sankofa-phoenix-api-env-lan-parity-7800.sh
+++ b/scripts/deployment/ensure-sankofa-phoenix-api-env-lan-parity-7800.sh
@@ -0,0 +1,169 @@
+#!/usr/bin/env bash
+# LAN parity for Phoenix sankofa-api (CT 7800) /opt/sankofa-api/.env:
+#   - Normalize NODE_ENV: strip all NODE_ENV= lines, then append NODE_ENV=<PHOENIX_API_NODE_ENV> (default
+#     **development**) so dotenv matches relaxed secret-validation / TLS behavior. Use
+#     PHOENIX_API_NODE_ENV=production **only** when DB_PASSWORD and KEYCLOAK_CLIENT_SECRET are each 32+
+#     chars (see src/lib/secret-validation.ts) and TLS is provisioned or tls-config terminate-at-edge patch applied.
+#   - Point DB_HOST at Sankofa Postgres (VMID 7803, default 192.168.11.53) instead of localhost.
+#   - Point KEYCLOAK_URL at Keycloak LXC (default IP_KEYCLOAK :8080).
+#   - Drop duplicate DB_PASSWORD lines (keeps first occurrence).
+#   - Append TERMINATE_TLS_AT_EDGE=1 when missing (with ensure-sankofa-phoenix-tls-config-terminate-at-edge-7800.sh).
+# Optional: if operator shell has KEYCLOAK_CLIENT_SECRET set, pass MERGE_KEYCLOAK_SECRET=1 to replace
+# placeholder on CT (never logs the secret).
+#
+# Usage:
+#   source scripts/lib/load-project-env.sh
+#   bash scripts/deployment/ensure-sankofa-phoenix-api-env-lan-parity-7800.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 bash scripts/deployment/ensure-sankofa-phoenix-api-env-lan-parity-7800.sh --apply --vmid 7800
+#   MERGE_KEYCLOAK_SECRET=1 KEYCLOAK_CLIENT_SECRET=... PROXMOX_OPS_APPLY=1 ... --apply --vmid 7800
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+ENV_PATH="${SANKOFA_PHOENIX_ENV_PATH:-/opt/sankofa-api/.env}"
+APPLY=false
+DRY_RUN=false
+VMID="${SANKOFA_PHOENIX_VMID:-7800}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+KC_IP="${IP_KEYCLOAK:-192.168.11.52}"
+PG_HOST="${PHOENIX_DB_HOST:-${DB_HOST:-192.168.11.53}}"
+KEYCLOAK_URL_EFFECTIVE="${PHOENIX_KEYCLOAK_URL:-http://${KC_IP}:8080}"
+MERGE_KC_SECRET="${MERGE_KEYCLOAK_SECRET:-0}"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-sankofa-phoenix-api-env-lan-parity-7800 ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}  env=${ENV_PATH}"
+echo "Planned: DB_HOST=${PG_HOST}  KEYCLOAK_URL=${KEYCLOAK_URL_EFFECTIVE}"
+echo "NODE_ENV: strip duplicates, then append NODE_ENV=${PHOENIX_API_NODE_ENV:-development} (override with PHOENIX_API_NODE_ENV=production when secrets meet policy)."
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  # shellcheck disable=SC2029
+  ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc \"
+    [[ -f '${ENV_PATH}' ]] || { echo 'missing ${ENV_PATH}'; exit 0; }
+    echo '--- current NODE_ENV / DB_HOST / KEYCLOAK_URL / DB_PASSWORD count:'
+    grep -nE '^(NODE_ENV|DB_HOST|KEYCLOAK_URL|DB_PASSWORD)=' '${ENV_PATH}' || true
+  \""
+  echo "Optional: MERGE_KEYCLOAK_SECRET=1 with KEYCLOAK_CLIENT_SECRET in shell to replace placeholder on CT."
+  echo "If PHOENIX_API_NODE_ENV=production: run ensure-sankofa-phoenix-tls-config-terminate-at-edge-7800.sh first (no local TLS) and use 32+ char DB_PASSWORD and KEYCLOAK_CLIENT_SECRET."
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+REMOTE_SH="${WORKDIR}/remote.sh"
+{
+  printf 'export ENV_PATH=%q\n' "$ENV_PATH"
+  printf 'export PG_HOST=%q\n' "$PG_HOST"
+  printf 'export KEYCLOAK_URL_EFFECTIVE=%q\n' "$KEYCLOAK_URL_EFFECTIVE"
+  printf 'export MERGE_KC_SECRET=%q\n' "${MERGE_KEYCLOAK_SECRET:-0}"
+  printf 'export KC_SECRET=%q\n' "${KEYCLOAK_CLIENT_SECRET:-}"
+  printf 'export PHOENIX_API_NODE_ENV=%q\n' "${PHOENIX_API_NODE_ENV:-development}"
+  printf 'export PHOENIX_DB_USER=%q\n' "${PHOENIX_DB_USER:-sankofa}"
+  printf 'export PHOENIX_DB_NAME=%q\n' "${PHOENIX_DB_NAME:-sankofa}"
+  cat <<'EOS'
+set -euo pipefail
+if [[ ! -f "$ENV_PATH" ]]; then
+  echo "ERROR: missing $ENV_PATH" >&2
+  exit 2
+fi
+cp -a "$ENV_PATH" "${ENV_PATH}.bak.lan-parity-$(date +%Y%m%d%H%M%S)"
+python3 <<PY
+import os
+import re
+from pathlib import Path
+
+p = Path(os.environ["ENV_PATH"])
+lines = p.read_text().splitlines(keepends=True)
+pg = os.environ["PG_HOST"]
+kc_url = os.environ["KEYCLOAK_URL_EFFECTIVE"]
+merge = os.environ.get("MERGE_KC_SECRET", "0") == "1"
+kc_sec = os.environ.get("KC_SECRET", "")
+node_env_val = os.environ.get("PHOENIX_API_NODE_ENV", "development")
+db_user = os.environ.get("PHOENIX_DB_USER", "sankofa")
+db_name = os.environ.get("PHOENIX_DB_NAME", "sankofa")
+
+out = []
+seen_pw = False
+seen_kc_secret = False
+for line in lines:
+    if re.match(r"^NODE_ENV=", line):
+        continue
+    if re.match(r"^DB_HOST=", line):
+        out.append(f"DB_HOST={pg}\n")
+        continue
+    if re.match(r"^DB_USER=", line):
+        out.append(f"DB_USER={db_user}\n")
+        continue
+    if re.match(r"^DB_NAME=", line):
+        out.append(f"DB_NAME={db_name}\n")
+        continue
+    if re.match(r"^KEYCLOAK_URL=", line):
+        out.append(f"KEYCLOAK_URL={kc_url}\n")
+        continue
+    if re.match(r"^DB_PASSWORD=", line):
+        if seen_pw:
+            continue
+        seen_pw = True
+        out.append(line)
+        continue
+    if merge and kc_sec and re.match(r"^KEYCLOAK_CLIENT_SECRET=", line):
+        if not seen_kc_secret:
+            out.append(f"KEYCLOAK_CLIENT_SECRET={kc_sec}\n")
+            seen_kc_secret = True
+        continue
+    out.append(line)
+
+text = "".join(out)
+if merge and kc_sec and not seen_kc_secret:
+    text += f"\nKEYCLOAK_CLIENT_SECRET={kc_sec}\n"
+
+text = text.rstrip() + (
+    "\n\n# NODE_ENV managed by ensure-sankofa-phoenix-api-env-lan-parity-7800.sh\n"
+    f"NODE_ENV={node_env_val}\n"
+)
+
+p.write_text(text)
+print("OK: wrote .env (NODE_ENV; DB_HOST; KEYCLOAK_URL; dedupe DB_PASSWORD)")
+PY
+if ! grep -qE '^TERMINATE_TLS_AT_EDGE=' "$ENV_PATH"; then
+  printf '\n# TLS terminates at NPM/hub; Apollo HTTP on loopback (requires tls-config patch script).\nTERMINATE_TLS_AT_EDGE=1\n' >>"$ENV_PATH"
+  echo "OK: appended TERMINATE_TLS_AT_EDGE=1"
+fi
+systemctl restart sankofa-api.service
+sleep 2
+systemctl is-active sankofa-api.service
+echo "OK: restarted sankofa-api"
+EOS
+} >"$REMOTE_SH"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -s" <"$REMOTE_SH"
+
+echo ""
+echo "Verify: curl https://phoenix.sankofa.nexus/health && pnpm run verify:phoenix-graphql-ws-subscription"
+echo "Check journal: journalctl -u sankofa-api -n 30 (audit DB errors should stop if schema+creds match Postgres)."
--- a/scripts/deployment/ensure-sankofa-phoenix-api-hub-graphql-ws-proxy-headers-7800.sh
+++ b/scripts/deployment/ensure-sankofa-phoenix-api-hub-graphql-ws-proxy-headers-7800.sh
@@ -0,0 +1,141 @@
+#!/usr/bin/env bash
+# Idempotent: align hub nginx location /graphql-ws on CT 7800 with install-sankofa-api-hub-nginx-on-pve.sh:
+#   Accept-Encoding cleared, proxy_buffering off, X-Real-IP / X-Forwarded-* (for TRUST_PROXY / logging).
+#
+# Usage:
+#   bash scripts/deployment/ensure-sankofa-phoenix-api-hub-graphql-ws-proxy-headers-7800.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 bash scripts/deployment/ensure-sankofa-phoenix-api-hub-graphql-ws-proxy-headers-7800.sh --apply --vmid 7800
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+SITE_CONF="${SANKOFA_PHOENIX_HUB_SITE_CONF:-/etc/sankofa-phoenix-api-hub/conf.d/site.conf}"
+APPLY=false
+DRY_RUN=false
+VMID="${SANKOFA_PHOENIX_VMID:-7800}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-sankofa-phoenix-api-hub-graphql-ws-proxy-headers-7800 ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}  conf=${SITE_CONF}"
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  echo "[DRY-RUN] Would ensure graphql-ws block has WS proxy + forwarded client headers."
+  # shellcheck disable=SC2029
+  ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc \"
+    if [[ ! -f '${SITE_CONF}' ]]; then echo 'missing ${SITE_CONF}'; exit 0; fi
+    awk '/location \\/graphql-ws/,/^    }/' '${SITE_CONF}' | head -30
+  \""
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+REMOTE_SH="${WORKDIR}/remote.sh"
+{
+  printf 'export SITE_CONF=%q\n' "$SITE_CONF"
+  cat <<'EOS'
+set -euo pipefail
+if [[ ! -f "$SITE_CONF" ]]; then
+  echo "ERROR: missing $SITE_CONF (install hub first?)" >&2
+  exit 2
+fi
+rc=0
+python3 <<'PY' || rc=$?
+import os
+import re
+import sys
+from datetime import datetime
+from pathlib import Path
+
+p = Path(os.environ["SITE_CONF"])
+t = p.read_text()
+if "location /graphql-ws" not in t:
+    print("ERROR: no location /graphql-ws in site.conf", file=sys.stderr)
+    sys.exit(2)
+
+m = re.search(r"location /graphql-ws \{([^}]*)\}", t, flags=re.DOTALL)
+if not m:
+    print("ERROR: could not parse graphql-ws block", file=sys.stderr)
+    sys.exit(2)
+orig_block = m.group(1)
+block = orig_block
+
+conn = '        proxy_set_header Connection "upgrade";\n'
+extra = (
+    '        proxy_set_header Connection "upgrade";\n'
+    '        proxy_set_header Accept-Encoding "";\n'
+    '        proxy_buffering off;\n'
+)
+if 'proxy_set_header Accept-Encoding ""' not in block or "proxy_buffering off" not in block:
+    if conn not in block:
+        print("ERROR: expected Connection upgrade line not found in graphql-ws block", file=sys.stderr)
+        sys.exit(2)
+    block = block.replace(conn, extra, 1)
+
+host_line = '        proxy_set_header Host $host;\n'
+xfwd = (
+    "        proxy_set_header X-Real-IP $remote_addr;\n"
+    "        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"
+    "        proxy_set_header X-Forwarded-Proto $scheme;\n"
+)
+if host_line in block and "proxy_set_header X-Real-IP" not in block:
+    block = block.replace(host_line, host_line + xfwd, 1)
+
+if block == orig_block:
+    print("OK: graphql-ws block already complete")
+    sys.exit(0)
+
+bak = p.with_name(p.name + ".bak.ws-proxy-" + datetime.utcnow().strftime("%Y%m%d%H%M%S"))
+bak.write_text(t)
+t2 = t[: m.start(1)] + block + t[m.end(1) :]
+p.write_text(t2)
+print("OK: patched graphql-ws block (backup " + bak.name + ")")
+sys.exit(10)
+PY
+if [[ "$rc" -eq 10 ]]; then
+  nginx -t -c /etc/sankofa-phoenix-api-hub/nginx.conf
+  if /usr/sbin/nginx -s reload -c /etc/sankofa-phoenix-api-hub/nginx.conf 2>/dev/null; then
+    echo "OK: hub nginx reloaded"
+  else
+    systemctl restart sankofa-phoenix-api-hub.service
+    systemctl is-active sankofa-phoenix-api-hub.service
+    echo "OK: hub nginx restarted"
+  fi
+elif [[ "$rc" -eq 0 ]]; then
+  echo "OK: hub nginx unchanged (already had headers)"
+else
+  exit "$rc"
+fi
+EOS
+} >"$REMOTE_SH"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -s" <"$REMOTE_SH"
+
+echo ""
+echo "Verify: bash scripts/verify/smoke-phoenix-graphql-wss-public.sh"
--- a/scripts/deployment/ensure-sankofa-phoenix-api-hub-systemd-exec-reload-7800.sh
+++ b/scripts/deployment/ensure-sankofa-phoenix-api-hub-systemd-exec-reload-7800.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# Add ExecReload to sankofa-phoenix-api-hub.service on CT 7800 if missing (matches install template).
+#
+# Usage:
+#   bash scripts/deployment/ensure-sankofa-phoenix-api-hub-systemd-exec-reload-7800.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 bash scripts/deployment/ensure-sankofa-phoenix-api-hub-systemd-exec-reload-7800.sh --apply --vmid 7800
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+UNIT="${SANKOFA_PHOENIX_HUB_UNIT:-/etc/systemd/system/sankofa-phoenix-api-hub.service}"
+APPLY=false
+DRY_RUN=false
+VMID="${SANKOFA_PHOENIX_VMID:-7800}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-sankofa-phoenix-api-hub-systemd-exec-reload-7800 ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}  unit=${UNIT}"
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  # shellcheck disable=SC2029
+  ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc \"
+    [[ -f '${UNIT}' ]] || { echo 'missing ${UNIT}'; exit 0; }
+    grep -n ExecReload '${UNIT}' || echo '(no ExecReload yet)'
+  \""
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+REMOTE_SH="${WORKDIR}/remote.sh"
+{
+  printf 'export UNIT=%q\n' "$UNIT"
+  cat <<'EOS'
+set -euo pipefail
+if [[ ! -f "$UNIT" ]]; then
+  echo "ERROR: missing $UNIT" >&2
+  exit 2
+fi
+if grep -q '^ExecReload=' "$UNIT"; then
+  echo "OK: ExecReload already present"
+  exit 0
+fi
+cp -a "$UNIT" "${UNIT}.bak.execreload-$(date +%Y%m%d%H%M%S)"
+python3 <<'PY'
+import os
+import re
+from pathlib import Path
+
+unit = Path(os.environ["UNIT"])
+text = unit.read_text()
+if re.search(r"^ExecReload=", text, flags=re.MULTILINE):
+    raise SystemExit(0)
+m = re.search(r"^(ExecStart=.*)$", text, flags=re.MULTILINE)
+if not m:
+    raise SystemExit("ERROR: ExecStart= not found in unit")
+insert = m.group(1) + "\nExecReload=/usr/sbin/nginx -s reload -c /etc/sankofa-phoenix-api-hub/nginx.conf"
+unit.write_text(text.replace(m.group(1), insert, 1))
+print("OK: inserted ExecReload after ExecStart=")
+PY
+systemctl daemon-reload
+echo "OK: daemon-reloaded (reload hub when needed: systemctl reload sankofa-phoenix-api-hub)"
+EOS
+} >"$REMOTE_SH"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -s" <"$REMOTE_SH"
--- a/scripts/deployment/ensure-sankofa-phoenix-apollo-bind-loopback-7800.sh
+++ b/scripts/deployment/ensure-sankofa-phoenix-apollo-bind-loopback-7800.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# Bind Sankofa Phoenix (Fastify) Apollo to loopback only: HOST=127.0.0.1 in /opt/sankofa-api/.env
+# so :4000 is not reachable from VLAN. Requires Tier-1 API hub nginx upstream **127.0.0.1:4000**
+# (default in install-sankofa-api-hub-nginx-on-pve.sh).
+#
+# Usage:
+#   bash scripts/deployment/ensure-sankofa-phoenix-apollo-bind-loopback-7800.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 bash scripts/deployment/ensure-sankofa-phoenix-apollo-bind-loopback-7800.sh --apply --vmid 7800
+#
+# Mutations: edits /opt/sankofa-api/.env (backup), restarts sankofa-api.service.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+ENV_PATH="${SANKOFA_PHOENIX_ENV_PATH:-/opt/sankofa-api/.env}"
+APPLY=false
+DRY_RUN=false
+VMID="${SANKOFA_PHOENIX_VMID:-7800}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-sankofa-phoenix-apollo-bind-loopback-7800 ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}  env=${ENV_PATH}"
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  echo "[DRY-RUN] Would set HOST=127.0.0.1 in ${ENV_PATH} and restart sankofa-api.service."
+  # shellcheck disable=SC2029
+  ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc \"
+    set -e
+    if [[ ! -f '${ENV_PATH}' ]]; then echo '(missing ${ENV_PATH})'; exit 0; fi
+    echo '--- HOST lines (redact values):'
+    grep -E '^HOST=' '${ENV_PATH}' 2>/dev/null | sed 's/=.*/=<set>/' || echo '(no HOST= line)'
+    echo '--- :4000 listener:'
+    command -v ss >/dev/null && ss -tlnp | grep ':4000' || true
+    echo '--- hub upstream (expect 127.0.0.1:4000):'
+    grep -A1 'upstream sankofa_phoenix_graphql' /etc/sankofa-phoenix-api-hub/conf.d/site.conf 2>/dev/null | head -3 || echo '(no hub conf)'
+  \""
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+REMOTE_SH="${WORKDIR}/remote.sh"
+{
+  printf 'ENV_PATH=%q\n' "$ENV_PATH"
+  cat <<'EOS'
+set -euo pipefail
+HUB_CONF="/etc/sankofa-phoenix-api-hub/conf.d/site.conf"
+if [[ -f "$HUB_CONF" ]] && ! grep -q "server 127.0.0.1:4000" "$HUB_CONF" 2>/dev/null; then
+  echo "ERROR: hub nginx must proxy Phoenix to 127.0.0.1:4000 (found other upstream in $HUB_CONF). Fix hub first." >&2
+  exit 2
+fi
+if [[ ! -f "$ENV_PATH" ]]; then
+  echo "ERROR: missing $ENV_PATH" >&2
+  exit 2
+fi
+if grep -qE '^[[:space:]]*HOST[[:space:]]*=[[:space:]]*127\.0\.0\.1' "$ENV_PATH"; then
+  echo "OK: HOST=127.0.0.1 already set"
+  exit 0
+fi
+cp -a "$ENV_PATH" "${ENV_PATH}.bak.loopback-$(date +%Y%m%d%H%M%S)"
+if grep -qE '^[[:space:]]*HOST[[:space:]]*=' "$ENV_PATH"; then
+  sed -i -E 's/^[[:space:]]*HOST[[:space:]]*=.*/HOST=127.0.0.1/' "$ENV_PATH"
+else
+  {
+    echo ""
+    echo "# Added by ensure-sankofa-phoenix-apollo-bind-loopback-7800.sh — VLAN cannot reach :4000; use hub :8080"
+    echo "HOST=127.0.0.1"
+  } >>"$ENV_PATH"
+fi
+systemctl restart sankofa-api.service
+systemctl is-active sankofa-api.service
+echo "OK: HOST=127.0.0.1 and sankofa-api restarted"
+EOS
+} >"$REMOTE_SH"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -s" <"$REMOTE_SH"
+
+echo ""
+echo "Post-check from operator (LAN): hub :8080 /health and GraphQL should still work; direct http://<CT_IP>:4000 should refuse from other hosts."
--- a/scripts/deployment/ensure-sankofa-phoenix-graphql-ws-remove-fastify-websocket-7800.sh
+++ b/scripts/deployment/ensure-sankofa-phoenix-graphql-ws-remove-fastify-websocket-7800.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# graphql-ws uses a standalone ws WebSocketServer on fastify.server. @fastify/websocket also
+# registers an 'upgrade' listener first; with no websocket routes it still races Fastify routing
+# against graphql-ws and can yield broken frames (clients: "Invalid WebSocket frame: RSV1 must be clear").
+# This script removes the unused plugin import + register from server.ts on CT 7800.
+#
+# Complements: ensure-sankofa-phoenix-ws-disable-permessage-deflate-7800.sh (perMessageDeflate on wss).
+#
+# Usage:
+#   bash scripts/deployment/ensure-sankofa-phoenix-graphql-ws-remove-fastify-websocket-7800.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 bash scripts/deployment/ensure-sankofa-phoenix-graphql-ws-remove-fastify-websocket-7800.sh --apply --vmid 7800
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+SERVER_TS="${SANKOFA_PHOENIX_SERVER_TS:-/opt/sankofa-api/src/server.ts}"
+APPLY=false
+DRY_RUN=false
+VMID="${SANKOFA_PHOENIX_VMID:-7800}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-sankofa-phoenix-graphql-ws-remove-fastify-websocket-7800 ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}  file=${SERVER_TS}"
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  echo "[DRY-RUN] Would remove @fastify/websocket import + register from ${SERVER_TS} if still present."
+  # shellcheck disable=SC2029
+  ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc \"
+    if [[ ! -f '${SERVER_TS}' ]]; then echo 'missing ${SERVER_TS}'; exit 0; fi
+    grep -n 'fastifyWebsocket\\|@fastify/websocket' '${SERVER_TS}' || echo '(no fastifyWebsocket references)'
+  \""
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+REMOTE_SH="${WORKDIR}/remote.sh"
+{
+  printf 'export TARGET_TS=%q\n' "$SERVER_TS"
+  cat <<'EOS'
+set -euo pipefail
+if [[ ! -f "$TARGET_TS" ]]; then
+  echo "ERROR: missing $TARGET_TS" >&2
+  exit 2
+fi
+if ! grep -q "fastifyWebsocket" "$TARGET_TS"; then
+  echo "OK: @fastify/websocket already removed from server.ts"
+  exit 0
+fi
+cp -a "$TARGET_TS" "${TARGET_TS}.bak.no-fastify-ws-$(date +%Y%m%d%H%M%S)"
+python3 <<'PY'
+from pathlib import Path
+import os
+import re
+
+p = Path(os.environ["TARGET_TS"])
+t = p.read_text()
+orig = t
+# Drop import line (with optional CRLF)
+t = re.sub(r"^import fastifyWebsocket from '@fastify/websocket'\r?\n", "", t, flags=re.MULTILINE)
+# Drop register block (comment + register)
+t = re.sub(
+    r"\n[ \t]*// Register WebSocket support\r?\n[ \t]*await fastify\.register\(fastifyWebsocket\)\r?\n",
+    "\n",
+    t,
+    count=1,
+)
+if t == orig:
+    raise SystemExit("ERROR: expected patterns not found (server.ts layout changed?)")
+p.write_text(t)
+print("OK: patched server.ts (removed @fastify/websocket)")
+PY
+systemctl restart sankofa-api.service
+systemctl is-active sankofa-api.service
+echo "OK: restarted sankofa-api"
+EOS
+} >"$REMOTE_SH"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -s" <"$REMOTE_SH"
+
+echo ""
+echo "Next: node scripts/verify/smoke-phoenix-graphql-ws-subscription.mjs"
--- a/scripts/deployment/ensure-sankofa-phoenix-tls-config-terminate-at-edge-7800.sh
+++ b/scripts/deployment/ensure-sankofa-phoenix-tls-config-terminate-at-edge-7800.sh
@@ -0,0 +1,117 @@
+#!/usr/bin/env bash
+# Apollo (Fastify) uses tls-config.ts: in NODE_ENV=production it throws if certs are missing.
+# Behind NPM + hub, TLS terminates at the edge; Apollo stays HTTP on 127.0.0.1:4000.
+# This patch allows production when TERMINATE_TLS_AT_EDGE=1 (see ensure-sankofa-phoenix-api-env-lan-parity-7800.sh).
+#
+# Patches /opt/sankofa-api/src/lib/tls-config.ts on CT 7800 (backup), restarts sankofa-api.
+#
+# Usage:
+#   bash scripts/deployment/ensure-sankofa-phoenix-tls-config-terminate-at-edge-7800.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 bash scripts/deployment/ensure-sankofa-phoenix-tls-config-terminate-at-edge-7800.sh --apply --vmid 7800
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+TLS_TS="${SANKOFA_PHOENIX_TLS_CONFIG_TS:-/opt/sankofa-api/src/lib/tls-config.ts}"
+APPLY=false
+DRY_RUN=false
+VMID="${SANKOFA_PHOENIX_VMID:-7800}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-sankofa-phoenix-tls-config-terminate-at-edge-7800 ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}  file=${TLS_TS}"
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  # shellcheck disable=SC2029
+  ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc \"
+    [[ -f '${TLS_TS}' ]] || { echo 'missing ${TLS_TS}'; exit 0; }
+    grep -n TERMINATE_TLS_AT_EDGE '${TLS_TS}' || echo '(not patched yet)'
+  \""
+  echo "Requires TERMINATE_TLS_AT_EDGE=1 in /opt/sankofa-api/.env (lan-parity script adds it)."
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+REMOTE_SH="${WORKDIR}/remote.sh"
+{
+  printf 'export TARGET_TS=%q\n' "$TLS_TS"
+  cat <<'EOS'
+set -euo pipefail
+if [[ ! -f "$TARGET_TS" ]]; then
+  echo "ERROR: missing $TARGET_TS" >&2
+  exit 2
+fi
+if grep -q 'TERMINATE_TLS_AT_EDGE' "$TARGET_TS"; then
+  echo "OK: tls-config already patched"
+  exit 0
+fi
+cp -a "$TARGET_TS" "${TARGET_TS}.bak.terminate-at-edge-$(date +%Y%m%d%H%M%S)"
+python3 <<'PY'
+from pathlib import Path
+import os
+p = Path(os.environ["TARGET_TS"])
+t = p.read_text()
+old1 = """  if (!fs.existsSync(certPath)) {
+    if (process.env.NODE_ENV === 'production') {
+      throw new Error(`TLS certificate not found: ${certPath}`)
+    }
+    logger.warn(`TLS certificate not found: ${certPath} - using HTTP only`)
+  }"""
+new1 = """  if (!fs.existsSync(certPath)) {
+    if (process.env.NODE_ENV === 'production' && process.env.TERMINATE_TLS_AT_EDGE !== '1') {
+      throw new Error(`TLS certificate not found: ${certPath}`)
+    }
+    logger.warn(`TLS certificate not found: ${certPath} - using HTTP only`)
+  }"""
+old2 = """  if (!fs.existsSync(keyPath)) {
+    if (process.env.NODE_ENV === 'production') {
+      throw new Error(`TLS key not found: ${keyPath}`)
+    }
+    logger.warn(`TLS key not found: ${keyPath} - using HTTP only`)
+  }"""
+new2 = """  if (!fs.existsSync(keyPath)) {
+    if (process.env.NODE_ENV === 'production' && process.env.TERMINATE_TLS_AT_EDGE !== '1') {
+      throw new Error(`TLS key not found: ${keyPath}`)
+    }
+    logger.warn(`TLS key not found: ${keyPath} - using HTTP only`)
+  }"""
+if old1 not in t or old2 not in t:
+    raise SystemExit("ERROR: tls-config.ts pattern not found (file changed?)")
+p.write_text(t.replace(old1, new1, 1).replace(old2, new2, 1))
+print("OK: patched tls-config.ts")
+PY
+systemctl restart sankofa-api.service
+sleep 2
+systemctl is-active sankofa-api.service
+echo "OK: restarted sankofa-api"
+EOS
+} >"$REMOTE_SH"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -s" <"$REMOTE_SH"
--- a/scripts/deployment/ensure-sankofa-phoenix-websocket-ts-import-logger-7800.sh
+++ b/scripts/deployment/ensure-sankofa-phoenix-websocket-ts-import-logger-7800.sh
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+# websocket.ts uses logger in onDisconnect/onError but must import it; otherwise disconnects
+# crash the whole process (ReferenceError: logger is not defined) → 502 on /graphql-ws.
+#
+# Patches /opt/sankofa-api/src/services/websocket.ts on CT 7800 (backup), restarts sankofa-api.
+#
+# Usage:
+#   bash scripts/deployment/ensure-sankofa-phoenix-websocket-ts-import-logger-7800.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 bash scripts/deployment/ensure-sankofa-phoenix-websocket-ts-import-logger-7800.sh --apply --vmid 7800
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+WS_TS="${SANKOFA_PHOENIX_WEBSOCKET_TS:-/opt/sankofa-api/src/services/websocket.ts}"
+APPLY=false
+DRY_RUN=false
+VMID="${SANKOFA_PHOENIX_VMID:-7800}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-sankofa-phoenix-websocket-ts-import-logger-7800 ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}  file=${WS_TS}"
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  echo "[DRY-RUN] Would ensure import { logger } from '../lib/logger' in ${WS_TS}."
+  # shellcheck disable=SC2029
+  ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc \"
+    if [[ ! -f '${WS_TS}' ]]; then echo 'missing ${WS_TS}'; exit 0; fi
+    head -15 '${WS_TS}' | sed 's/^/    /'
+    grep -n \"from '../lib/logger'\" '${WS_TS}' || echo '(no logger import yet)'
+  \""
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+REMOTE_SH="${WORKDIR}/remote.sh"
+{
+  printf 'export TARGET_TS=%q\n' "$WS_TS"
+  cat <<'EOS'
+set -euo pipefail
+if [[ ! -f "$TARGET_TS" ]]; then
+  echo "ERROR: missing $TARGET_TS" >&2
+  exit 2
+fi
+if grep -q "from '../lib/logger'" "$TARGET_TS"; then
+  echo "OK: logger import already present"
+  exit 0
+fi
+cp -a "$TARGET_TS" "${TARGET_TS}.bak.logger-import-$(date +%Y%m%d%H%M%S)"
+python3 <<'PY'
+from pathlib import Path
+import os
+p = Path(os.environ["TARGET_TS"])
+t = p.read_text()
+needle = "import { FastifyRequest } from 'fastify'\n"
+ins = needle + "\nimport { logger } from '../lib/logger'\n"
+if needle not in t:
+    raise SystemExit("ERROR: FastifyRequest import anchor not found")
+if "from '../lib/logger'" in t:
+    raise SystemExit(0)
+p.write_text(t.replace(needle, ins, 1))
+print("OK: inserted logger import")
+PY
+systemctl restart sankofa-api.service
+systemctl is-active sankofa-api.service
+echo "OK: restarted sankofa-api"
+EOS
+} >"$REMOTE_SH"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -s" <"$REMOTE_SH"
+
+echo ""
+echo "Verify: node scripts/verify/smoke-phoenix-graphql-ws-subscription.mjs"
--- a/scripts/deployment/ensure-sankofa-phoenix-ws-disable-permessage-deflate-7800.sh
+++ b/scripts/deployment/ensure-sankofa-phoenix-ws-disable-permessage-deflate-7800.sh
@@ -0,0 +1,105 @@
+#!/usr/bin/env bash
+# Disable permessage-deflate on Phoenix graphql-ws (WebSocketServer in ws). If clients still
+# see "RSV1 must be clear", remove the unused @fastify/websocket upgrade listener:
+#   ensure-sankofa-phoenix-graphql-ws-remove-fastify-websocket-7800.sh
+#
+# Patches /opt/sankofa-api/src/services/websocket.ts on CT 7800 (backup first), restarts sankofa-api.
+#
+# Usage:
+#   bash scripts/deployment/ensure-sankofa-phoenix-ws-disable-permessage-deflate-7800.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 bash scripts/deployment/ensure-sankofa-phoenix-ws-disable-permessage-deflate-7800.sh --apply --vmid 7800
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+WS_TS="${SANKOFA_PHOENIX_WEBSOCKET_TS:-/opt/sankofa-api/src/services/websocket.ts}"
+APPLY=false
+DRY_RUN=false
+VMID="${SANKOFA_PHOENIX_VMID:-7800}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+
+echo "=== ensure-sankofa-phoenix-ws-disable-permessage-deflate-7800 ==="
+echo "PVE: root@${PROXMOX_HOST}  VMID=${VMID}  file=${WS_TS}"
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  echo "[DRY-RUN] Would inject perMessageDeflate: false into WebSocketServer options if missing."
+  # shellcheck disable=SC2029
+  ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -lc \"
+    if [[ ! -f '${WS_TS}' ]]; then echo 'missing ${WS_TS}'; exit 0; fi
+    if grep -q 'perMessageDeflate' '${WS_TS}'; then grep -n 'perMessageDeflate' '${WS_TS}' | head -5; else echo '(no perMessageDeflate yet)'; fi
+    sed -n '12,22p' '${WS_TS}' | sed 's/^/    /'
+  \""
+  echo "For apply: --apply and PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="$(mktemp -d)"
+trap 'rm -rf "$WORKDIR"' EXIT
+REMOTE_SH="${WORKDIR}/remote.sh"
+{
+  printf 'export TARGET_TS=%q\n' "$WS_TS"
+  cat <<'EOS'
+set -euo pipefail
+if [[ ! -f "$TARGET_TS" ]]; then
+  echo "ERROR: missing $TARGET_TS" >&2
+  exit 2
+fi
+if grep -q 'perMessageDeflate' "$TARGET_TS"; then
+  echo "OK: perMessageDeflate already present"
+  exit 0
+fi
+cp -a "$TARGET_TS" "${TARGET_TS}.bak.ws-deflate-$(date +%Y%m%d%H%M%S)"
+python3 <<'PY'
+from pathlib import Path
+import os
+p = Path(os.environ["TARGET_TS"])
+t = p.read_text()
+old = """  const wss = new WebSocketServer({
+    server: httpServer,
+    path,
+  })"""
+new = """  const wss = new WebSocketServer({
+    server: httpServer,
+    path,
+    perMessageDeflate: false,
+  })"""
+if old not in t:
+    raise SystemExit("ERROR: WebSocketServer block not found or already modified")
+p.write_text(t.replace(old, new, 1))
+print("OK: patched WebSocketServer")
+PY
+systemctl restart sankofa-api.service
+systemctl is-active sankofa-api.service
+echo "OK: restarted sankofa-api"
+EOS
+} >"$REMOTE_SH"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "pct exec ${VMID} -- bash -s" <"$REMOTE_SH"
+
+echo ""
+echo "Next: pnpm exec node scripts/verify/smoke-phoenix-graphql-ws-subscription.mjs"
--- a/scripts/deployment/install-sankofa-api-hub-nginx-on-pve.sh
+++ b/scripts/deployment/install-sankofa-api-hub-nginx-on-pve.sh
@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+# Install Tier-1 Phoenix API hub (nginx :8080) on an existing LXC.
+#   /graphql* → SANKOFA_API_HUB_UPSTREAM_PHOENIX (default 127.0.0.1:4000)
+#   /api*, /api-docs → SANKOFA_API_HUB_UPSTREAM_DBIS (default IP_DBIS_API:3000)
+#
+# Usage:
+#   ./scripts/deployment/install-sankofa-api-hub-nginx-on-pve.sh --dry-run --vmid 7800
+#   PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 ./scripts/deployment/install-sankofa-api-hub-nginx-on-pve.sh --apply --vmid 7800
+#
+# Requires: SSH root@PROXMOX_HOST; pct; Debian/Ubuntu CT (apt-get). Does not change NPM.
+# Upstream Phoenix should be 127.0.0.1:4000 when Apollo binds loopback (see ensure-sankofa-phoenix-apollo-bind-loopback-7800.sh).
+# PROXMOX_HOST must be the PVE node that hosts this CT (where `pct exec` works), not the CT IP.
+#
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/proxmox-production-guard.sh"
+
+PROXMOX_HOST="${PROXMOX_HOST:-${PROXMOX_HOST_R630_01:-192.168.11.11}}"
+SSH_OPTS="-o BatchMode=yes -o ConnectTimeout=15 -o StrictHostKeyChecking=accept-new"
+APPLY=false
+VMID=""
+DRY_RUN=false
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --apply) APPLY=true ;;
+    --dry-run) DRY_RUN=true ;;
+    --vmid) VMID="${2:?}"; shift ;;
+    *) echo "Unknown arg: $1" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+[[ -n "$VMID" ]] || { echo "ERROR: --vmid <n> required (e.g. 7800)." >&2; exit 2; }
+
+PH_UP="${SANKOFA_API_HUB_UPSTREAM_PHOENIX:-127.0.0.1:4000}"
+DB_UP="${SANKOFA_API_HUB_UPSTREAM_DBIS:-${IP_DBIS_API:-192.168.11.155}:3000}"
+
+echo "=== install-sankofa-api-hub-nginx-on-pve ==="
+echo "PVE:  root@${PROXMOX_HOST}  VMID=${VMID}"
+echo "Upstream Phoenix: ${PH_UP}  dbis_core: ${DB_UP}"
+if command -v get_host_for_vmid >/dev/null 2>&1; then
+  echo "get_host_for_vmid ${VMID}: $(get_host_for_vmid "${VMID}")"
+fi
+echo ""
+
+if $DRY_RUN || ! $APPLY; then
+  echo "[DRY-RUN] Would: ssh → pct push → pct exec (apt nginx, conf under /etc/sankofa-phoenix-api-hub/, systemd sankofa-phoenix-api-hub)."
+  $APPLY || true
+  echo "For live install: --apply + PROXMOX_OPS_APPLY=1 + PROXMOX_OPS_ALLOWED_VMIDS=${VMID}"
+  exit 0
+fi
+
+if ! pguard_require_apply_flag true; then
+  echo "Refused: set PROXMOX_OPS_APPLY=1" >&2
+  exit 3
+fi
+if ! pguard_vmid_allowed "$VMID"; then
+  exit 3
+fi
+
+WORKDIR="${TMPDIR:-/tmp}/sankofa-hub-pkg-$$"
+mkdir -p "$WORKDIR"
+cleanup() { rm -rf "$WORKDIR"; }
+trap cleanup EXIT
+
+GEN="${WORKDIR}/site.conf"
+MAIN="${WORKDIR}/nginx.conf"
+UNIT="${WORKDIR}/sankofa-phoenix-api-hub.service"
+
+cat >"$GEN" <<EOF
+upstream sankofa_phoenix_graphql {
+    server ${PH_UP};
+    keepalive 32;
+}
+upstream dbis_core_rest {
+    server ${DB_UP};
+    keepalive 32;
+}
+server {
+    listen 8080;
+    server_name _;
+    location = /health {
+        default_type application/json;
+        return 200 '{"status":"hub-up","vmid":"${VMID}"}';
+    }
+    location /graphql {
+        proxy_http_version 1.1;
+        proxy_set_header Host \$host;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto \$scheme;
+        proxy_read_timeout 300s;
+        proxy_pass http://sankofa_phoenix_graphql;
+    }
+    location /graphql-ws {
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade \$http_upgrade;
+        proxy_set_header Connection "upgrade";
+        proxy_set_header Accept-Encoding "";
+        proxy_buffering off;
+        proxy_set_header Host \$host;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto \$scheme;
+        proxy_read_timeout 3600s;
+        proxy_pass http://sankofa_phoenix_graphql;
+    }
+    location /api/ {
+        proxy_http_version 1.1;
+        proxy_set_header Host \$host;
+        proxy_set_header X-Real-IP \$remote_addr;
+        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto \$scheme;
+        proxy_pass http://dbis_core_rest;
+    }
+    location /api-docs {
+        proxy_pass http://dbis_core_rest;
+    }
+}
+EOF
+
+cat >"$MAIN" <<'MAINEOF'
+user www-data;
+worker_processes auto;
+error_log  /var/log/nginx/sankofa-api-hub-error.log warn;
+pid        /tmp/sankofa-api-hub-nginx.pid;
+events { worker_connections 1024; }
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+    sendfile        on;
+    keepalive_timeout  65;
+    include /etc/sankofa-phoenix-api-hub/conf.d/*.conf;
+}
+MAINEOF
+
+cat >"$UNIT" <<'UNITEOF'
+[Unit]
+Description=Sankofa Phoenix API hub (nginx :8080)
+After=network.target
+
+[Service]
+Type=simple
+ExecStartPre=/usr/sbin/nginx -t -c /etc/sankofa-phoenix-api-hub/nginx.conf
+ExecStart=/usr/sbin/nginx -g "daemon off;" -c /etc/sankofa-phoenix-api-hub/nginx.conf
+ExecReload=/usr/sbin/nginx -s reload -c /etc/sankofa-phoenix-api-hub/nginx.conf
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target
+UNITEOF
+
+REMOTE="/tmp/sankofa-hub-ssh-$$"
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" "mkdir -p ${REMOTE}"
+scp $SSH_OPTS "$GEN" "$MAIN" "$UNIT" "root@${PROXMOX_HOST}:${REMOTE}/"
+
+ssh $SSH_OPTS "root@${PROXMOX_HOST}" bash -s -- "$VMID" "$REMOTE" <<'REMOTE'
+set -euo pipefail
+VMID="$1"
+REMOTE="$2"
+pct push "${VMID}" "${REMOTE}/site.conf" /tmp/sankofa-hub-site.conf
+pct push "${VMID}" "${REMOTE}/nginx.conf" /tmp/sankofa-hub-nginx-main.conf
+pct push "${VMID}" "${REMOTE}/sankofa-phoenix-api-hub.service" /tmp/sankofa-phoenix-api-hub.service
+rm -rf "${REMOTE}"
+pct exec "${VMID}" -- bash -lc '
+set -euo pipefail
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -qq
+apt-get install -y -qq nginx
+rm -f /etc/nginx/sites-enabled/default
+mkdir -p /etc/sankofa-phoenix-api-hub/conf.d
+install -m 0644 /tmp/sankofa-hub-site.conf /etc/sankofa-phoenix-api-hub/conf.d/site.conf
+install -m 0644 /tmp/sankofa-hub-nginx-main.conf /etc/sankofa-phoenix-api-hub/nginx.conf
+install -m 0644 /tmp/sankofa-phoenix-api-hub.service /etc/systemd/system/sankofa-phoenix-api-hub.service
+rm -f /tmp/sankofa-hub-site.conf /tmp/sankofa-hub-nginx-main.conf /tmp/sankofa-phoenix-api-hub.service
+nginx -t -c /etc/sankofa-phoenix-api-hub/nginx.conf
+systemctl stop nginx 2>/dev/null || true
+systemctl disable nginx 2>/dev/null || true
+systemctl daemon-reload
+systemctl enable sankofa-phoenix-api-hub
+systemctl restart sankofa-phoenix-api-hub
+systemctl is-active sankofa-phoenix-api-hub
+'
+REMOTE
+
+echo ""
+echo "Smoke (Phoenix CT LAN IP, port 8080):"
+echo "  curl -sS http://${IP_SANKOFA_PHOENIX_API}:8080/health"
+echo "Next: NPM maintenance — point phoenix.sankofa.nexus upstream to :8080 if desired."
--- a/scripts/deployment/plan-phoenix-apollo-port-4000-restrict-7800.sh
+++ b/scripts/deployment/plan-phoenix-apollo-port-4000-restrict-7800.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Read-only: show how Apollo :4000 is bound on Phoenix LXC (7800) and safe options to restrict it
+# after NPM uses the Tier-1 hub on :8080 only. Does not change iptables or Phoenix config.
+#
+# Usage:
+#   bash scripts/deployment/plan-phoenix-apollo-port-4000-restrict-7800.sh [--ssh]
+#   --ssh   run ss on CT via PVE (needs SSH root@PROXMOX_HOST + pct)
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+
+VMID="${PHOENIX_API_VMID:-7800}"
+PROXMOX_HOST="${PROXMOX_HOST:-$(get_host_for_vmid "$VMID")}"
+SSH=false
+[[ "${1:-}" == "--ssh" ]] && SSH=true
+
+echo "=== plan-phoenix-apollo-port-4000-restrict-7800 (read-only) ==="
+echo "VMID=${VMID}  PVE=${PROXMOX_HOST}"
+echo ""
+echo "Goal: after NPM → :8080 hub, avoid LAN-wide exposure of raw Apollo :4000 (bypass hub CORS/WAF intent)."
+echo ""
+echo "1) **Bind Apollo to loopback** (preferred — Fastify uses HOST from env):"
+echo "     Run: PROXMOX_OPS_APPLY=1 PROXMOX_OPS_ALLOWED_VMIDS=7800 \\"
+echo "       bash scripts/deployment/ensure-sankofa-phoenix-apollo-bind-loopback-7800.sh --apply --vmid 7800"
+echo "     Hub upstream must stay server 127.0.0.1:4000; restart order: .env then sankofa-api.service."
+echo ""
+echo "2) **Host firewall on CT ${VMID}** (nftables/iptables): allow TCP :4000 only from 127.0.0.1 (and"
+echo "     same-container processes). Drop from 192.168.11.0/24. Add an explicit operator allowlist"
+echo "     for break-glass IPs if needed. Dry-run with nft -c; apply in a maintenance window."
+echo ""
+echo "3) Document the chosen option in docs/04-configuration/ALL_VMIDS_ENDPOINTS.md."
+echo ""
+
+if $SSH; then
+  echo "--- Live listeners (pct exec ${VMID}):"
+  # shellcheck disable=SC2029
+  ssh -o BatchMode=yes -o ConnectTimeout=12 "root@${PROXMOX_HOST}" \
+    "pct exec ${VMID} -- bash -lc \"command -v ss >/dev/null && ss -tlnp | grep -E ':4000|:8080' || ( netstat -tlnp 2>/dev/null | grep -E ':4000|:8080' ) || echo '(ss/netstat unavailable)'\"" \
+    || echo "SSH/pct failed (skip if not on LAN)."
+else
+  echo "Re-run with --ssh on LAN to print :4000 / :8080 listeners from the CT."
+fi
--- a/scripts/deployment/plan-sankofa-consolidated-hub-cutover.sh
+++ b/scripts/deployment/plan-sankofa-consolidated-hub-cutover.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Read-only plan: consolidated web hub + API hub cutover reminders.
+# Does not SSH, mutate NPM, or change Proxmox. Load dotenv when available.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck disable=SC1090
+[[ -f "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" ]] && source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" || true
+
+echo "=== Sankofa consolidated hub — operator plan (dry text only) ==="
+echo ""
+echo "1) Validate example nginx syntax on operator workstation or CI:"
+echo "     bash scripts/verify/check-sankofa-consolidated-nginx-examples.sh"
+echo ""
+echo "2) After provisioning hub LXCs, set in repo .env (overrides ip-addresses defaults):"
+echo "     IP_SANKOFA_WEB_HUB=<lan-ip>"
+echo "     SANKOFA_WEB_HUB_PORT=80"
+echo "     IP_SANKOFA_PHOENIX_API_HUB=<lan-ip>"
+echo "     SANKOFA_PHOENIX_API_HUB_PORT=8080   # example when nginx listens for NPM"
+echo ""
+echo "3) Install configs on hub CT (paths match systemd examples):"
+echo "     /etc/sankofa-web-hub/nginx.conf        ← config/nginx/sankofa-hub-main.example.conf"
+echo "     /etc/sankofa-web-hub/conf.d/site.conf  ← sankofa-non-chain-frontends.example.conf (tuned)"
+echo "     /etc/sankofa-phoenix-api-hub/nginx.conf + conf.d/ ← sankofa-api-hub-main + phoenix-api-hub"
+echo ""
+echo "4) Point upstream blocks in API hub nginx to real Phoenix (:${SANKOFA_PHOENIX_API_PORT:-4000}) and dbis_core (:3000 or your LAN)."
+echo ""
+echo "5) NPM: point affected FQDNs to IP_SANKOFA_WEB_HUB; for phoenix.sankofa.nexus set SANKOFA_NPM_PHOENIX_PORT=8080 (and IP_SANKOFA_NPM_PHOENIX_API if hub IP differs) then run scripts/nginx-proxy-manager/update-npmplus-proxy-hosts-api.sh"
+echo "    Full sequence: docs/03-deployment/SANKOFA_API_HUB_NPM_CUTOVER_AND_POST_CUTOVER_RUNBOOK.md"
+echo ""
+echo "6) Update docs/04-configuration/ALL_VMIDS_ENDPOINTS.md and get_host_for_vmid when VMIDs are retired."
+echo ""
+echo "Current resolved defaults (from config):"
+echo "  IP_SANKOFA_WEB_HUB=${IP_SANKOFA_WEB_HUB:-unset}"
+echo "  IP_SANKOFA_PHOENIX_API_HUB=${IP_SANKOFA_PHOENIX_API_HUB:-unset}"
+echo ""
+echo "Architecture: docs/02-architecture/SANKOFA_PHOENIX_CONSOLIDATED_FRONTEND_AND_API.md"
+echo "r630-01 goal (phases + placement): docs/03-deployment/SANKOFA_R630_01_CONSOLIDATION_AND_HUB_PLACEMENT_GOAL.md"
--- a/scripts/env.r630-01.example
+++ b/scripts/env.r630-01.example
@@ -47,6 +47,8 @@ KEYCLOAK_MULTI_REALM=false
 # API Configuration
 API_HOST=192.168.11.50
 API_PORT=4000
+# When Tier-1 hub terminates public GraphQL, bind Fastify to loopback on the Phoenix CT (7800):
+# HOST=127.0.0.1
 NEXT_PUBLIC_GRAPHQL_ENDPOINT=http://192.168.11.50:4000/graphql
 NEXT_PUBLIC_GRAPHQL_WS_ENDPOINT=ws://192.168.11.50:4000/graphql-ws
 JWT_SECRET=CHANGE_THIS_JWT_SECRET_IN_PRODUCTION
--- a/scripts/lib/load-project-env.sh
+++ b/scripts/lib/load-project-env.sh
@@ -5,7 +5,7 @@
 # Usage: source "${SCRIPT_DIR}/lib/load-project-env.sh"
 #
 # Env precedence (first wins): 1) .env 2) config/ip-addresses.conf 3) smom-dbis-138/.env 4) dbis_core config
-# Version: 2026-04-12 (get_host_for_vmid aligned with 2026-04-09 live Chain 138 placement)
+# Version: 2026-04-13 (get_host_for_vmid: explicit Sankofa 7800–7806 on r630-01)

 [[ -n "${PROJECT_ROOT:-}" ]] || PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
 export PROJECT_ROOT
@@ -128,12 +128,13 @@ export SMOM_DIR="${SMOM_DBIS_138_DIR:-${PROJECT_ROOT}/smom-dbis-138}"
 export DBIS_CORE_DIR="${DBIS_CORE_DIR:-${PROJECT_ROOT}/dbis_core}"

 # VMID -> Proxmox host (for pct/qm operations)
-# Covers: DBIS (101xx), RPC (2101-2103, 2201, 2301, etc.), Blockscout (5000), CCIP (5400-5476), NPMplus (10233, 10234)
+# Covers: DBIS (101xx), RPC (2101-2103, 2201, 2301, etc.), Blockscout (5000), CCIP (5400-5476), NPMplus (10233, 10234), Sankofa stack (7800–7806)
 # Live placement (2026-04-09): validators 1003/1004, sentries 1503-1510, and RPCs 2102, 2301, 2304, 2400, 2402, 2403 on r630-03;
 # RPCs 2201, 2303, 2305-2308, 2401 on r630-02; 2101 + 2103 remain on r630-01 — see ALL_VMIDS_ENDPOINTS.md
 get_host_for_vmid() {
    local vmid="$1"
    case "$vmid" in
+        7800|7801|7802|7803|7804|7805|7806) echo "${PROXMOX_HOST_R630_01}";;
        10130|10150|10151|106|107|108|10000|10001|10020|10100|10101|10120|10203|10233|10235) echo "${PROXMOX_HOST_R630_01}";;
        1000|1001|1002|1500|1501|1502|2101|2103) echo "${PROXMOX_HOST_R630_01}";;
        1003|1004|1503|1504|1505|1506|1507|1508|1509|1510|2102|2301|2304|2400|2402|2403) echo "${PROXMOX_HOST_R630_03}";;
--- a/scripts/nginx-proxy-manager/update-npmplus-proxy-hosts-api.sh
+++ b/scripts/nginx-proxy-manager/update-npmplus-proxy-hosts-api.sh
@@ -468,6 +468,10 @@ IP_SANKOFA_PORTAL="${IP_SANKOFA_PORTAL:-${IP_SERVICE_51:-192.168.11.51}}"
 IP_SANKOFA_PHOENIX_API="${IP_SANKOFA_PHOENIX_API:-${IP_SERVICE_50:-192.168.11.50}}"
 SANKOFA_PORTAL_PORT="${SANKOFA_PORTAL_PORT:-3000}"
 SANKOFA_PHOENIX_API_PORT="${SANKOFA_PHOENIX_API_PORT:-4000}"
+# NPM upstream for phoenix.sankofa.nexus (default same IP as Phoenix CT, port SANKOFA_PHOENIX_API_PORT, usually :4000).
+# Set SANKOFA_NPM_PHOENIX_PORT=8080 when NPM should hit Tier-1 API hub nginx. Optional IP_SANKOFA_NPM_PHOENIX_API if hub is on another CT.
+IP_SANKOFA_NPM_PHOENIX_API="${IP_SANKOFA_NPM_PHOENIX_API:-${IP_SANKOFA_PHOENIX_API}}"
+SANKOFA_NPM_PHOENIX_PORT="${SANKOFA_NPM_PHOENIX_PORT:-${SANKOFA_PHOENIX_API_PORT}}"
 # Resolved before portal/SSO rows so dash can default to client SSO stack
 IP_SANKOFA_CLIENT_SSO="${IP_SANKOFA_CLIENT_SSO:-${IP_SANKOFA_PORTAL}}"
 SANKOFA_CLIENT_SSO_PORT="${SANKOFA_CLIENT_SSO_PORT:-${SANKOFA_PORTAL_PORT}}"
@@ -475,8 +479,9 @@ IP_SANKOFA_PUBLIC_WEB="${IP_SANKOFA_PUBLIC_WEB:-${IP_SANKOFA_PORTAL}}"
 SANKOFA_PUBLIC_WEB_PORT="${SANKOFA_PUBLIC_WEB_PORT:-${SANKOFA_PORTAL_PORT}}"
 update_proxy_host "sankofa.nexus" "http://${IP_SANKOFA_PUBLIC_WEB}:${SANKOFA_PUBLIC_WEB_PORT}" false false && updated_count=$((updated_count + 1)) || { add_proxy_host "sankofa.nexus" "${IP_SANKOFA_PUBLIC_WEB}" "${SANKOFA_PUBLIC_WEB_PORT}" false false && updated_count=$((updated_count + 1)); } || failed_count=$((failed_count + 1))
 update_proxy_host "www.sankofa.nexus" "http://${IP_SANKOFA_PUBLIC_WEB}:${SANKOFA_PUBLIC_WEB_PORT}" false false "https://sankofa.nexus" && updated_count=$((updated_count + 1)) || { add_proxy_host "www.sankofa.nexus" "${IP_SANKOFA_PUBLIC_WEB}" "${SANKOFA_PUBLIC_WEB_PORT}" false false "https://sankofa.nexus" && updated_count=$((updated_count + 1)); } || failed_count=$((failed_count + 1))
-update_proxy_host "phoenix.sankofa.nexus" "http://${IP_SANKOFA_PHOENIX_API}:${SANKOFA_PHOENIX_API_PORT}" false false && updated_count=$((updated_count + 1)) || { add_proxy_host "phoenix.sankofa.nexus" "${IP_SANKOFA_PHOENIX_API}" "${SANKOFA_PHOENIX_API_PORT}" false false && updated_count=$((updated_count + 1)); } || failed_count=$((failed_count + 1))
-update_proxy_host "www.phoenix.sankofa.nexus" "http://${IP_SANKOFA_PHOENIX_API}:${SANKOFA_PHOENIX_API_PORT}" false false "https://phoenix.sankofa.nexus" && updated_count=$((updated_count + 1)) || { add_proxy_host "www.phoenix.sankofa.nexus" "${IP_SANKOFA_PHOENIX_API}" "${SANKOFA_PHOENIX_API_PORT}" false false "https://phoenix.sankofa.nexus" && updated_count=$((updated_count + 1)); } || failed_count=$((failed_count + 1))
+# WebSocket true — GraphQL subscriptions (/graphql-ws) via hub or direct Apollo
+update_proxy_host "phoenix.sankofa.nexus" "http://${IP_SANKOFA_NPM_PHOENIX_API}:${SANKOFA_NPM_PHOENIX_PORT}" true false && updated_count=$((updated_count + 1)) || { add_proxy_host "phoenix.sankofa.nexus" "${IP_SANKOFA_NPM_PHOENIX_API}" "${SANKOFA_NPM_PHOENIX_PORT}" true false && updated_count=$((updated_count + 1)); } || failed_count=$((failed_count + 1))
+update_proxy_host "www.phoenix.sankofa.nexus" "http://${IP_SANKOFA_NPM_PHOENIX_API}:${SANKOFA_NPM_PHOENIX_PORT}" true false "https://phoenix.sankofa.nexus" && updated_count=$((updated_count + 1)) || { add_proxy_host "www.phoenix.sankofa.nexus" "${IP_SANKOFA_NPM_PHOENIX_API}" "${SANKOFA_NPM_PHOENIX_PORT}" true false "https://phoenix.sankofa.nexus" && updated_count=$((updated_count + 1)); } || failed_count=$((failed_count + 1))
 # Keycloak (CT 7802) — portal SSO; NPM must forward X-Forwarded-* (Keycloak KC_PROXY_HEADERS=xforwarded on upstream)
 IP_KEYCLOAK="${IP_KEYCLOAK:-192.168.11.52}"
 update_proxy_host "keycloak.sankofa.nexus" "http://${IP_KEYCLOAK}:8080" false false && updated_count=$((updated_count + 1)) || { add_proxy_host "keycloak.sankofa.nexus" "${IP_KEYCLOAK}" 8080 false false && updated_count=$((updated_count + 1)); } || failed_count=$((failed_count + 1))
--- a/scripts/verify/check-sankofa-consolidated-nginx-examples.sh
+++ b/scripts/verify/check-sankofa-consolidated-nginx-examples.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Validate example nginx configs for Sankofa consolidated web/API hub (syntax only).
+# Read-only; no mutations. Uses host `nginx -t` when available, else Docker `nginx:1.27-alpine`.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+NGINX_DIR="${PROJECT_ROOT}/config/nginx"
+TMPDIR="${TMPDIR:-/tmp}"
+WRAP="${TMPDIR}/sankofa-nginx-test-$$.conf"
+
+cleanup() {
+  rm -f "$WRAP" "${TMPDIR}/sankofa-nginx-wrap-docker-"*.conf 2>/dev/null || true
+  rm -f "${TMPDIR}/sankofa-nginx-test-$$"/*.conf 2>/dev/null || true
+  rmdir "${TMPDIR}/sankofa-nginx-test-$$" 2>/dev/null || true
+}
+trap cleanup EXIT
+
+mkdir -p "${TMPDIR}/sankofa-nginx-test-$$"
+cp "${NGINX_DIR}/sankofa-non-chain-frontends.example.conf" "${TMPDIR}/sankofa-nginx-test-$$/01-web.conf"
+cp "${NGINX_DIR}/sankofa-phoenix-api-hub.example.conf" "${TMPDIR}/sankofa-nginx-test-$$/02-api.conf"
+
+cat >"$WRAP" <<'EOF'
+events { worker_connections 1024; }
+http {
+    include __INCLUDE_DIR__/*.conf;
+}
+EOF
+sed -i "s|__INCLUDE_DIR__|${TMPDIR}/sankofa-nginx-test-$$|g" "$WRAP"
+
+if command -v nginx >/dev/null 2>&1; then
+  echo "== nginx -t (host binary, wrapped includes) =="
+  nginx -t -c "$WRAP"
+elif command -v docker >/dev/null 2>&1; then
+  DOCKER_WRAP="${TMPDIR}/sankofa-nginx-wrap-docker-$$.conf"
+  cat >"$DOCKER_WRAP" <<'INNER'
+events { worker_connections 1024; }
+http {
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+    include /tmp/inc/*.conf;
+}
+INNER
+  echo "== nginx -t (docker nginx:1.27-alpine) =="
+  docker run --rm \
+    -v "$DOCKER_WRAP:/etc/nginx/nginx.conf:ro" \
+    -v "${TMPDIR}/sankofa-nginx-test-$$:/tmp/inc:ro" \
+    nginx:1.27-alpine \
+    nginx -t
+else
+  echo "SKIP: need host nginx or docker to run syntax check."
+  exit 0
+fi
+
+echo "OK: example configs parse."
--- a/scripts/verify/smoke-phoenix-api-hub-lan.sh
+++ b/scripts/verify/smoke-phoenix-api-hub-lan.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Read-only LAN smoke: Tier-1 Phoenix API hub (:8080) — health + GraphQL + proxied api-docs.
+# Usage: bash scripts/verify/smoke-phoenix-api-hub-lan.sh
+# Env: IP_SANKOFA_PHOENIX_API, SANKOFA_API_HUB_PORT (default 8080) from load-project-env / ip-addresses.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+
+HUB_IP="${IP_SANKOFA_PHOENIX_API:-192.168.11.50}"
+# Tier-1 nginx hub listens on :8080 by default (not SANKOFA_PHOENIX_API_HUB_PORT, which often tracks Apollo :4000).
+HUB_PORT="${SANKOFA_API_HUB_LISTEN_PORT:-8080}"
+BASE="http://${HUB_IP}:${HUB_PORT}"
+
+echo "=== smoke-phoenix-api-hub-lan ==="
+echo "Base: ${BASE}"
+echo ""
+
+curl -fsS -m 8 "${BASE}/health" | head -c 200
+echo ""
+echo "--- GraphQL POST /graphql"
+curl -fsS -m 12 "${BASE}/graphql" \
+  -H 'Content-Type: application/json' \
+  -d '{"query":"query { __typename }"}' | head -c 300
+echo ""
+echo "--- GET /api-docs (optional)"
+_ad="/tmp/hub-api-docs-$$"
+code="$(curl -sS -m 12 -o "$_ad" -w "%{http_code}" "${BASE}/api-docs" || echo 000)"
+if [[ "$code" == "200" ]]; then head -c 120 "$_ad"; echo ""; else echo "HTTP ${code} (hub still OK if GraphQL passed)"; fi
+rm -f "$_ad"
+echo ""
+echo "OK: hub smoke passed."
--- a/scripts/verify/smoke-phoenix-graphql-ws-subscription.mjs
+++ b/scripts/verify/smoke-phoenix-graphql-ws-subscription.mjs
@@ -0,0 +1,68 @@
+#!/usr/bin/env node
+/**
+ * Optional: full graphql-ws handshake — connection_init → connection_ack over wss://
+ * Server must expose a single clean upgrade path (standalone `ws` + graphql-ws; remove unused
+ * `@fastify/websocket` on CT 7800 if clients see RSV1 — see ensure-sankofa-phoenix-graphql-ws-remove-fastify-websocket-7800.sh).
+ *
+ * Usage:
+ *   node scripts/verify/smoke-phoenix-graphql-ws-subscription.mjs
+ *   PHOENIX_GRAPHQL_WSS_URL=wss://host/graphql-ws node scripts/verify/smoke-phoenix-graphql-ws-subscription.mjs
+ */
+import { createRequire } from 'node:module';
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+
+const require = createRequire(import.meta.url);
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..', '..');
+const WebSocket = require(path.join(repoRoot, 'node_modules', 'ws'));
+
+const url = process.env.PHOENIX_GRAPHQL_WSS_URL || 'wss://phoenix.sankofa.nexus/graphql-ws';
+const timeoutMs = Number(process.env.PHOENIX_WS_SUB_TIMEOUT_MS || 15000);
+
+const ws = new WebSocket(url, ['graphql-transport-ws'], { perMessageDeflate: false });
+
+const timer = setTimeout(() => {
+  console.error('TIMEOUT waiting for connection_ack');
+  ws.terminate();
+  process.exit(1);
+}, timeoutMs);
+
+ws.on('open', () => {
+  ws.send(JSON.stringify({ type: 'connection_init' }));
+});
+
+ws.on('message', (data) => {
+  const text = String(data);
+  let msg;
+  try {
+    msg = JSON.parse(text);
+  } catch {
+    console.error('Non-JSON message:', text.slice(0, 200));
+    return;
+  }
+  if (msg.type === 'connection_ack') {
+    clearTimeout(timer);
+    console.log('OK: graphql-ws connection_ack');
+    ws.close(1000, 'smoke-ok');
+    process.exit(0);
+  }
+  if (msg.type === 'ping') {
+    ws.send(JSON.stringify({ type: 'pong' }));
+    return;
+  }
+  console.log('msg:', msg.type, JSON.stringify(msg).slice(0, 200));
+});
+
+ws.on('error', (err) => {
+  clearTimeout(timer);
+  console.error('WebSocket error:', err.message);
+  process.exit(2);
+});
+
+ws.on('close', (code, reason) => {
+  clearTimeout(timer);
+  if (code !== 1000) {
+    console.error('Closed:', code, String(reason));
+    process.exit(3);
+  }
+});
--- a/scripts/verify/smoke-phoenix-graphql-wss-public.sh
+++ b/scripts/verify/smoke-phoenix-graphql-wss-public.sh
@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# Smoke: WebSocket upgrade to Phoenix GraphQL WS (graphql-transport-ws) end-to-end through TLS.
+# Uses curl --http1.1 (HTTP/2 cannot complete WS upgrade on many edges). Expects HTTP 101.
+# Each successful probe waits up to PHOENIX_WSS_CURL_MAXTIME seconds (default 8): curl has no EOF on WS.
+#
+# Usage:
+#   bash scripts/verify/smoke-phoenix-graphql-wss-public.sh
+#   PHOENIX_GRAPHQL_WSS_URL=wss://phoenix.example/graphql-ws bash scripts/verify/smoke-phoenix-graphql-wss-public.sh
+# Optional LAN hub (no TLS):
+#   PHOENIX_GRAPHQL_WS_LAN=http://192.168.11.50:8080/graphql-ws PHOENIX_WS_HOST_HEADER=phoenix.sankofa.nexus \
+#     bash scripts/verify/smoke-phoenix-graphql-wss-public.sh
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true
+
+PUBLIC_WSS="${PHOENIX_GRAPHQL_WSS_URL:-https://phoenix.sankofa.nexus/graphql-ws}"
+LAN_WS="${PHOENIX_GRAPHQL_WS_LAN:-}"
+LAN_HOST="${PHOENIX_WS_HOST_HEADER:-phoenix.sankofa.nexus}"
+# After HTTP 101, curl waits for the WebSocket stream until --max-time (no clean EOF). Keep this
+# modest so LAN+public probes do not sit for minutes (override with PHOENIX_WSS_CURL_MAXTIME if needed).
+CURL_MAXTIME="${PHOENIX_WSS_CURL_MAXTIME:-8}"
+# Opt-in LAN hub probe (same CT, HTTP): PHOENIX_WSS_INCLUDE_LAN=1 with load-project-env / ip-addresses
+if [[ "${PHOENIX_WSS_INCLUDE_LAN:-0}" == "1" && -z "$LAN_WS" && -n "${IP_SANKOFA_PHOENIX_API:-}" ]]; then
+  LAN_WS="http://${IP_SANKOFA_PHOENIX_API}:8080/graphql-ws"
+fi
+
+probe_upgrade() {
+  local name="$1"
+  local url="$2"
+  shift 2
+  echo "--- ${name}"
+  echo "    URL: ${url}"
+  local first
+  first="$(curl --http1.1 -sS --connect-timeout 10 -m "${CURL_MAXTIME}" -D- -o /dev/null \
+    "$@" \
+    -H 'Connection: Upgrade' \
+    -H 'Upgrade: websocket' \
+    -H 'Sec-WebSocket-Version: 13' \
+    -H 'Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==' \
+    -H 'Sec-WebSocket-Protocol: graphql-transport-ws' \
+    "$url" 2>&1 | head -1 | tr -d '\r')"
+  if [[ "$first" == *"101"* ]]; then
+    echo "    OK (${first})"
+    return 0
+  fi
+  echo "    FAIL (expected HTTP/1.1 101; first line: ${first})"
+  return 1
+}
+
+echo "=== smoke-phoenix-graphql-wss-public (curl WS upgrade) ==="
+fail=0
+probe_upgrade "Public WSS (NPM → hub → Apollo)" "$PUBLIC_WSS" || fail=1
+
+if [[ -n "$LAN_WS" ]]; then
+  probe_upgrade "LAN hub (optional)" "$LAN_WS" -H "Host: ${LAN_HOST}" || fail=1
+fi
+
+if [[ "$fail" -ne 0 ]]; then
+  echo ""
+  echo "RESULT: one or more upgrade probes failed."
+  exit 1
+fi
+echo ""
+echo "RESULT: WebSocket upgrade path OK (HTTP 101). Full handshake: pnpm run verify:phoenix-graphql-ws-subscription"
--- a/scripts/verify/verify-sankofa-consolidated-hub-lan.sh
+++ b/scripts/verify/verify-sankofa-consolidated-hub-lan.sh
@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Read-only LAN checks for Sankofa Phoenix + dbis_core + optional Keycloak / corporate web.
+# Exit 0 if all probes that are attempted succeed; exit 1 if any required probe fails.
+# Phoenix: prefers Tier-1 hub :8080 /health when hub is up (Apollo may be 127.0.0.1:4000 only).
+# Optional: SANKOFA_VERIFY_PHOENIX_DIRECT_PORT=1 to probe direct :4000 (fails when loopback-bound).
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# shellcheck source=/dev/null
+source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh"
+
+fail=0
+probe() {
+  local name="$1" url="$2"
+  echo "--- ${name}: ${url}"
+  if curl -fsS -m 8 -o /dev/null -w "    HTTP %{http_code}\n" "$url"; then
+    return 0
+  fi
+  echo "    FAIL"
+  fail=1
+}
+
+echo "=== Sankofa / Phoenix / DBIS LAN readiness (read-only) ==="
+echo ""
+
+# Prefer Tier-1 hub :8080 when present (Apollo may bind loopback :4000 only).
+PH_HUB_HEALTH="http://${IP_SANKOFA_PHOENIX_API}:${SANKOFA_API_HUB_LISTEN_PORT:-8080}/health"
+PH_DIRECT_HEALTH="http://${IP_SANKOFA_PHOENIX_API}:${SANKOFA_PHOENIX_API_PORT}/health"
+if curl -fsS -m 6 "$PH_HUB_HEALTH" -o /dev/null 2>/dev/null; then
+  probe "Phoenix API hub /health" "$PH_HUB_HEALTH"
+else
+  probe "Phoenix API /health (direct Apollo)" "$PH_DIRECT_HEALTH"
+fi
+if [[ "${SANKOFA_VERIFY_PHOENIX_DIRECT_PORT:-0}" == "1" ]]; then
+  probe "Phoenix API /health (direct :${SANKOFA_PHOENIX_API_PORT}, optional)" "$PH_DIRECT_HEALTH" || true
+fi
+probe "Portal /" "http://${IP_SANKOFA_PORTAL}:${SANKOFA_PORTAL_PORT}/"
+
+if [[ -n "${IP_SANKOFA_PUBLIC_WEB:-}" ]]; then
+  probe "Corporate web /" "http://${IP_SANKOFA_PUBLIC_WEB}:${SANKOFA_PUBLIC_WEB_PORT:-3000}/" || true
+fi
+
+if [[ -n "${IP_DBIS_API:-}" ]]; then
+  probe "dbis_core /health" "http://${IP_DBIS_API}:3000/health" || true
+fi
+
+# Keycloak (7802 typical) — optional
+if [[ -n "${IP_KEYCLOAK:-}" ]] || [[ -n "${KEYCLOAK_URL:-}" ]]; then
+  _kc="${IP_KEYCLOAK:-}"
+  if [[ -z "$_kc" && "${KEYCLOAK_URL:-}" =~ http://([^:/]+) ]]; then
+    _kc="${BASH_REMATCH[1]}"
+  fi
+  if [[ -n "$_kc" ]]; then
+    echo "--- Keycloak (optional): realm metadata"
+    curl -fsS -m 8 -o /dev/null -w "    HTTP %{http_code} http://${_kc}:8080/realms/master\n" "http://${_kc}:8080/realms/master" \
+      || echo "    SKIP (unreachable or wrong path)"
+  fi
+fi
+
+echo ""
+echo "Resolved hub env (for NPM / nginx cutover):"
+echo "  IP_SANKOFA_WEB_HUB=${IP_SANKOFA_WEB_HUB:-} port ${SANKOFA_WEB_HUB_PORT:-}"
+echo "  IP_SANKOFA_PHOENIX_API_HUB=${IP_SANKOFA_PHOENIX_API_HUB:-} port ${SANKOFA_PHOENIX_API_HUB_PORT:-}"
+echo ""
+
+echo "--- Phoenix Tier-1 API hub (informational, :8080)"
+HUB_LAN="http://${IP_SANKOFA_PHOENIX_API}:${SANKOFA_API_HUB_LISTEN_PORT:-8080}"
+if curl -fsS -m 6 "${HUB_LAN}/health" -o /dev/null 2>/dev/null; then
+  echo "    OK ${HUB_LAN}/health (also covered by required probe above when hub is primary)"
+else
+  echo "    SKIP (no hub on ${HUB_LAN} — install: install-sankofa-api-hub-nginx-on-pve.sh)"
+fi
+echo ""
+
+if [[ "$fail" -ne 0 ]]; then
+  echo "RESULT: one or more probes failed."
+  exit 1
+fi
+echo "RESULT: required probes OK."
+exit 0