Tighten MEV rollout verification and deploy timeouts

docs/04-configuration/MEV_CHAIN138_ALL_MAINNET_ROLLOUT_TRACKER.md

@@ -39,9 +39,11 @@ It is intentionally split into:
   - `cWUSDC/USDC` reserve imbalance `7227 bps`
   - `cWUSDT/USDT` reserve imbalance `5025 bps`
   - `cWUSDC/USDT` reserve imbalance `7956 bps`
-  - `cWUSDT/cWUSDC` token resolution incomplete (`quote` token missing)
-- Additional public pools returned reserves, but the readiness gate ended with:
-  - `5 hard failure(s), 0 warning(s)`
+- `2026-04-14` repo-side inventory fix landed for `cWUSDT/cWUSDC` quote-token resolution in
+  `scripts/verify/check-mainnet-pmm-peg-bot-readiness.sh`.
+- Re-run result after the fix:
+  - `cWUSDT/cWUSDC` now resolves and reads healthy reserves
+  - readiness gate now ends with `4 hard failure(s), 0 warning(s)`
 
 This means Mainnet public PMM operation is not yet in a fully healthy state even
 though large parts of the deployment inventory exist.
@@ -144,7 +146,6 @@ though large parts of the deployment inventory exist.
    - rebalance `cWUSDC/USDC`
    - rebalance `cWUSDT/USDT`
    - rebalance `cWUSDC/USDT`
-   - fix `cWUSDT/cWUSDC` token-resolution/inventory drift
 6. Publish the final ALL Mainnet native venue matrix before trying to wire full MEV discovery there.
 
 ## Truthful Completion Condition

scripts/deployment/deploy-mev-backend-ct2421-latest.sh

@@ -8,6 +8,10 @@ MEV_DIR="${MEV_BACKEND_MEV_DIR:-$CT_ROOT/mev-platform}"
 MEV_BRANCH="${MEV_BACKEND_BRANCH:-master}"
 PARENT_BRANCH="${MEV_BACKEND_PARENT_BRANCH:-master}"
 API_KEY="${MEV_API_KEY:-}"
+SSH_CONNECT_TIMEOUT="${MEV_BACKEND_SSH_CONNECT_TIMEOUT:-10}"
+SSH_SERVER_ALIVE_INTERVAL="${MEV_BACKEND_SSH_SERVER_ALIVE_INTERVAL:-10}"
+SSH_SERVER_ALIVE_COUNT_MAX="${MEV_BACKEND_SSH_SERVER_ALIVE_COUNT_MAX:-6}"
+REMOTE_TIMEOUT_SECONDS="${MEV_BACKEND_REMOTE_TIMEOUT_SECONDS:-1800}"
 APPLY=0
 
 usage() {
@@ -27,6 +31,7 @@ Options:
   --branch NAME          mev-platform branch to fast-forward (default: master)
   --parent-branch NAME   MEV_Bot branch to fast-forward (default: master)
   --api-key KEY          Optional API key for protected verification routes
+  --remote-timeout SEC   Max remote execution time before timeout (default: 1800)
   --dry-run              Print the planned remote command (default behavior)
   --apply                Execute the deployment
   -h, --help             Show this help
@@ -63,6 +68,10 @@ while [[ $# -gt 0 ]]; do
       API_KEY="$2"
       shift 2
       ;;
+    --remote-timeout)
+      REMOTE_TIMEOUT_SECONDS="$2"
+      shift 2
+      ;;
     --dry-run)
       APPLY=0
       shift
@@ -95,6 +104,8 @@ require_cmd bash
 
 REMOTE_CMD=$(cat <<EOF
 set -euo pipefail
+command -v timeout >/dev/null 2>&1 || { echo "missing timeout inside CT" >&2; exit 127; }
+timeout ${REMOTE_TIMEOUT_SECONDS}s bash -lc '
 cd "$CT_ROOT"
 echo "== parent repo =="
 git fetch origin
@@ -133,6 +144,7 @@ if [ -n "${API_KEY}" ]; then
   curl -fsS -H "X-API-Key: ${API_KEY}" http://127.0.0.1:9090/api/stats/metrics-surfaces
   echo
 fi
+'
 EOF
 )
 
@@ -143,10 +155,12 @@ echo "CT root:       $CT_ROOT"
 echo "MEV dir:       $MEV_DIR"
 echo "Parent branch: $PARENT_BRANCH"
 echo "MEV branch:    $MEV_BRANCH"
+echo "SSH timeout:   ${SSH_CONNECT_TIMEOUT}s connect, ${SSH_SERVER_ALIVE_INTERVAL}s keepalive x ${SSH_SERVER_ALIVE_COUNT_MAX}"
+echo "Remote limit:  ${REMOTE_TIMEOUT_SECONDS}s"
 echo "Apply:         $APPLY"
 echo
 echo "Planned remote command:"
-echo "ssh root@$PVE_HOST \"pct exec $CT_VMID -- bash -lc $(printf '%q' "$REMOTE_CMD")\""
+echo "ssh -o BatchMode=yes -o ConnectTimeout=$SSH_CONNECT_TIMEOUT -o ServerAliveInterval=$SSH_SERVER_ALIVE_INTERVAL -o ServerAliveCountMax=$SSH_SERVER_ALIVE_COUNT_MAX root@$PVE_HOST \"pct exec $CT_VMID -- bash -lc $(printf '%q' "$REMOTE_CMD")\""
 
 if [[ "$APPLY" -ne 1 ]]; then
   echo
@@ -154,4 +168,10 @@ if [[ "$APPLY" -ne 1 ]]; then
   exit 0
 fi
 
-ssh "root@$PVE_HOST" "pct exec $CT_VMID -- bash -lc $(printf '%q' "$REMOTE_CMD")"
+ssh \
+  -o BatchMode=yes \
+  -o ConnectTimeout="$SSH_CONNECT_TIMEOUT" \
+  -o ServerAliveInterval="$SSH_SERVER_ALIVE_INTERVAL" \
+  -o ServerAliveCountMax="$SSH_SERVER_ALIVE_COUNT_MAX" \
+  "root@$PVE_HOST" \
+  "pct exec $CT_VMID -- bash -lc $(printf '%q' "$REMOTE_CMD")"

scripts/verify/check-mainnet-pmm-peg-bot-readiness.sh

@@ -86,12 +86,20 @@ check_pool_row() {
   local label="${base_sym}/${quote_sym}"
 
   local base_addr
-  base_addr="$(jq -r --arg b "$base_sym" '.chains["1"].cwTokens[$b] // empty' "$STATUS_JSON")"
+  base_addr="$(jq -r --arg b "$base_sym" '
+    .chains["1"].cwTokens[$b]
+    // .chains["1"].anchorAddresses[$b]
+    // empty
+  ' "$STATUS_JSON")"
   local quote_addr
   if [[ "$quote_sym" == "TRUU" ]]; then
     quote_addr="$(jq -r '.chains["1"].anchorAddresses.TRUU // empty' "$STATUS_JSON")"
   else
-    quote_addr="$(jq -r --arg q "$quote_sym" '.chains["1"].anchorAddresses[$q] // empty' "$STATUS_JSON")"
+    quote_addr="$(jq -r --arg q "$quote_sym" '
+      .chains["1"].anchorAddresses[$q]
+      // .chains["1"].cwTokens[$q]
+      // empty
+    ' "$STATUS_JSON")"
   fi
 
   if [[ -z "$base_addr" || -z "$quote_addr" ]]; then