proxmox/scripts/maintenance/check-and-fix-explorer-lag.sh

#!/usr/bin/env bash
# Check explorer data-plane lag; if block lag or transaction visibility lag is high,
# run fix-explorer-indexer-lag.sh (restart Blockscout).
# For use from cron. Run from project root. Requires LAN/SSH to r630-02 for the fix.
# Usage: bash scripts/maintenance/check-and-fix-explorer-lag.sh
# Env:
#   EXPLORER_INDEXER_LAG_THRESHOLD (default 500)
#   EXPLORER_TX_VISIBILITY_LAG_BLOCK_THRESHOLD (default 32)
#   EXPLORER_TX_VISIBILITY_SAMPLE_BLOCKS (default 128)

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true

IP_RPC_2201="${RPC_2201:-192.168.11.221}"
IP_BLOCKSCOUT="${IP_BLOCKSCOUT:-192.168.11.140}"
BLOCKSCOUT_API_PORT="${BLOCKSCOUT_API_PORT:-4000}"
EXPLORER_STATS_URL="${EXPLORER_STATS_URL:-https://blockscout.defi-oracle.io/api/v2/stats}"
EXPLORER_INDEXER_LAG_THRESHOLD="${EXPLORER_INDEXER_LAG_THRESHOLD:-500}"
EXPLORER_TX_VISIBILITY_LAG_BLOCK_THRESHOLD="${EXPLORER_TX_VISIBILITY_LAG_BLOCK_THRESHOLD:-32}"
EXPLORER_TX_VISIBILITY_SAMPLE_BLOCKS="${EXPLORER_TX_VISIBILITY_SAMPLE_BLOCKS:-128}"

get_rpc_block() {
  local hex
  hex=$(curl -sf --max-time 10 -X POST -H "Content-Type: application/json" \
    -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
    "http://${IP_RPC_2201}:8545" 2>/dev/null | sed -n 's/.*"result":"\(0x[0-9a-fA-F]*\)".*/\1/p')
  [ -n "$hex" ] && echo $((hex)) || true
}

get_explorer_block() {
  local body block
  body=$(curl -sf --max-time 10 "${EXPLORER_STATS_URL}" 2>/dev/null || true)
  [ -z "$body" ] && return
  block=$(echo "$body" | jq -r '.freshness.latest_indexed_block.block_number // .latest_block // .total_blocks // empty' 2>/dev/null || true)
  [ -n "$block" ] && echo "$block"
}

get_explorer_latest_tx_block() {
  local body block
  body=$(curl -sf --max-time 10 "${EXPLORER_STATS_URL}" 2>/dev/null || true)
  [ -z "$body" ] && return
  block=$(echo "$body" | jq -r '.freshness.latest_indexed_transaction.block_number // empty' 2>/dev/null || true)
  [ -n "$block" ] && [ "$block" != "null" ] && echo "$block"
}

get_recent_chain_activity() {
  local latest="$1"
  local newest_non_empty=""
  local total_txs=0
  local non_empty_count=0
  local block_num block_json tx_count

  for ((offset=0; offset<EXPLORER_TX_VISIBILITY_SAMPLE_BLOCKS; offset++)); do
    block_num=$((latest - offset))
    [ "$block_num" -lt 0 ] && break
    block_json=$(curl -sf --max-time 10 -X POST -H "Content-Type: application/json" \
      -d "{\"jsonrpc\":\"2.0\",\"method\":\"eth_getBlockByNumber\",\"params\":[\"$(printf '0x%x' "$block_num")\",false],\"id\":1}" \
      "http://${IP_RPC_2201}:8545" 2>/dev/null || true)
    [ -z "$block_json" ] && continue
    tx_count=$(echo "$block_json" | jq -r '.result.transactions | length' 2>/dev/null || echo "")
    [ -z "$tx_count" ] && continue
    total_txs=$((total_txs + tx_count))
    if [ "$tx_count" -gt 0 ] 2>/dev/null; then
      non_empty_count=$((non_empty_count + 1))
      if [ -z "$newest_non_empty" ]; then
        newest_non_empty="$block_num"
      fi
    fi
  done

  printf '%s|%s|%s\n' "${newest_non_empty:-none}" "$non_empty_count" "$total_txs"
}

rpc_block=$(get_rpc_block)
explorer_block=$(get_explorer_block)
explorer_latest_tx_block=$(get_explorer_latest_tx_block)

if [ -z "$rpc_block" ] || [ -z "$explorer_block" ]; then
  echo "$(date -Iseconds) SKIP (RPC or Blockscout unreachable)"
  exit 0
fi

lag=$((rpc_block - explorer_block))
if [ "$lag" -le "${EXPLORER_INDEXER_LAG_THRESHOLD}" ] 2>/dev/null; then
  echo "$(date -Iseconds) OK block_lag=$lag (threshold=${EXPLORER_INDEXER_LAG_THRESHOLD})"
else
  echo "$(date -Iseconds) BLOCK_LAG $lag > ${EXPLORER_INDEXER_LAG_THRESHOLD} — running fix"
  bash "$PROJECT_ROOT/scripts/fix-explorer-indexer-lag.sh" 2>&1 || true
  exit 0
fi

IFS='|' read -r newest_non_empty recent_non_empty_count recent_tx_total <<<"$(get_recent_chain_activity "$rpc_block")"

if [ "$newest_non_empty" = "none" ]; then
  echo "$(date -Iseconds) QUIET_CHAIN sample_blocks=${EXPLORER_TX_VISIBILITY_SAMPLE_BLOCKS} non_empty=0 txs=0"
  exit 0
fi

if [ -z "$explorer_latest_tx_block" ]; then
  echo "$(date -Iseconds) TX_VISIBILITY missing explorer latest tx while chain shows recent txs (newest_non_empty=$newest_non_empty) — running fix"
  bash "$PROJECT_ROOT/scripts/fix-explorer-indexer-lag.sh" 2>&1 || true
  exit 0
fi

tx_visibility_gap=$((newest_non_empty - explorer_latest_tx_block))
if [ "$tx_visibility_gap" -gt "${EXPLORER_TX_VISIBILITY_LAG_BLOCK_THRESHOLD}" ] 2>/dev/null; then
  echo "$(date -Iseconds) TX_VISIBILITY_LAG gap=$tx_visibility_gap newest_non_empty=$newest_non_empty explorer_latest_tx=$explorer_latest_tx_block sample_non_empty=$recent_non_empty_count sample_txs=$recent_tx_total — running fix"
  bash "$PROJECT_ROOT/scripts/fix-explorer-indexer-lag.sh" 2>&1 || true
  exit 0
fi

echo "$(date -Iseconds) OK tx_visibility_gap=$tx_visibility_gap newest_non_empty=$newest_non_empty explorer_latest_tx=$explorer_latest_tx_block sample_non_empty=$recent_non_empty_count sample_txs=$recent_tx_total"