#!/usr/bin/env bash # Check explorer data-plane lag; if block lag or transaction visibility lag is high, # run fix-explorer-indexer-lag.sh (restart Blockscout). # For use from cron. Run from project root. Requires LAN/SSH to r630-02 for the fix. # Usage: bash scripts/maintenance/check-and-fix-explorer-lag.sh # Env: # EXPLORER_INDEXER_LAG_THRESHOLD (default 500) # EXPLORER_TX_VISIBILITY_LAG_BLOCK_THRESHOLD (default 32) # EXPLORER_TX_VISIBILITY_SAMPLE_BLOCKS (default 128) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true IP_RPC_2201="${RPC_2201:-192.168.11.221}" IP_BLOCKSCOUT="${IP_BLOCKSCOUT:-192.168.11.140}" BLOCKSCOUT_API_PORT="${BLOCKSCOUT_API_PORT:-4000}" EXPLORER_STATS_URL="${EXPLORER_STATS_URL:-https://blockscout.defi-oracle.io/api/v2/stats}" EXPLORER_INDEXER_LAG_THRESHOLD="${EXPLORER_INDEXER_LAG_THRESHOLD:-500}" EXPLORER_TX_VISIBILITY_LAG_BLOCK_THRESHOLD="${EXPLORER_TX_VISIBILITY_LAG_BLOCK_THRESHOLD:-32}" EXPLORER_TX_VISIBILITY_SAMPLE_BLOCKS="${EXPLORER_TX_VISIBILITY_SAMPLE_BLOCKS:-128}" get_rpc_block() { local hex hex=$(curl -sf --max-time 10 -X POST -H "Content-Type: application/json" \ -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \ "http://${IP_RPC_2201}:8545" 2>/dev/null | sed -n 's/.*"result":"\(0x[0-9a-fA-F]*\)".*/\1/p') [ -n "$hex" ] && echo $((hex)) || true } get_explorer_block() { local body block body=$(curl -sf --max-time 10 "${EXPLORER_STATS_URL}" 2>/dev/null || true) [ -z "$body" ] && return block=$(echo "$body" | jq -r '.freshness.latest_indexed_block.block_number // .latest_block // .total_blocks // empty' 2>/dev/null || true) [ -n "$block" ] && echo "$block" } get_explorer_latest_tx_block() { local body block body=$(curl -sf --max-time 10 "${EXPLORER_STATS_URL}" 2>/dev/null || true) [ -z "$body" ] && return block=$(echo "$body" | jq -r '.freshness.latest_indexed_transaction.block_number // empty' 2>/dev/null || true) [ -n "$block" ] && [ "$block" != "null" ] && echo "$block" } get_recent_chain_activity() { local latest="$1" local newest_non_empty="" local total_txs=0 local non_empty_count=0 local block_num block_json tx_count for ((offset=0; offset/dev/null || true) [ -z "$block_json" ] && continue tx_count=$(echo "$block_json" | jq -r '.result.transactions | length' 2>/dev/null || echo "") [ -z "$tx_count" ] && continue total_txs=$((total_txs + tx_count)) if [ "$tx_count" -gt 0 ] 2>/dev/null; then non_empty_count=$((non_empty_count + 1)) if [ -z "$newest_non_empty" ]; then newest_non_empty="$block_num" fi fi done printf '%s|%s|%s\n' "${newest_non_empty:-none}" "$non_empty_count" "$total_txs" } rpc_block=$(get_rpc_block) explorer_block=$(get_explorer_block) explorer_latest_tx_block=$(get_explorer_latest_tx_block) if [ -z "$rpc_block" ] || [ -z "$explorer_block" ]; then echo "$(date -Iseconds) SKIP (RPC or Blockscout unreachable)" exit 0 fi lag=$((rpc_block - explorer_block)) if [ "$lag" -le "${EXPLORER_INDEXER_LAG_THRESHOLD}" ] 2>/dev/null; then echo "$(date -Iseconds) OK block_lag=$lag (threshold=${EXPLORER_INDEXER_LAG_THRESHOLD})" else echo "$(date -Iseconds) BLOCK_LAG $lag > ${EXPLORER_INDEXER_LAG_THRESHOLD} — running fix" bash "$PROJECT_ROOT/scripts/fix-explorer-indexer-lag.sh" 2>&1 || true exit 0 fi IFS='|' read -r newest_non_empty recent_non_empty_count recent_tx_total <<<"$(get_recent_chain_activity "$rpc_block")" if [ "$newest_non_empty" = "none" ]; then echo "$(date -Iseconds) QUIET_CHAIN sample_blocks=${EXPLORER_TX_VISIBILITY_SAMPLE_BLOCKS} non_empty=0 txs=0" exit 0 fi if [ -z "$explorer_latest_tx_block" ]; then echo "$(date -Iseconds) TX_VISIBILITY missing explorer latest tx while chain shows recent txs (newest_non_empty=$newest_non_empty) — running fix" bash "$PROJECT_ROOT/scripts/fix-explorer-indexer-lag.sh" 2>&1 || true exit 0 fi tx_visibility_gap=$((newest_non_empty - explorer_latest_tx_block)) if [ "$tx_visibility_gap" -gt "${EXPLORER_TX_VISIBILITY_LAG_BLOCK_THRESHOLD}" ] 2>/dev/null; then echo "$(date -Iseconds) TX_VISIBILITY_LAG gap=$tx_visibility_gap newest_non_empty=$newest_non_empty explorer_latest_tx=$explorer_latest_tx_block sample_non_empty=$recent_non_empty_count sample_txs=$recent_tx_total — running fix" bash "$PROJECT_ROOT/scripts/fix-explorer-indexer-lag.sh" 2>&1 || true exit 0 fi echo "$(date -Iseconds) OK tx_visibility_gap=$tx_visibility_gap newest_non_empty=$newest_non_empty explorer_latest_tx=$explorer_latest_tx_block sample_non_empty=$recent_non_empty_count sample_txs=$recent_tx_total"