fbda1b4beb
Some checks failed
Deploy to Phoenix / deploy (push) Has been cancelled
- ADD_CHAIN138_TO_LEDGER_LIVE: Ledger form done; public code review repo bis-innovations/LedgerLive; init/push commands - CONTRACT_DEPLOYMENT_RUNBOOK: Chain 138 gas price 1 gwei, 36-addr check, TransactionMirror workaround - CONTRACT_*: AddressMapper, MirrorManager deployed 2026-02-12; 36-address on-chain check - NEXT_STEPS_FOR_YOU: Ledger done; steps completable now (no LAN); run-completable-tasks-from-anywhere - MASTER_INDEX, OPERATOR_OPTIONAL, SMART_CONTRACTS_INVENTORY_SIMPLE: updates - LEDGER_BLOCKCHAIN_INTEGRATION_COMPLETE: bis-innovations/LedgerLive reference Co-authored-by: Cursor <cursoragent@cursor.com>
289 lines
12 KiB
Bash
Executable File
289 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# Check all Validator and Sentry node logs for errors
|
||
# Validators: VMIDs 1000-1004
|
||
# Sentries: VMIDs 1500-1503
|
||
|
||
set -euo pipefail
|
||
|
||
# Load IP configuration
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||
source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true
|
||
|
||
|
||
# Colors
|
||
RED='\033[0;31m'
|
||
GREEN='\033[0;32m'
|
||
YELLOW='\033[1;33m'
|
||
BLUE='\033[0;34m'
|
||
NC='\033[0m' # No Color
|
||
|
||
# Proxmox host configuration
|
||
PROXMOX_HOST="${PROXMOX_HOST:-192.168.11.10}"
|
||
SSH_PASSWORD="${SSH_PASSWORD:-L@kers2010}"
|
||
|
||
# Node IP mappings
|
||
declare -A NODE_IPS=(
|
||
[1000]="${IP_VALIDATOR_0:-${IP_VALIDATOR_0:-${IP_VALIDATOR_0:-${IP_VALIDATOR_0:-192.168.11.100}}}}"
|
||
[1001]="${IP_VALIDATOR_1:-${IP_VALIDATOR_1:-${IP_VALIDATOR_1:-${IP_VALIDATOR_1:-192.168.11.101}}}}"
|
||
[1002]="${IP_VALIDATOR_2:-${IP_VALIDATOR_2:-${IP_VALIDATOR_2:-${IP_VALIDATOR_2:-192.168.11.102}}}}"
|
||
[1003]="${IP_VALIDATOR_3:-${IP_VALIDATOR_3:-${IP_VALIDATOR_3:-${IP_VALIDATOR_3:-192.168.11.103}}}}"
|
||
[1004]="${IP_VALIDATOR_4:-${IP_VALIDATOR_4:-${IP_VALIDATOR_4:-${IP_VALIDATOR_4:-192.168.11.104}}}}"
|
||
[1500]="${IP_BESU_RPC_0:-${IP_BESU_RPC_0:-${IP_BESU_RPC_0:-${IP_BESU_RPC_0:-192.168.11.150}}}}"
|
||
[1501]="${IP_BESU_RPC_1:-${IP_BESU_RPC_1:-${IP_BESU_RPC_1:-${IP_BESU_RPC_1:-192.168.11.151}}}}"
|
||
[1502]="${IP_BESU_RPC_2:-${IP_BESU_RPC_2:-${IP_BESU_RPC_2:-${IP_BESU_RPC_2:-192.168.11.152}}}}"
|
||
[1503]="${IP_BESU_RPC_3:-${IP_BESU_RPC_3:-${IP_BESU_RPC_3:-${IP_BESU_RPC_3:-192.168.11.153}}}}"
|
||
)
|
||
|
||
# Node definitions
|
||
VALIDATORS=(1000 1001 1002 1003 1004)
|
||
SENTRIES=(1500 1501 1502 1503)
|
||
LOG_LINES="${1:-100}"
|
||
|
||
# Check if sshpass is available
|
||
if ! command -v sshpass >/dev/null 2>&1; then
|
||
echo "⚠️ sshpass not installed. Attempting to install..."
|
||
sudo apt-get update -qq && sudo apt-get install -y sshpass 2>/dev/null || {
|
||
echo "❌ Cannot install sshpass automatically"
|
||
echo "Please install manually: sudo apt-get install sshpass"
|
||
exit 1
|
||
}
|
||
fi
|
||
|
||
# Error patterns to search for
|
||
ERROR_PATTERNS=(
|
||
"error"
|
||
"Error"
|
||
"ERROR"
|
||
"failed"
|
||
"Failed"
|
||
"FAILED"
|
||
"exception"
|
||
"Exception"
|
||
"EXCEPTION"
|
||
"fatal"
|
||
"Fatal"
|
||
"FATAL"
|
||
"panic"
|
||
"Panic"
|
||
"PANIC"
|
||
"Unable to read"
|
||
"file not found"
|
||
"configuration"
|
||
"restart"
|
||
"crash"
|
||
"timeout"
|
||
"Timeout"
|
||
"connection refused"
|
||
"Connection refused"
|
||
)
|
||
|
||
echo -e "${BLUE}╔══════════════════════════════════════════════════════════════╗${NC}"
|
||
echo -e "${BLUE}║ CHECKING ALL VALIDATOR AND SENTRY NODE LOGS ║${NC}"
|
||
echo -e "${BLUE}╚══════════════════════════════════════════════════════════════╝${NC}"
|
||
echo ""
|
||
echo "Checking last $LOG_LINES lines of logs for each node"
|
||
echo ""
|
||
|
||
# Function to check logs for a node
|
||
check_node_logs() {
|
||
local vmid=$1
|
||
local service_name=$2
|
||
local node_type=$3
|
||
|
||
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
|
||
echo -e "${BLUE}Checking ${node_type} VMID $vmid (service: $service_name)${NC}"
|
||
echo -e "${BLUE}═══════════════════════════════════════════════════════════════${NC}"
|
||
|
||
# Get container IP
|
||
local container_ip="${NODE_IPS[$vmid]}"
|
||
if [ -z "$container_ip" ]; then
|
||
echo -e "${RED}❌ VMID $vmid: IP address not found in mapping${NC}"
|
||
echo ""
|
||
return 1
|
||
fi
|
||
|
||
# Try to access container directly via SSH first
|
||
local logs=""
|
||
local service_status="unknown"
|
||
|
||
# Check if we can access via Proxmox host (preferred method)
|
||
if ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=3 -i ~/.ssh/id_ed25519_proxmox "root@${PROXMOX_HOST}" "pct status $vmid 2>/dev/null" &>/dev/null; then
|
||
# Access via Proxmox host
|
||
local status_output
|
||
status_output=$(ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 -i ~/.ssh/id_ed25519_proxmox "root@${PROXMOX_HOST}" \
|
||
"pct status $vmid 2>/dev/null" || echo "")
|
||
|
||
if [ -z "$status_output" ]; then
|
||
echo -e "${RED}❌ VMID $vmid: Container not found or not accessible${NC}"
|
||
echo ""
|
||
return 1
|
||
fi
|
||
|
||
local status=$(echo "$status_output" | awk '{print $2}' || echo "unknown")
|
||
if [ "$status" != "running" ]; then
|
||
echo -e "${YELLOW}⚠️ VMID $vmid: Container is not running (status: $status)${NC}"
|
||
echo ""
|
||
return 1
|
||
fi
|
||
|
||
# Check service status
|
||
service_status=$(ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 -i ~/.ssh/id_ed25519_proxmox "root@${PROXMOX_HOST}" \
|
||
"pct exec $vmid -- systemctl is-active $service_name.service 2>/dev/null" || echo "inactive")
|
||
|
||
# Get recent logs
|
||
logs=$(ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 -i ~/.ssh/id_ed25519_proxmox "root@${PROXMOX_HOST}" \
|
||
"pct exec $vmid -- journalctl -u $service_name.service -n $LOG_LINES --no-pager 2>/dev/null" || echo "")
|
||
else
|
||
# Fallback: Try direct SSH to container
|
||
echo -e "${YELLOW}⚠️ Cannot access via Proxmox host, trying direct SSH to container...${NC}"
|
||
|
||
# Check service status via direct SSH
|
||
service_status=$(sshpass -p "$SSH_PASSWORD" ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 \
|
||
"root@${container_ip}" \
|
||
"systemctl is-active $service_name.service 2>/dev/null" || echo "inactive")
|
||
|
||
# Get recent logs via direct SSH
|
||
logs=$(sshpass -p "$SSH_PASSWORD" ssh -o StrictHostKeyChecking=accept-new -o ConnectTimeout=5 \
|
||
"root@${container_ip}" \
|
||
"journalctl -u $service_name.service -n $LOG_LINES --no-pager 2>/dev/null" || echo "")
|
||
fi
|
||
|
||
if [ "$service_status" != "active" ]; then
|
||
echo -e "${YELLOW}⚠️ Service $service_name is not active (status: $service_status)${NC}"
|
||
else
|
||
echo -e "${GREEN}✅ Service $service_name is active${NC}"
|
||
fi
|
||
|
||
# Get recent logs
|
||
echo ""
|
||
echo "Recent logs (last $LOG_LINES lines):"
|
||
echo "---"
|
||
|
||
if [ -z "$logs" ]; then
|
||
echo -e "${YELLOW}⚠️ No logs found for service $service_name${NC}"
|
||
echo ""
|
||
return 1
|
||
fi
|
||
|
||
# Display logs
|
||
echo "$logs"
|
||
echo "---"
|
||
echo ""
|
||
|
||
# Check for errors
|
||
echo "Checking for errors..."
|
||
local error_found=false
|
||
local error_count=0
|
||
|
||
for pattern in "${ERROR_PATTERNS[@]}"; do
|
||
local matches=$(echo "$logs" | grep -i "$pattern" | grep -v "restart counter" | grep -v "Scheduled restart" | grep -v "CORS Rejected" || true)
|
||
if [ -n "$matches" ]; then
|
||
local match_count=$(echo "$matches" | wc -l)
|
||
error_count=$((error_count + match_count))
|
||
if [ "$error_found" = false ]; then
|
||
error_found=true
|
||
echo -e "${RED}❌ ERRORS FOUND:${NC}"
|
||
fi
|
||
echo -e "${RED} Pattern '$pattern' found $match_count time(s):${NC}"
|
||
echo "$matches" | head -5 | sed 's/^/ /'
|
||
if [ "$match_count" -gt 5 ]; then
|
||
echo -e "${YELLOW} ... and $((match_count - 5)) more occurrence(s)${NC}"
|
||
fi
|
||
fi
|
||
done
|
||
|
||
# Check restart count
|
||
local restart_count=$(echo "$logs" | grep -i "restart counter" | tail -1 | grep -oP 'restart counter is at \K\d+' || echo "0")
|
||
if [ "$restart_count" != "0" ] && [ -n "$restart_count" ]; then
|
||
if [ "$restart_count" -gt 10 ]; then
|
||
echo -e "${RED}⚠️ High restart count: $restart_count${NC}"
|
||
error_found=true
|
||
elif [ "$restart_count" -gt 0 ]; then
|
||
echo -e "${YELLOW}ℹ️ Restart count: $restart_count${NC}"
|
||
fi
|
||
fi
|
||
|
||
echo ""
|
||
|
||
if [ "$error_found" = false ]; then
|
||
echo -e "${GREEN}✅ No errors found in recent logs${NC}"
|
||
return 0
|
||
else
|
||
echo -e "${RED}❌ Total error occurrences: $error_count${NC}"
|
||
return 1
|
||
fi
|
||
}
|
||
|
||
# Summary tracking
|
||
total_validators=0
|
||
total_sentries=0
|
||
validators_with_errors=0
|
||
sentries_with_errors=0
|
||
validators_checked=0
|
||
sentries_checked=0
|
||
|
||
# Check all Validator nodes
|
||
echo -e "${BLUE}╔══════════════════════════════════════════════════════════════╗${NC}"
|
||
echo -e "${BLUE}║ VALIDATOR NODES (VMIDs 1000-1004) ║${NC}"
|
||
echo -e "${BLUE}╚══════════════════════════════════════════════════════════════╝${NC}"
|
||
echo ""
|
||
|
||
for vmid in "${VALIDATORS[@]}"; do
|
||
if check_node_logs "$vmid" "besu-validator" "Validator"; then
|
||
validators_checked=$((validators_checked + 1))
|
||
else
|
||
validators_with_errors=$((validators_with_errors + 1))
|
||
validators_checked=$((validators_checked + 1))
|
||
fi
|
||
total_validators=$((total_validators + 1))
|
||
done
|
||
|
||
# Check all Sentry nodes
|
||
echo -e "${BLUE}╔══════════════════════════════════════════════════════════════╗${NC}"
|
||
echo -e "${BLUE}║ SENTRY NODES (VMIDs 1500-1503) ║${NC}"
|
||
echo -e "${BLUE}╚══════════════════════════════════════════════════════════════╝${NC}"
|
||
echo ""
|
||
|
||
for vmid in "${SENTRIES[@]}"; do
|
||
if check_node_logs "$vmid" "besu-sentry" "Sentry"; then
|
||
sentries_checked=$((sentries_checked + 1))
|
||
else
|
||
sentries_with_errors=$((sentries_with_errors + 1))
|
||
sentries_checked=$((sentries_checked + 1))
|
||
fi
|
||
total_sentries=$((total_sentries + 1))
|
||
done
|
||
|
||
# Final Summary
|
||
echo -e "${BLUE}╔══════════════════════════════════════════════════════════════╗${NC}"
|
||
echo -e "${BLUE}║ SUMMARY ║${NC}"
|
||
echo -e "${BLUE}╚══════════════════════════════════════════════════════════════╝${NC}"
|
||
echo ""
|
||
echo "Validators:"
|
||
echo " Total: $total_validators"
|
||
echo " Checked: $validators_checked"
|
||
if [ "$validators_with_errors" -eq 0 ]; then
|
||
echo -e " Errors: ${GREEN}✅ None found${NC}"
|
||
else
|
||
echo -e " Errors: ${RED}❌ Found in $validators_with_errors node(s)${NC}"
|
||
fi
|
||
echo ""
|
||
echo "Sentries:"
|
||
echo " Total: $total_sentries"
|
||
echo " Checked: $sentries_checked"
|
||
if [ "$sentries_with_errors" -eq 0 ]; then
|
||
echo -e " Errors: ${GREEN}✅ None found${NC}"
|
||
else
|
||
echo -e " Errors: ${RED}❌ Found in $sentries_with_errors node(s)${NC}"
|
||
fi
|
||
echo ""
|
||
|
||
if [ "$validators_with_errors" -eq 0 ] && [ "$sentries_with_errors" -eq 0 ]; then
|
||
echo -e "${GREEN}✅ All logs checked - No current errors found!${NC}"
|
||
exit 0
|
||
else
|
||
echo -e "${RED}❌ Errors found in some nodes. Review logs above.${NC}"
|
||
exit 1
|
||
fi
|