Add Oracle Aggregator and CCIP Integration

- Introduced Aggregator.sol for Chainlink-compatible oracle functionality, including round-based updates and access control.
- Added OracleWithCCIP.sol to extend Aggregator with CCIP cross-chain messaging capabilities.
- Created .gitmodules to include OpenZeppelin contracts as a submodule.
- Developed a comprehensive deployment guide in NEXT_STEPS_COMPLETE_GUIDE.md for Phase 2 and smart contract deployment.
- Implemented Vite configuration for the orchestration portal, supporting both Vue and React frameworks.
- Added server-side logic for the Multi-Cloud Orchestration Portal, including API endpoints for environment management and monitoring.
- Created scripts for resource import and usage validation across non-US regions.
- Added tests for CCIP error handling and integration to ensure robust functionality.
- Included various new files and directories for the orchestration portal and deployment scripts.
This commit is contained in:
defiQUG
2025-12-12 14:57:48 -08:00
parent a1466e4005
commit 1fb7266469
1720 changed files with 241279 additions and 16 deletions

View File

@@ -0,0 +1,214 @@
#!/usr/bin/env bash
# Disaster Recovery Test Script
# This script tests disaster recovery procedures
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../lib/init.sh"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
NAMESPACE="${NAMESPACE:-besu-network}"
BACKUP_DIR="${BACKUP_DIR:-/tmp/backup-test}"
log_success "Running Disaster Recovery Tests..."
# Test backup procedures
log_warn "Testing backup procedures..."
# Create backup directory
mkdir -p "$BACKUP_DIR"
# Test chaindata backup
if [ -f "$PROJECT_ROOT/scripts/backup/backup-chaindata.sh" ]; then
log_warn " Testing chaindata backup..."
# Run backup (dry-run if possible)
if "$PROJECT_ROOT/scripts/backup/backup-chaindata.sh" 2>&1 | tee /tmp/backup-test.log; then
log_success "✓ Backup script executed successfully"
else
log_warn "⚠ Backup script execution had issues (check logs)"
fi
else
log_warn "⚠ Backup script not found"
fi
# Test restore procedures
log_warn "Testing restore procedures..."
if [ -f "$PROJECT_ROOT/scripts/backup/restore-chaindata.sh" ]; then
log_success "✓ Restore script exists"
# Validate restore script syntax
if bash -n "$PROJECT_ROOT/scripts/backup/restore-chaindata.sh" 2>&1; then
log_success "✓ Restore script syntax is valid"
else
log_error "✗ Restore script has syntax errors"
exit 1
fi
else
log_warn "⚠ Restore script not found"
fi
# Test failover scenarios
log_warn "Testing failover scenarios..."
# Test validator failover
VALIDATOR_PODS=$(kubectl get pods -n "$NAMESPACE" -l component=validator -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
if [ -n "$VALIDATOR_PODS" ]; then
log_warn " Testing validator pod failure..."
# Get initial pod count
INITIAL_COUNT=$(kubectl get pods -n "$NAMESPACE" -l component=validator --no-headers | wc -l)
log_warn " Initial validator pods: $INITIAL_COUNT"
# Delete a pod
log_warn " Deleting pod: $VALIDATOR_PODS"
kubectl delete pod "$VALIDATOR_PODS" -n "$NAMESPACE" --wait=false
# Wait for pod to be recreated
log_warn " Waiting for pod to be recreated (60 seconds)..."
sleep 60
# Check if pod was recreated
NEW_COUNT=$(kubectl get pods -n "$NAMESPACE" -l component=validator --no-headers | wc -l)
log_warn " Current validator pods: $NEW_COUNT"
if [ "$NEW_COUNT" -eq "$INITIAL_COUNT" ]; then
log_success "✓ Validator pod was recreated successfully"
else
log_warn "⚠ Validator pod count changed (may still be recovering)"
fi
else
log_warn "⚠ No validator pods available for failover test"
fi
# Test RPC node failover
RPC_PODS=$(kubectl get pods -n "$NAMESPACE" -l component=rpc -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
if [ -n "$RPC_PODS" ]; then
log_warn " Testing RPC node failover..."
INITIAL_COUNT=$(kubectl get pods -n "$NAMESPACE" -l component=rpc --no-headers | wc -l)
log_warn " Initial RPC pods: $INITIAL_COUNT"
# Delete a pod
log_warn " Deleting pod: $RPC_PODS"
kubectl delete pod "$RPC_PODS" -n "$NAMESPACE" --wait=false
# Wait for pod to be recreated
log_warn " Waiting for pod to be recreated (60 seconds)..."
sleep 60
# Check if pod was recreated
NEW_COUNT=$(kubectl get pods -n "$NAMESPACE" -l component=rpc --no-headers | wc -l)
log_warn " Current RPC pods: $NEW_COUNT"
if [ "$NEW_COUNT" -eq "$INITIAL_COUNT" ]; then
log_success "✓ RPC pod was recreated successfully"
else
log_warn "⚠ RPC pod count changed (may still be recovering)"
fi
else
log_warn "⚠ No RPC pods available for failover test"
fi
# Test recovery time objectives
log_warn "Testing recovery time objectives..."
# Measure pod restart time
if [ -n "$VALIDATOR_PODS" ]; then
log_warn " Measuring pod restart time..."
START_TIME=$(date +%s)
kubectl delete pod "$VALIDATOR_PODS" -n "$NAMESPACE" --wait=false
# Wait for pod to be ready
if kubectl wait --for=condition=ready pod -l component=validator -n "$NAMESPACE" --timeout=300s 2>/dev/null; then
END_TIME=$(date +%s)
RESTART_TIME=$((END_TIME - START_TIME))
log_success "✓ Pod restarted in ${RESTART_TIME} seconds"
# Check against RTO (1 hour for critical services)
if [ "$RESTART_TIME" -lt 3600 ]; then
log_success "✓ Restart time is within RTO (1 hour)"
else
log_error "✗ Restart time exceeds RTO (1 hour)"
fi
else
log_error "✗ Pod did not restart within timeout"
fi
else
log_warn "⚠ No pods available for RTO test"
fi
# Test backup validation
log_warn "Testing backup validation..."
# Check if backup files exist
BACKUP_FILES=$(find "$BACKUP_DIR" -name "*.tar.gz" 2>/dev/null || echo "")
if [ -n "$BACKUP_FILES" ]; then
log_success "✓ Backup files found"
# Validate backup file integrity
for backup in $BACKUP_FILES; do
log_warn " Validating $backup..."
if tar -tzf "$backup" > /dev/null 2>&1; then
log_success "✓ Backup file is valid: $backup"
else
log_error "✗ Backup file is corrupted: $backup"
fi
done
else
log_warn "⚠ No backup files found for validation"
fi
# Test key recovery
log_warn "Testing key recovery..."
if [ -f "$PROJECT_ROOT/scripts/key-management/rotate-keys.sh" ]; then
log_success "✓ Key rotation script exists"
# Validate key rotation script syntax
if bash -n "$PROJECT_ROOT/scripts/key-management/rotate-keys.sh" 2>&1; then
log_success "✓ Key rotation script syntax is valid"
else
log_error "✗ Key rotation script has syntax errors"
exit 1
fi
else
log_warn "⚠ Key rotation script not found"
fi
# Document test results
log_warn "Documenting test results..."
TEST_RESULTS_FILE="/tmp/disaster-recovery-test-results.txt"
cat > "$TEST_RESULTS_FILE" <<EOF
Disaster Recovery Test Results
Date: $(date)
Namespace: $NAMESPACE
Test Summary:
- Backup procedures: Tested
- Restore procedures: Tested
- Failover scenarios: Tested
- Recovery time objectives: Tested
- Backup validation: Tested
- Key recovery: Tested
Notes:
- Tests require actual deployment environment
- Some tests may be inconclusive without running pods
- Review logs for detailed results
EOF
log_success "✓ Test results documented: $TEST_RESULTS_FILE"
log_success "Disaster recovery testing completed"

198
scripts/validation/load-test.sh Executable file
View File

@@ -0,0 +1,198 @@
#!/usr/bin/env bash
# Load Test Script
# This script runs load tests on RPC endpoints
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../lib/init.sh"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
# Configuration
RPC_URL="${RPC_URL:-http://localhost:8545}"
DURATION="${DURATION:-60}" # seconds
RPS="${RPS:-100}" # requests per second
CONCURRENT="${CONCURRENT:-10}" # concurrent requests
log_success "Running Load Tests..."
log_warn "RPC URL: $RPC_URL"
log_warn "Duration: $DURATION seconds"
log_warn "Requests per second: $RPS"
log_warn "Concurrent requests: $CONCURRENT"
# Test RPC endpoint availability
log_warn "Testing RPC endpoint availability..."
if curl -s -X POST -H "Content-Type: application/json" \
--data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
"$RPC_URL" | grep -q "result"; then
log_success "✓ RPC endpoint is available"
else
log_error "✗ RPC endpoint is not available"
exit 1
fi
# Load test with k6 (if available)
if command -v k6 &> /dev/null; then
log_success "✓ k6 is available"
# Create k6 test script
cat > /tmp/k6-load-test.js <<EOF
import http from 'k6/http';
import { check, sleep } from 'k6';
export const options = {
stages: [
{ duration: '30s', target: ${RPS} },
{ duration: '${DURATION}s', target: ${RPS} },
{ duration: '30s', target: 0 },
],
thresholds: {
http_req_duration: ['p(95)<500', 'p(99)<1000'],
http_req_failed: ['rate<0.01'],
},
};
export default function () {
const payload = JSON.stringify({
jsonrpc: '2.0',
method: 'eth_blockNumber',
params: [],
id: 1
});
const params = {
headers: {
'Content-Type': 'application/json',
},
};
const res = http.post('${RPC_URL}', payload, params);
check(res, {
'status is 200': (r) => r.status === 200,
'response has result': (r) => JSON.parse(r.body).result !== undefined,
});
sleep(1);
}
EOF
log_warn "Running k6 load test..."
if k6 run /tmp/k6-load-test.js 2>&1 | tee /tmp/k6-load-test.log; then
log_success "✓ k6 load test completed"
else
log_warn "⚠ k6 load test completed with issues (check logs)"
fi
else
log_warn "⚠ k6 not available. Install it for load testing:"
echo " https://k6.io/docs/getting-started/installation/"
# Fallback: Simple load test with curl
log_warn "Running simple load test with curl..."
SUCCESS=0
FAILED=0
TOTAL=0
END_TIME=$((SECONDS + DURATION))
while [ $SECONDS -lt $END_TIME ]; do
for i in $(seq 1 $CONCURRENT); do
(
if curl -s -X POST -H "Content-Type: application/json" \
--data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
--max-time 5 \
"$RPC_URL" | grep -q "result"; then
echo "SUCCESS" >> /tmp/load-test-results.txt
else
echo "FAILED" >> /tmp/load-test-results.txt
fi
) &
done
sleep 1
TOTAL=$((TOTAL + CONCURRENT))
done
wait
SUCCESS=$(grep -c "SUCCESS" /tmp/load-test-results.txt 2>/dev/null || echo "0")
FAILED=$(grep -c "FAILED" /tmp/load-test-results.txt 2>/dev/null || echo "0")
log_success "Load test results:"
echo -e " Total requests: $TOTAL"
echo -e " Successful: $SUCCESS"
echo -e " Failed: $FAILED"
echo -e " Success rate: $((SUCCESS * 100 / TOTAL))%"
fi
# Test autoscaling
log_warn "Testing autoscaling..."
NAMESPACE="${NAMESPACE:-besu-network}"
if kubectl get hpa besu-rpc-hpa -n "$NAMESPACE" &>/dev/null; then
log_success "✓ HPA exists"
# Get initial replica count
INITIAL_REPLICAS=$(kubectl get statefulset besu-rpc -n "$NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
log_warn " Initial replicas: $INITIAL_REPLICAS"
# Generate load
log_warn " Generating load for 60 seconds..."
END_TIME=$((SECONDS + 60))
while [ $SECONDS -lt $END_TIME ]; do
for i in $(seq 1 50); do
curl -s -X POST -H "Content-Type: application/json" \
--data '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
"$RPC_URL" > /dev/null 2>&1 &
done
sleep 1
done
wait
# Wait for scaling
log_warn " Waiting for scaling (30 seconds)..."
sleep 30
# Check replica count
CURRENT_REPLICAS=$(kubectl get statefulset besu-rpc -n "$NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
log_warn " Current replicas: $CURRENT_REPLICAS"
if [ "$CURRENT_REPLICAS" -gt "$INITIAL_REPLICAS" ]; then
log_success "✓ Autoscaling is working (scaled from $INITIAL_REPLICAS to $CURRENT_REPLICAS)"
else
log_warn "⚠ Autoscaling did not trigger (may need more load or time)"
fi
else
log_warn "⚠ HPA not found"
fi
# Performance metrics
log_warn "Collecting performance metrics..."
# Test various RPC methods
RPC_METHODS=(
"eth_blockNumber"
"eth_getBlockByNumber"
"eth_getBalance"
"eth_call"
)
for method in "${RPC_METHODS[@]}"; do
log_warn " Testing $method..."
START_TIME=$(date +%s%N)
curl -s -X POST -H "Content-Type: application/json" \
--data "{\"jsonrpc\":\"2.0\",\"method\":\"$method\",\"params\":[],\"id\":1}" \
"$RPC_URL" > /dev/null
END_TIME=$(date +%s%N)
DURATION_MS=$(( (END_TIME - START_TIME) / 1000000 ))
log_success " $method: ${DURATION_MS}ms"
done
log_success "Load testing completed"

View File

@@ -0,0 +1,64 @@
#!/usr/bin/env bash
# Run All Validations
# This script runs all validation tests
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../lib/init.sh"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
log_success "Running All Validations..."
# Run all validation scripts
VALIDATION_SCRIPTS=(
"validate-genesis.sh"
"validate-deployment.sh"
"validate-network-policies.sh"
"validate-rbac.sh"
"validate-hpa.sh"
"validate-monitoring.sh"
"security-scan.sh"
"load-test.sh"
"disaster-recovery-test.sh"
)
FAILED=0
PASSED=0
for script in "${VALIDATION_SCRIPTS[@]}"; do
SCRIPT_PATH="$SCRIPT_DIR/$script"
if [ -f "$SCRIPT_PATH" ]; then
log_warn "Running $script..."
echo "----------------------------------------"
if bash "$SCRIPT_PATH"; then
log_success "$script passed"
PASSED=$((PASSED + 1))
else
log_error "$script failed"
FAILED=$((FAILED + 1))
fi
else
log_warn "$script not found"
fi
done
# Summary
log_success "Validation Summary:"
echo -e " Passed: $PASSED"
echo -e " Failed: $FAILED"
echo -e " Total: $((PASSED + FAILED))"
if [ $FAILED -eq 0 ]; then
log_success "✓ All validations passed"
exit 0
else
log_error "✗ Some validations failed"
exit 1
fi

View File

@@ -0,0 +1,166 @@
#!/usr/bin/env bash
# Security Scan Script
# This script runs security scans on containers and smart contracts
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../lib/init.sh"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
log_success "Running Security Scans..."
# Container image scanning
log_warn "Scanning container images..."
IMAGES=(
"hyperledger/besu:23.10.0"
"blockscout/blockscout:v5.1.5"
"prom/prometheus:v2.45.0"
"grafana/grafana:10.1.0"
"busybox:1.36"
)
# Check if trivy is available
if command -v trivy &> /dev/null; then
log_success "✓ Trivy is available"
for image in "${IMAGES[@]}"; do
log_warn "Scanning $image..."
if trivy image --severity HIGH,CRITICAL "$image" 2>&1 | tee /tmp/trivy-scan-$(echo $image | tr '/:' '-').log; then
log_success "✓ Scan completed for $image"
else
log_warn "⚠ Scan completed with issues for $image (check logs)"
fi
done
else
log_warn "⚠ Trivy not available. Install it for container scanning:"
echo " https://aquasecurity.github.io/trivy/latest/getting-started/installation/"
fi
# Smart contract security scanning
log_warn "Scanning smart contracts..."
# Check if slither is available
if command -v slither &> /dev/null; then
log_success "✓ Slither is available"
CONTRACTS=(
"contracts/oracle/Aggregator.sol"
"contracts/oracle/Proxy.sol"
"contracts/tokens/WETH.sol"
"contracts/utils/Multicall.sol"
"contracts/utils/CREATE2Factory.sol"
)
for contract in "${CONTRACTS[@]}"; do
if [ -f "$PROJECT_ROOT/$contract" ]; then
log_warn "Scanning $contract..."
if slither "$PROJECT_ROOT/$contract" 2>&1 | tee /tmp/slither-scan-$(basename $contract .sol).log; then
log_success "✓ Scan completed for $contract"
else
log_warn "⚠ Scan completed with issues for $contract (check logs)"
fi
fi
done
else
log_warn "⚠ Slither not available. Install it for smart contract scanning:"
echo " pip install slither-analyzer"
fi
# Foundry security tests
log_warn "Running Foundry security tests..."
if command -v forge &> /dev/null; then
log_success "✓ Foundry is available"
cd "$PROJECT_ROOT"
# Run tests
if forge test --gas-report 2>&1 | tee /tmp/foundry-tests.log; then
log_success "✓ Foundry tests passed"
else
log_warn "⚠ Some Foundry tests failed (check logs)"
fi
# Run fuzz tests
if forge test --fuzz-runs 1000 2>&1 | tee /tmp/foundry-fuzz.log; then
log_success "✓ Foundry fuzz tests passed"
else
log_warn "⚠ Some Foundry fuzz tests failed (check logs)"
fi
else
log_warn "⚠ Foundry not available. Install it for testing:"
echo " https://book.getfoundry.sh/getting-started/installation"
fi
# Dependency scanning
log_warn "Scanning dependencies..."
# Python dependencies
if [ -f "$PROJECT_ROOT/services/oracle-publisher/requirements.txt" ]; then
log_warn "Scanning Python dependencies..."
if command -v safety &> /dev/null; then
if safety check --file "$PROJECT_ROOT/services/oracle-publisher/requirements.txt" 2>&1 | tee /tmp/safety-scan.log; then
log_success "✓ Python dependencies scan completed"
else
log_warn "⚠ Python dependencies scan found issues (check logs)"
fi
else
log_warn "⚠ Safety not available. Install it for Python dependency scanning:"
echo " pip install safety"
fi
fi
# Node.js dependencies (SDK)
if [ -f "$PROJECT_ROOT/sdk/package.json" ]; then
log_warn "Scanning Node.js dependencies..."
if command -v npm &> /dev/null; then
cd "$PROJECT_ROOT/sdk"
if npm audit --audit-level=moderate 2>&1 | tee /tmp/npm-audit.log; then
log_success "✓ Node.js dependencies scan completed"
else
log_warn "⚠ Node.js dependencies scan found issues (check logs)"
fi
else
log_warn "⚠ npm not available"
fi
fi
# Terraform security scanning
log_warn "Scanning Terraform configuration..."
if command -v checkov &> /dev/null; then
log_success "✓ Checkov is available"
if checkov -d "$PROJECT_ROOT/terraform" --framework terraform 2>&1 | tee /tmp/checkov-scan.log; then
log_success "✓ Terraform security scan completed"
else
log_warn "⚠ Terraform security scan found issues (check logs)"
fi
else
log_warn "⚠ Checkov not available. Install it for Terraform scanning:"
echo " pip install checkov"
fi
# Kubernetes manifest scanning
log_warn "Scanning Kubernetes manifests..."
if command -v kube-score &> /dev/null; then
log_success "✓ kube-score is available"
if kube-score score "$PROJECT_ROOT/k8s" -o human 2>&1 | tee /tmp/kube-score-scan.log; then
log_success "✓ Kubernetes manifest scan completed"
else
log_warn "⚠ Kubernetes manifest scan found issues (check logs)"
fi
else
log_warn "⚠ kube-score not available. Install it for Kubernetes scanning:"
echo " https://github.com/zegl/kube-score#installation"
fi
log_success "Security scanning completed"
log_warn "Scan results are saved in /tmp/*.log"

View File

@@ -0,0 +1,149 @@
#!/usr/bin/env bash
# Validate Deployment
# This script validates that all deployments are working correctly
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../lib/init.sh"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
NAMESPACE="${NAMESPACE:-besu-network}"
log_success "Validating Deployment..."
# Check namespace
log_warn "Checking namespace..."
if kubectl get namespace "$NAMESPACE" &>/dev/null; then
log_success "✓ Namespace $NAMESPACE exists"
else
log_warn "⚠ Namespace $NAMESPACE not found, creating..."
kubectl create namespace "$NAMESPACE"
fi
# Check validators
log_warn "Checking validators..."
VALIDATOR_STS=$(kubectl get statefulset besu-validator -n "$NAMESPACE" -o name 2>/dev/null || echo "")
if [ -n "$VALIDATOR_STS" ]; then
log_success "✓ Validator StatefulSet exists"
# Check replica count
DESIRED=$(kubectl get statefulset besu-validator -n "$NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
READY=$(kubectl get statefulset besu-validator -n "$NAMESPACE" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
log_warn " Desired: $DESIRED, Ready: $READY"
if [ "$DESIRED" == "$READY" ] && [ "$READY" -gt 0 ]; then
log_success "✓ All validator pods are ready"
else
log_warn "⚠ Not all validator pods are ready"
fi
else
log_warn "⚠ Validator StatefulSet not found"
fi
# Check sentries
log_warn "Checking sentries..."
SENTRY_STS=$(kubectl get statefulset besu-sentry -n "$NAMESPACE" -o name 2>/dev/null || echo "")
if [ -n "$SENTRY_STS" ]; then
log_success "✓ Sentry StatefulSet exists"
DESIRED=$(kubectl get statefulset besu-sentry -n "$NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
READY=$(kubectl get statefulset besu-sentry -n "$NAMESPACE" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
log_warn " Desired: $DESIRED, Ready: $READY"
if [ "$DESIRED" == "$READY" ] && [ "$READY" -gt 0 ]; then
log_success "✓ All sentry pods are ready"
else
log_warn "⚠ Not all sentry pods are ready"
fi
else
log_warn "⚠ Sentry StatefulSet not found"
fi
# Check RPC nodes
log_warn "Checking RPC nodes..."
RPC_STS=$(kubectl get statefulset besu-rpc -n "$NAMESPACE" -o name 2>/dev/null || echo "")
if [ -n "$RPC_STS" ]; then
log_success "✓ RPC StatefulSet exists"
DESIRED=$(kubectl get statefulset besu-rpc -n "$NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
READY=$(kubectl get statefulset besu-rpc -n "$NAMESPACE" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
log_warn " Desired: $DESIRED, Ready: $READY"
if [ "$DESIRED" == "$READY" ] && [ "$READY" -gt 0 ]; then
log_success "✓ All RPC pods are ready"
else
log_warn "⚠ Not all RPC pods are ready"
fi
else
log_warn "⚠ RPC StatefulSet not found"
fi
# Check health checks
log_warn "Checking health checks..."
VALIDATOR_PODS=$(kubectl get pods -n "$NAMESPACE" -l component=validator -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
if [ -n "$VALIDATOR_PODS" ]; then
# Check if pod has health check probes
LIVENESS=$(kubectl get pod "$VALIDATOR_PODS" -n "$NAMESPACE" -o jsonpath='{.spec.containers[0].livenessProbe.httpGet.path}' 2>/dev/null || echo "")
READINESS=$(kubectl get pod "$VALIDATOR_PODS" -n "$NAMESPACE" -o jsonpath='{.spec.containers[0].readinessProbe.httpGet.path}' 2>/dev/null || echo "")
if [ "$LIVENESS" == "/metrics" ]; then
log_success "✓ Liveness probe is configured correctly"
else
log_warn "⚠ Liveness probe path: $LIVENESS"
fi
if [ "$READINESS" == "/metrics" ]; then
log_success "✓ Readiness probe is configured correctly"
else
log_warn "⚠ Readiness probe path: $READINESS"
fi
else
log_warn "⚠ No validator pods found for health check validation"
fi
# Test pod restart
log_warn "Testing pod restart..."
if [ -n "$VALIDATOR_PODS" ]; then
log_warn " Restarting pod: $VALIDATOR_PODS"
kubectl delete pod "$VALIDATOR_PODS" -n "$NAMESPACE" --wait=false
# Wait for pod to be ready
log_warn " Waiting for pod to be ready..."
if kubectl wait --for=condition=ready pod -l component=validator -n "$NAMESPACE" --timeout=300s 2>/dev/null; then
log_success "✓ Pod restarted successfully"
else
log_warn "⚠ Pod restart test inconclusive"
fi
else
log_warn "⚠ No pods available for restart test"
fi
# Check services
log_warn "Checking services..."
SERVICES=("besu-validator" "besu-sentry" "besu-rpc")
for svc in "${SERVICES[@]}"; do
if kubectl get service "$svc" -n "$NAMESPACE" &>/dev/null; then
log_success "✓ Service $svc exists"
else
log_warn "⚠ Service $svc not found"
fi
done
# Check ConfigMaps
log_warn "Checking ConfigMaps..."
CONFIGMAPS=("besu-validator-config" "besu-sentry-config" "besu-rpc-config")
for cm in "${CONFIGMAPS[@]}"; do
if kubectl get configmap "$cm" -n "$NAMESPACE" &>/dev/null; then
log_success "✓ ConfigMap $cm exists"
else
log_warn "⚠ ConfigMap $cm not found"
fi
done
log_success "Deployment validation completed"

View File

@@ -0,0 +1,114 @@
#!/usr/bin/env bash
# Validate Genesis File
# This script validates the genesis file for ChainID 138
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../lib/init.sh"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
CONFIG_DIR="$PROJECT_ROOT/config"
GENESIS_FILE="$CONFIG_DIR/genesis.json"
log_success "Validating Genesis File..."
# Check if genesis file exists
if [ ! -f "$GENESIS_FILE" ]; then
log_error "✗ Genesis file not found: $GENESIS_FILE"
log_warn "Generating genesis file..."
"$PROJECT_ROOT/scripts/generate-genesis-proper.sh" 4
fi
# Validate JSON syntax
log_warn "Validating JSON syntax..."
if jq empty "$GENESIS_FILE" 2>/dev/null; then
log_success "✓ Genesis file has valid JSON syntax"
else
log_error "✗ Genesis file has invalid JSON syntax"
exit 1
fi
# Check chain ID
log_warn "Checking chain ID..."
CHAIN_ID=$(jq -r '.config.chainId' "$GENESIS_FILE")
if [ "$CHAIN_ID" == "138" ]; then
log_success "✓ Chain ID is correct: $CHAIN_ID"
else
log_error "✗ Chain ID is incorrect: $CHAIN_ID (expected 138)"
exit 1
fi
# Check extraData
log_warn "Checking extraData..."
EXTRA_DATA=$(jq -r '.extraData' "$GENESIS_FILE")
if [ "$EXTRA_DATA" != "0x" ] && [ -n "$EXTRA_DATA" ]; then
log_success "✓ extraData is set: ${EXTRA_DATA:0:50}..."
# Validate extraData length (should be reasonable for IBFT)
EXTRA_DATA_LENGTH=$(echo -n "$EXTRA_DATA" | wc -c)
if [ "$EXTRA_DATA_LENGTH" -gt 2 ]; then
log_success "✓ extraData has content (length: $EXTRA_DATA_LENGTH)"
else
log_error "✗ extraData appears to be empty"
exit 1
fi
else
log_error "✗ extraData is empty or invalid"
log_warn "Note: extraData must be generated using Besu operator generate-blockchain-config"
exit 1
fi
# Check IBFT configuration
log_warn "Checking IBFT configuration..."
IBFT_CONFIG=$(jq -r '.config.ibft2' "$GENESIS_FILE")
if [ "$IBFT_CONFIG" != "null" ]; then
log_success "✓ IBFT 2.0 configuration exists"
# Check block period
BLOCK_PERIOD=$(jq -r '.config.ibft2.blockperiodseconds' "$GENESIS_FILE")
if [ "$BLOCK_PERIOD" == "2" ]; then
log_success "✓ Block period is correct: $BLOCK_PERIOD seconds"
else
log_warn "⚠ Block period is $BLOCK_PERIOD (expected 2)"
fi
# Check epoch length
EPOCH_LENGTH=$(jq -r '.config.ibft2.epochlength' "$GENESIS_FILE")
if [ "$EPOCH_LENGTH" == "30000" ]; then
log_success "✓ Epoch length is correct: $EPOCH_LENGTH"
else
log_warn "⚠ Epoch length is $EPOCH_LENGTH (expected 30000)"
fi
else
log_error "✗ IBFT 2.0 configuration not found"
exit 1
fi
# Check gas limit
log_warn "Checking gas limit..."
GAS_LIMIT=$(jq -r '.gasLimit' "$GENESIS_FILE")
if [ "$GAS_LIMIT" == "0x1c9c380" ]; then
log_success "✓ Gas limit is correct: $GAS_LIMIT"
else
log_warn "⚠ Gas limit is $GAS_LIMIT (expected 0x1c9c380)"
fi
# Validate with Besu (if available)
log_warn "Validating with Besu..."
if command -v besu &> /dev/null; then
# Try to validate genesis file with Besu
if besu blocks import --from="$GENESIS_FILE" --to=/tmp/besu-test 2>&1 | grep -q "success\|imported"; then
log_success "✓ Genesis file validated with Besu"
rm -rf /tmp/besu-test
else
log_warn "⚠ Besu validation inconclusive (this is expected for genesis files)"
fi
else
log_warn "⚠ Besu not available for validation"
fi
log_success "Genesis file validation completed"
log_success "✓ All validations passed"

View File

@@ -0,0 +1,83 @@
#!/usr/bin/env bash
# Validate HPA Configuration
# This script validates that HPA is correctly configured and working
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../lib/init.sh"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
NAMESPACE="${NAMESPACE:-besu-network}"
log_success "Validating HPA Configuration..."
# Check if HPA exists
log_warn "Checking HPA..."
if kubectl get hpa besu-rpc-hpa -n "$NAMESPACE" &>/dev/null; then
log_success "✓ HPA besu-rpc-hpa exists"
# Get HPA details
log_warn "HPA Details:"
kubectl get hpa besu-rpc-hpa -n "$NAMESPACE" -o yaml | grep -A 10 "spec:"
else
log_warn "⚠ HPA not found, applying..."
kubectl apply -f "$PROJECT_ROOT/k8s/base/rpc/hpa.yaml"
fi
# Validate HPA configuration
log_warn "Validating HPA configuration..."
# Check min replicas
MIN_REPLICAS=$(kubectl get hpa besu-rpc-hpa -n "$NAMESPACE" -o jsonpath='{.spec.minReplicas}' 2>/dev/null || echo "")
if [ "$MIN_REPLICAS" == "2" ]; then
log_success "✓ Min replicas is set to 2"
else
log_warn "⚠ Min replicas is $MIN_REPLICAS (expected 2)"
fi
# Check max replicas
MAX_REPLICAS=$(kubectl get hpa besu-rpc-hpa -n "$NAMESPACE" -o jsonpath='{.spec.maxReplicas}' 2>/dev/null || echo "")
if [ "$MAX_REPLICAS" == "10" ]; then
log_success "✓ Max replicas is set to 10"
else
log_warn "⚠ Max replicas is $MAX_REPLICAS (expected 10)"
fi
# Check metrics
METRICS=$(kubectl get hpa besu-rpc-hpa -n "$NAMESPACE" -o jsonpath='{.spec.metrics[*].type}' 2>/dev/null || echo "")
if echo "$METRICS" | grep -q "Resource"; then
log_success "✓ Resource metrics configured"
else
log_warn "⚠ Resource metrics not found"
fi
# Check if target StatefulSet exists
log_warn "Checking target StatefulSet..."
if kubectl get statefulset besu-rpc -n "$NAMESPACE" &>/dev/null; then
log_success "✓ Target StatefulSet besu-rpc exists"
# Check current replicas
CURRENT_REPLICAS=$(kubectl get statefulset besu-rpc -n "$NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "")
log_warn "Current replicas: $CURRENT_REPLICAS"
# Check HPA status
log_warn "HPA Status:"
kubectl get hpa besu-rpc-hpa -n "$NAMESPACE" -o jsonpath='{.status}' | jq '.' 2>/dev/null || kubectl get hpa besu-rpc-hpa -n "$NAMESPACE"
else
log_warn "⚠ Target StatefulSet besu-rpc not found"
fi
# Test autoscaling (if metrics server is available)
log_warn "Testing autoscaling..."
if kubectl top nodes &>/dev/null; then
log_success "✓ Metrics server is available"
log_warn "Note: HPA will scale based on CPU/memory usage"
log_warn "To test autoscaling, generate load on RPC endpoints"
else
log_warn "⚠ Metrics server not available (HPA requires metrics server)"
fi
log_success "HPA validation completed"

View File

@@ -0,0 +1,144 @@
#!/usr/bin/env bash
# Validate Monitoring Setup
# This script validates that monitoring is correctly configured and working
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../lib/init.sh"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
NAMESPACE="${NAMESPACE:-besu-network}"
log_success "Validating Monitoring Setup..."
# Check Prometheus
log_warn "Checking Prometheus..."
if kubectl get deployment prometheus -n "$NAMESPACE" &>/dev/null; then
log_success "✓ Prometheus deployment exists"
# Check if Prometheus is ready
READY=$(kubectl get deployment prometheus -n "$NAMESPACE" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
DESIRED=$(kubectl get deployment prometheus -n "$NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
if [ "$READY" == "$DESIRED" ] && [ "$READY" -gt 0 ]; then
log_success "✓ Prometheus is ready"
# Test Prometheus API
PROMETHEUS_POD=$(kubectl get pods -n "$NAMESPACE" -l app=prometheus -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
if [ -n "$PROMETHEUS_POD" ]; then
if kubectl exec -n "$NAMESPACE" "$PROMETHEUS_POD" -- wget -qO- http://localhost:9090/api/v1/status/config 2>/dev/null | grep -q "yaml"; then
log_success "✓ Prometheus API is responding"
else
log_warn "⚠ Prometheus API test inconclusive"
fi
fi
else
log_warn "⚠ Prometheus is not ready (Ready: $READY, Desired: $DESIRED)"
fi
else
log_warn "⚠ Prometheus deployment not found"
fi
# Check Grafana
log_warn "Checking Grafana..."
if kubectl get deployment grafana -n "$NAMESPACE" &>/dev/null; then
log_success "✓ Grafana deployment exists"
READY=$(kubectl get deployment grafana -n "$NAMESPACE" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
DESIRED=$(kubectl get deployment grafana -n "$NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
if [ "$READY" == "$DESIRED" ] && [ "$READY" -gt 0 ]; then
log_success "✓ Grafana is ready"
else
log_warn "⚠ Grafana is not ready (Ready: $READY, Desired: $DESIRED)"
fi
else
log_warn "⚠ Grafana deployment not found"
fi
# Check ServiceMonitors
log_warn "Checking ServiceMonitors..."
SERVICE_MONITORS=("besu-validators" "besu-sentries" "besu-rpc" "oracle-publisher")
for sm in "${SERVICE_MONITORS[@]}"; do
if kubectl get servicemonitor "$sm" -n "$NAMESPACE" &>/dev/null; then
log_success "✓ ServiceMonitor $sm exists"
else
log_warn "⚠ ServiceMonitor $sm not found, applying..."
kubectl apply -f "$PROJECT_ROOT/monitoring/k8s/servicemonitor.yaml"
break
fi
done
# Check if metrics are being collected
log_warn "Checking if metrics are being collected..."
VALIDATOR_PODS=$(kubectl get pods -n "$NAMESPACE" -l component=validator -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
if [ -n "$VALIDATOR_PODS" ]; then
# Test metrics endpoint
if kubectl exec -n "$NAMESPACE" "$VALIDATOR_PODS" -- wget -qO- http://localhost:9545/metrics 2>/dev/null | grep -q "besu"; then
log_success "✓ Metrics endpoint is working"
else
log_warn "⚠ Metrics endpoint test inconclusive"
fi
else
log_warn "⚠ No pods available for metrics testing"
fi
# Check Alertmanager
log_warn "Checking Alertmanager..."
if kubectl get deployment alertmanager -n "$NAMESPACE" &>/dev/null; then
log_success "✓ Alertmanager deployment exists"
READY=$(kubectl get deployment alertmanager -n "$NAMESPACE" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
DESIRED=$(kubectl get deployment alertmanager -n "$NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
if [ "$READY" == "$DESIRED" ] && [ "$READY" -gt 0 ]; then
log_success "✓ Alertmanager is ready"
else
log_warn "⚠ Alertmanager is not ready (Ready: $READY, Desired: $DESIRED)"
fi
else
log_warn "⚠ Alertmanager deployment not found"
fi
# Check alert rules
log_warn "Checking alert rules..."
if [ -f "$PROJECT_ROOT/monitoring/prometheus/alerts/besu.yml" ]; then
log_success "✓ Alert rules file exists"
# Validate alert rules syntax
if command -v promtool &> /dev/null; then
if promtool check rules "$PROJECT_ROOT/monitoring/prometheus/alerts/besu.yml" 2>/dev/null; then
log_success "✓ Alert rules syntax is valid"
else
log_warn "⚠ Alert rules syntax validation inconclusive"
fi
else
log_warn "⚠ promtool not available for validation"
fi
else
log_warn "⚠ Alert rules file not found"
fi
# Check Loki (if deployed)
log_warn "Checking Loki..."
if kubectl get deployment loki -n "$NAMESPACE" &>/dev/null; then
log_success "✓ Loki deployment exists"
READY=$(kubectl get deployment loki -n "$NAMESPACE" -o jsonpath='{.status.readyReplicas}' 2>/dev/null || echo "0")
DESIRED=$(kubectl get deployment loki -n "$NAMESPACE" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo "0")
if [ "$READY" == "$DESIRED" ] && [ "$READY" -gt 0 ]; then
log_success "✓ Loki is ready"
else
log_warn "⚠ Loki is not ready (Ready: $READY, Desired: $DESIRED)"
fi
else
log_warn "⚠ Loki deployment not found (optional)"
fi
log_success "Monitoring validation completed"

View File

@@ -0,0 +1,83 @@
#!/usr/bin/env bash
# Validate Network Policies
# This script validates that Network Policies are correctly applied and working
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../lib/init.sh"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
NAMESPACE="${NAMESPACE:-besu-network}"
log_success "Validating Network Policies..."
# Check if Network Policies are applied
log_warn "Checking Network Policies..."
NETWORK_POLICIES=$(kubectl get networkpolicies -n "$NAMESPACE" -o name 2>/dev/null || echo "")
if [ -z "$NETWORK_POLICIES" ]; then
log_error "✗ No Network Policies found in namespace $NAMESPACE"
log_warn "Applying Network Policies..."
kubectl apply -f "$PROJECT_ROOT/k8s/network-policies/default-deny.yaml"
else
log_success "✓ Network Policies found:"
echo "$NETWORK_POLICIES"
fi
# Validate specific policies
log_warn "Validating specific policies..."
# Check default-deny policy
if kubectl get networkpolicy default-deny-all -n "$NAMESPACE" &>/dev/null; then
log_success "✓ default-deny-all policy exists"
else
log_error "✗ default-deny-all policy not found"
exit 1
fi
# Check validator policy
if kubectl get networkpolicy allow-validator-internal -n "$NAMESPACE" &>/dev/null; then
log_success "✓ allow-validator-internal policy exists"
else
log_error "✗ allow-validator-internal policy not found"
exit 1
fi
# Check sentry policy
if kubectl get networkpolicy allow-sentry-p2p -n "$NAMESPACE" &>/dev/null; then
log_success "✓ allow-sentry-p2p policy exists"
else
log_error "✗ allow-sentry-p2p policy not found"
exit 1
fi
# Check RPC policy
if kubectl get networkpolicy allow-rpc-http -n "$NAMESPACE" &>/dev/null; then
log_success "✓ allow-rpc-http policy exists"
else
log_error "✗ allow-rpc-http policy not found"
exit 1
fi
# Test pod-to-pod communication (if pods are running)
log_warn "Testing pod-to-pod communication..."
VALIDATOR_PODS=$(kubectl get pods -n "$NAMESPACE" -l component=validator -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
RPC_PODS=$(kubectl get pods -n "$NAMESPACE" -l component=rpc -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
if [ -n "$VALIDATOR_PODS" ] && [ -n "$RPC_PODS" ]; then
log_warn "Testing connectivity from validator to RPC (should be blocked)..."
# This should be blocked by Network Policies
if kubectl exec -n "$NAMESPACE" "$VALIDATOR_PODS" -- nc -zv besu-rpc-0.besu-rpc.besu-network.svc.cluster.local 8545 2>&1 | grep -q "Connection refused\|timeout"; then
log_success "✓ Network Policies are working (connection blocked as expected)"
else
log_warn "⚠ Connection test inconclusive (pods may not be ready)"
fi
else
log_warn "⚠ Pods not ready for connectivity testing"
fi
log_success "Network Policies validation completed"

View File

@@ -0,0 +1,89 @@
#!/usr/bin/env bash
# Validate RBAC Configuration
# This script validates that RBAC is correctly configured
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "$SCRIPT_DIR/../lib/init.sh"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
NAMESPACE="${NAMESPACE:-besu-network}"
log_success "Validating RBAC Configuration..."
# Check if Service Accounts exist
log_warn "Checking Service Accounts..."
SERVICE_ACCOUNTS=(
"besu-validator"
"besu-sentry"
"besu-rpc"
"oracle-publisher"
"rpc-gateway"
)
for sa in "${SERVICE_ACCOUNTS[@]}"; do
if kubectl get serviceaccount "$sa" -n "$NAMESPACE" &>/dev/null; then
log_success "✓ Service Account $sa exists"
else
log_warn "⚠ Service Account $sa not found, applying..."
kubectl apply -f "$PROJECT_ROOT/k8s/rbac/service-accounts.yaml"
break
fi
done
# Check if Roles exist
log_warn "Checking Roles..."
if kubectl get role keyvault-reader -n "$NAMESPACE" &>/dev/null; then
log_success "✓ Role keyvault-reader exists"
else
log_warn "⚠ Role keyvault-reader not found, applying..."
kubectl apply -f "$PROJECT_ROOT/k8s/rbac/service-accounts.yaml"
fi
# Check if RoleBindings exist
log_warn "Checking RoleBindings..."
ROLE_BINDINGS=(
"validator-keyvault-reader"
"oracle-keyvault-reader"
)
for rb in "${ROLE_BINDINGS[@]}"; do
if kubectl get rolebinding "$rb" -n "$NAMESPACE" &>/dev/null; then
log_success "✓ RoleBinding $rb exists"
else
log_warn "⚠ RoleBinding $rb not found"
fi
done
# Validate Service Account permissions
log_warn "Validating Service Account permissions..."
# Test if validator service account can read secrets
VALIDATOR_SA="besu-validator"
if kubectl auth can-i get secrets --as=system:serviceaccount:$NAMESPACE:$VALIDATOR_SA -n "$NAMESPACE" 2>/dev/null | grep -q "yes"; then
log_success "✓ Service Account $VALIDATOR_SA can read secrets"
else
log_warn "⚠ Service Account $VALIDATOR_SA cannot read secrets (may be expected)"
fi
# Check if pods are using correct service accounts
log_warn "Checking pod service accounts..."
VALIDATOR_PODS=$(kubectl get pods -n "$NAMESPACE" -l component=validator -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "")
if [ -n "$VALIDATOR_PODS" ]; then
for pod in $VALIDATOR_PODS; do
SA=$(kubectl get pod "$pod" -n "$NAMESPACE" -o jsonpath='{.spec.serviceAccountName}' 2>/dev/null || echo "")
if [ "$SA" == "$VALIDATOR_SA" ]; then
log_success "✓ Pod $pod is using correct service account"
else
log_warn "⚠ Pod $pod is using service account: $SA"
fi
done
else
log_warn "⚠ No validator pods found for validation"
fi
log_success "RBAC validation completed"