#!/usr/bin/env bash # Generate source-of-truth JSON from verification outputs # Combines all verification results into canonical data model set -euo pipefail # Load IP configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" source "${PROJECT_ROOT}/config/ip-addresses.conf" 2>/dev/null || true SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" EVIDENCE_DIR="$PROJECT_ROOT/docs/04-configuration/verification-evidence" OUTPUT_FILE="$PROJECT_ROOT/docs/04-configuration/INGRESS_SOURCE_OF_TRUTH.json" # Colors RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' NC='\033[0m' log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } log_success() { echo -e "${GREEN}[✓]${NC} $1"; } log_warn() { echo -e "${YELLOW}[⚠]${NC} $1"; } log_error() { echo -e "${RED}[✗]${NC} $1"; } cd "$PROJECT_ROOT" echo "" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "🔍 Generate Source-of-Truth JSON" echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" echo "" # Find latest verification outputs LATEST_DNS_DIR=$(ls -td "$EVIDENCE_DIR"/dns-verification-* 2>/dev/null | head -1 || echo "") LATEST_UDM_DIR=$(ls -td "$EVIDENCE_DIR"/udm-pro-verification-* 2>/dev/null | head -1 || echo "") LATEST_NPM_DIR=$(ls -td "$EVIDENCE_DIR"/npmplus-verification-* 2>/dev/null | head -1 || echo "") LATEST_VM_DIR=$(ls -td "$EVIDENCE_DIR"/backend-vms-verification-* 2>/dev/null | head -1 || echo "") LATEST_E2E_DIR=$(ls -td "$EVIDENCE_DIR"/e2e-verification-* 2>/dev/null | head -1 || echo "") # Validate that source files exist log_info "Validating source files..." MISSING_FILES=() if [ -z "$LATEST_DNS_DIR" ] || [ ! -f "$LATEST_DNS_DIR/all_dns_records.json" ]; then log_warn "DNS verification results not found. Run: bash scripts/verify/export-cloudflare-dns-records.sh" MISSING_FILES+=("DNS verification") fi if [ -z "$LATEST_NPM_DIR" ] || [ ! -f "$LATEST_NPM_DIR/proxy_hosts.json" ]; then log_warn "NPMplus verification results not found. Run: bash scripts/verify/export-npmplus-config.sh" MISSING_FILES+=("NPMplus verification") fi if [ -z "$LATEST_VM_DIR" ] || [ ! -f "$LATEST_VM_DIR/all_vms_verification.json" ]; then log_warn "Backend VM verification results not found. Run: bash scripts/verify/verify-backend-vms.sh" MISSING_FILES+=("Backend VM verification") fi if [ ${#MISSING_FILES[@]} -gt 0 ]; then log_warn "Some verification results are missing. Source of truth will be incomplete." log_info "Missing: ${MISSING_FILES[*]}" if [ "${CONTINUE_PARTIAL:-0}" = "1" ] || [ "${CONTINUE_PARTIAL}" = "true" ]; then log_info "Continuing (CONTINUE_PARTIAL=1)" else log_info "You can still generate a partial source of truth, or run full verification first." echo "" read -p "Continue with partial source of truth? (y/N): " -n 1 -r echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then log_info "Exiting. Run full verification first." exit 0 fi fi fi # Allow partial generation if at least DNS or NPM data exists if [ -z "$LATEST_DNS_DIR" ] && [ -z "$LATEST_NPM_DIR" ]; then log_error "No verification outputs found. Run verification scripts first." log_info "Required: DNS verification OR NPMplus verification" exit 1 fi log_info "Using verification outputs:" [ -n "$LATEST_DNS_DIR" ] && log_info " DNS: $(basename "$LATEST_DNS_DIR")" [ -n "$LATEST_UDM_DIR" ] && log_info " UDM Pro: $(basename "$LATEST_UDM_DIR")" [ -n "$LATEST_NPM_DIR" ] && log_info " NPMplus: $(basename "$LATEST_NPM_DIR")" [ -n "$LATEST_VM_DIR" ] && log_info " Backend VMs: $(basename "$LATEST_VM_DIR")" [ -n "$LATEST_E2E_DIR" ] && log_info " E2E: $(basename "$LATEST_E2E_DIR")" echo "" # Validate and load DNS records log_info "Loading DNS records..." DNS_RECORDS="[]" if [ -f "$LATEST_DNS_DIR/all_dns_records.json" ]; then if jq empty "$LATEST_DNS_DIR/all_dns_records.json" 2>/dev/null; then DNS_RECORDS=$(cat "$LATEST_DNS_DIR/all_dns_records.json" 2>/dev/null || echo "[]") else log_error "Invalid JSON in DNS records file" DNS_RECORDS="[]" fi fi # Validate and load NPMplus config log_info "Loading NPMplus configuration..." PROXY_HOSTS="[]" CERTIFICATES="[]" if [ -f "$LATEST_NPM_DIR/proxy_hosts.json" ]; then if jq empty "$LATEST_NPM_DIR/proxy_hosts.json" 2>/dev/null; then PROXY_HOSTS=$(cat "$LATEST_NPM_DIR/proxy_hosts.json" 2>/dev/null || echo "[]") else log_error "Invalid JSON in proxy hosts file" PROXY_HOSTS="[]" fi fi if [ -f "$LATEST_NPM_DIR/certificates.json" ]; then if jq empty "$LATEST_NPM_DIR/certificates.json" 2>/dev/null; then CERTIFICATES=$(cat "$LATEST_NPM_DIR/certificates.json" 2>/dev/null || echo "[]") else log_error "Invalid JSON in certificates file" CERTIFICATES="[]" fi fi # Validate and load backend VMs log_info "Loading backend VMs..." BACKEND_VMS="[]" if [ -f "$LATEST_VM_DIR/all_vms_verification.json" ]; then if jq empty "$LATEST_VM_DIR/all_vms_verification.json" 2>/dev/null; then BACKEND_VMS=$(cat "$LATEST_VM_DIR/all_vms_verification.json" 2>/dev/null || echo "[]") else log_error "Invalid JSON in backend VMs file" BACKEND_VMS="[]" fi fi # Validate and load UDM Pro config log_info "Loading UDM Pro configuration..." UDM_CONFIG="{}" if [ -f "$LATEST_UDM_DIR/verification_results.json" ]; then if jq empty "$LATEST_UDM_DIR/verification_results.json" 2>/dev/null; then UDM_CONFIG=$(cat "$LATEST_UDM_DIR/verification_results.json" 2>/dev/null || echo "{}") else log_error "Invalid JSON in UDM Pro config file" UDM_CONFIG="{}" fi fi # Build source-of-truth JSON log_info "Generating source-of-truth JSON..." # Transform DNS records dns_records_array=$(echo "$DNS_RECORDS" | jq -c '.[] | { zone: (.zone // ""), hostname: .name, record_type: .type, record_value: .content, proxied: (.proxied // false), ttl: (.ttl // 1), status: "verified", verified_at: (now | strftime("%Y-%m-%dT%H:%M:%SZ")), notes: "" }' 2>/dev/null || echo "[]") # Transform proxy hosts proxy_hosts_array=$(echo "$PROXY_HOSTS" | jq -c '.[] | { id: .id, domain_names: (.domain_names // []), forward_scheme: (.forward_scheme // "http"), forward_host: (.forward_host // ""), forward_port: (.forward_port // 80), ssl_certificate_id: (.certificate_id // null), force_ssl: (.ssl_forced // false), allow_websocket_upgrade: (.allow_websocket_upgrade // false), access_list_id: (.access_list_id // null), advanced_config: (.advanced_config // ""), status: "verified", verified_at: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) }' 2>/dev/null || echo "[]") # Transform certificates certificates_array=$(echo "$CERTIFICATES" | jq -c '.[] | { id: .id, provider_name: (.provider // "letsencrypt"), nice_name: (.nice_name // ""), domain_names: (.domain_names // []), expires_at: (if .expires then (.expires | strftime("%Y-%m-%dT%H:%M:%SZ")) else "" end), enabled: (.enabled // true), auto_renewal: (.auto_renewal // true), certificate_files: { fullchain: "/data/tls/certbot/live/npm-\(.id)/fullchain.pem", privkey: "/data/tls/certbot/live/npm-\(.id)/privkey.pem" }, status: "verified", verified_at: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) }' 2>/dev/null || echo "[]") # Transform backend VMs (keep as-is but ensure status) backend_vms_array=$(echo "$BACKEND_VMS" | jq -c '.[] | . + { status: (if .status then .status else "verified" end), verified_at: (now | strftime("%Y-%m-%dT%H:%M:%SZ")) }' 2>/dev/null || echo "[]") # Extract UDM Pro info udm_wan_ip=$(echo "$UDM_CONFIG" | jq -r '.expected_configuration.public_ip // "76.53.10.36"' 2>/dev/null || echo "76.53.10.36") udm_port_forwarding=$(echo "$UDM_CONFIG" | jq -c '.expected_configuration.port_forwarding_rules // []' 2>/dev/null || echo "[]") # Build complete JSON structure SOURCE_OF_TRUTH=$(jq -n \ --argjson dns_records "$(echo "$dns_records_array" | jq -s '.')" \ --argjson proxy_hosts "$(echo "$proxy_hosts_array" | jq -s '.')" \ --argjson certificates "$(echo "$certificates_array" | jq -s '.')" \ --argjson backend_vms "$(echo "$backend_vms_array" | jq -s '.')" \ --argjson port_forwarding "$udm_port_forwarding" \ --arg wan_ip "$udm_wan_ip" \ '{ metadata: { version: "1.0.0", last_verified: (now | strftime("%Y-%m-%dT%H:%M:%SZ")), verifier: (env.USER // "unknown"), baseline_docs: [ "docs/04-configuration/DNS_NPMPLUS_VM_COMPREHENSIVE_ARCHITECTURE.md", "docs/04-configuration/DNS_NPMPLUS_VM_STREAMLINED_TABLE.md" ] }, dns_records: $dns_records, edge_routing: { wan_ip: $wan_ip, port_forwarding_rules: $port_forwarding }, npmplus: { container: { vmid: 10233, host: "r630-01", host_ip: "${PROXMOX_HOST_R630_01:-192.168.11.11}", internal_ips: { eth0: "${IP_NPMPLUS_ETH0:-${IP_NPMPLUS_ETH0:-192.168.11.166}}", eth1: "${IP_NPMPLUS:-${IP_NPMPLUS:-192.168.11.167}}" }, management_ui: "https://${IP_NPMPLUS_ETH0:-${IP_NPMPLUS_ETH0:-192.168.11.166}}:81", status: "running" }, proxy_hosts: $proxy_hosts, certificates: $certificates }, backend_vms: $backend_vms, issues: [ { severity: "critical", component: "backend", domain: "sankofa.nexus", description: "Sankofa services not deployed, routing to Blockscout", status: "known", action_required: "Deploy Sankofa services and update NPMplus routing" } ] }' 2>/dev/null || echo "{}") # Write to file # Validate final JSON before writing if echo "$SOURCE_OF_TRUTH" | jq empty 2>/dev/null; then echo "$SOURCE_OF_TRUTH" | jq '.' > "$OUTPUT_FILE" log_success "Source of truth JSON validated and written" else log_error "Generated JSON is invalid - not writing file" exit 1 fi log_success "Source-of-truth JSON generated: $OUTPUT_FILE" # Show summary DNS_COUNT=$(echo "$SOURCE_OF_TRUTH" | jq '.dns_records | length' 2>/dev/null || echo "0") PROXY_COUNT=$(echo "$SOURCE_OF_TRUTH" | jq '.npmplus.proxy_hosts | length' 2>/dev/null || echo "0") CERT_COUNT=$(echo "$SOURCE_OF_TRUTH" | jq '.npmplus.certificates | length' 2>/dev/null || echo "0") VM_COUNT=$(echo "$SOURCE_OF_TRUTH" | jq '.backend_vms | length' 2>/dev/null || echo "0") log_info "" log_info "Summary:" log_info " DNS Records: $DNS_COUNT" log_info " Proxy Hosts: $PROXY_COUNT" log_info " Certificates: $CERT_COUNT" log_info " Backend VMs: $VM_COUNT"