proxmox/scripts/maintenance/ensure-fabric-sample-network-via-ssh.sh

#!/usr/bin/env bash
# Ensure the Hyperledger Fabric sample network on VMID 6000 is up, queryable,
# and boot-recoverable after container restarts.
#
# Expected runtime:
# - VMID 6000 running on r630-02
# - docker + nested LXC features enabled
# - fabric-samples test-network payload under /opt/fabric/fabric-samples/test-network
# - orderer.example.com, peer0.org1.example.com, peer0.org2.example.com running
# - peer channel getinfo -c mychannel succeeds for Org1
#
# Usage: ./scripts/maintenance/ensure-fabric-sample-network-via-ssh.sh [--dry-run]
# Env: PROXMOX_HOST_R630_02 (default 192.168.11.12)
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
[[ -f "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" ]] && source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true

DRY_RUN=false
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=true

PROXMOX_HOST="${PROXMOX_HOST_R630_02:-192.168.11.12}"
VMID=6000

log_info() { echo -e "\033[0;34m[INFO]\033[0m $1"; }
log_ok()   { echo -e "\033[0;32m[✓]\033[0m $1"; }
log_warn() { echo -e "\033[0;33m[⚠]\033[0m $1"; }
log_err()  { echo -e "\033[0;31m[ERR]\033[0m $1"; }

run_ssh() { ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "$@"; }
run_scp() { scp -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$@"; }

ensure_ct_features() {
  local conf="/etc/pve/lxc/${VMID}.conf"
  local features

  features="$(run_ssh "awk -F': ' '/^features:/{print \$2}' ${conf@Q} 2>/dev/null || true" | tr -d '\r\n')"
  if [[ "$features" == *"nesting=1"* && "$features" == *"keyctl=1"* ]]; then
    return 0
  fi

  if [[ "$DRY_RUN" == true ]]; then
    log_info "Would add features: nesting=1,keyctl=1 to VMID $VMID and restart the CT"
    return 0
  fi

  run_ssh "cp ${conf@Q} /root/${VMID}.conf.pre-codex.\$(date +%Y%m%d_%H%M%S)"
  if [[ -n "$features" ]]; then
    run_ssh "sed -i 's/^features:.*/features: nesting=1,keyctl=1/' ${conf@Q}"
  else
    run_ssh "printf '%s\n' 'features: nesting=1,keyctl=1' >> ${conf@Q}"
  fi
  run_ssh "pct shutdown $VMID --timeout 30 >/dev/null 2>&1 || pct stop $VMID >/dev/null 2>&1 || true"
  run_ssh "pct start $VMID"
  sleep 8
}

ensure_boot_service() {
  if [[ "$DRY_RUN" == true ]]; then
    log_info "Would install and enable fabric-sample-network.service in VMID $VMID"
    return 0
  fi

  local helper_tmp unit_tmp
  helper_tmp="$(mktemp)"
  unit_tmp="$(mktemp)"

  cat > "$helper_tmp" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

cd /opt/fabric/fabric-samples/test-network

verify() {
  docker ps --format '{{.Names}}' | grep -qx orderer.example.com
  docker ps --format '{{.Names}}' | grep -qx peer0.org1.example.com
  docker ps --format '{{.Names}}' | grep -qx peer0.org2.example.com
  export PATH=/opt/fabric/fabric-samples/bin:$PATH
  export FABRIC_CFG_PATH=/opt/fabric/fabric-samples/config
  export $(./setOrgEnv.sh Org1 | xargs)
  peer channel getinfo -c mychannel >/tmp/fabric-channel-info.txt
}

if verify 2>/dev/null; then
  exit 0
fi

./network.sh up >/tmp/fabric-network-up.log 2>&1 || true
verify
EOF

  cat > "$unit_tmp" <<'EOF'
[Unit]
Description=Ensure Hyperledger Fabric sample network
After=docker.service network-online.target
Requires=docker.service

[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory=/opt/fabric/fabric-samples/test-network
ExecStart=/usr/local/bin/ensure-fabric-sample-network

[Install]
WantedBy=multi-user.target
EOF

  run_scp "$helper_tmp" "root@$PROXMOX_HOST:/tmp/ensure-fabric-sample-network"
  run_scp "$unit_tmp" "root@$PROXMOX_HOST:/tmp/fabric-sample-network.service"
  run_ssh "pct exec $VMID -- rm -f /usr/local/bin/ensure-fabric-sample-network /etc/systemd/system/fabric-sample-network.service"
  run_ssh "pct push $VMID /tmp/ensure-fabric-sample-network /usr/local/bin/ensure-fabric-sample-network --perms 755"
  run_ssh "pct push $VMID /tmp/fabric-sample-network.service /etc/systemd/system/fabric-sample-network.service --perms 644"
  run_ssh "rm -f /tmp/ensure-fabric-sample-network /tmp/fabric-sample-network.service"
  rm -f "$helper_tmp" "$unit_tmp"

  run_ssh "pct exec $VMID -- bash -lc 'systemctl daemon-reload && systemctl enable fabric-sample-network.service >/dev/null 2>&1 && systemctl start fabric-sample-network.service'"
}

verify_fabric_sample_network() {
  run_ssh "pct exec $VMID -- bash -lc '
    set -euo pipefail
    cd /opt/fabric/fabric-samples/test-network
    echo service=\$(systemctl is-active fabric-sample-network.service 2>/dev/null || echo unknown)
    docker ps --format \"{{.Names}}\" | grep -qx orderer.example.com
    docker ps --format \"{{.Names}}\" | grep -qx peer0.org1.example.com
    docker ps --format \"{{.Names}}\" | grep -qx peer0.org2.example.com
    export PATH=/opt/fabric/fabric-samples/bin:\$PATH
    export FABRIC_CFG_PATH=/opt/fabric/fabric-samples/config
    export \$(./setOrgEnv.sh Org1 | xargs)
    peer channel getinfo -c mychannel >/tmp/fabric-channel-info.txt
    cat /tmp/fabric-channel-info.txt
  '" 2>/dev/null
}

restore_fabric_sample_network() {
  if [[ "$DRY_RUN" == true ]]; then
    log_info "Would run ./network.sh up inside VMID $VMID and then verify mychannel"
    return 0
  fi

  run_ssh "pct exec $VMID -- bash -lc '
    set -euo pipefail
    cd /opt/fabric/fabric-samples/test-network
    ./network.sh up >/tmp/fabric-network-up.log 2>&1 || true
    cat /tmp/fabric-network-up.log
  '"
}

echo ""
echo "=== Ensure Fabric sample network ==="
echo "  Host: $PROXMOX_HOST  vmid=$VMID  dry-run=$DRY_RUN"
echo ""

ensure_ct_features

status="$(run_ssh "pct status $VMID 2>/dev/null | awk '{print \$2}'" 2>/dev/null || echo "missing")"
if [[ "$status" != "running" ]]; then
  if [[ "$DRY_RUN" == true ]]; then
    log_info "Would start VMID $VMID"
  else
    run_ssh "pct start $VMID"
    sleep 8
  fi
fi

ensure_boot_service

if [[ "$DRY_RUN" == true ]]; then
  log_info "Would verify running orderer/peer containers and peer channel getinfo -c mychannel"
  exit 0
fi

if fabric_info="$(verify_fabric_sample_network)"; then
  log_ok "Fabric sample network already healthy"
  printf '%s\n' "$fabric_info"
  exit 0
fi

log_warn "Fabric sample network not fully healthy; attempting restore"
restore_fabric_sample_network

if fabric_info="$(verify_fabric_sample_network)"; then
  log_ok "Fabric sample network restored"
  printf '%s\n' "$fabric_info"
else
  log_err "Fabric sample network is still not healthy after restore attempt"
  exit 1
fi