proxmox/scripts/maintenance/ensure-firefly-primary-via-ssh.sh

#!/usr/bin/env bash
# Ensure the Hyperledger FireFly primary on VMID 6200 has a valid compose file
# and an active systemd unit.
#
# Expected runtime:
# - VMID 6200 running on r630-02
# - /opt/firefly/docker-compose.yml present
# - firefly.service enabled and active
# - firefly-core, firefly-postgres, firefly-ipfs using restart=unless-stopped
# - GET /api/v1/status succeeds on localhost:5000
#
# Usage: ./scripts/maintenance/ensure-firefly-primary-via-ssh.sh [--dry-run]
# Env: PROXMOX_HOST_R630_02 (default 192.168.11.12)
set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
[[ -f "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" ]] && source "${PROJECT_ROOT}/scripts/lib/load-project-env.sh" 2>/dev/null || true

DRY_RUN=false
[[ "${1:-}" == "--dry-run" ]] && DRY_RUN=true

PROXMOX_HOST="${PROXMOX_HOST_R630_02:-192.168.11.12}"
VMID=6200

log_info() { echo -e "\033[0;34m[INFO]\033[0m $1"; }
log_ok()   { echo -e "\033[0;32m[✓]\033[0m $1"; }
log_err()  { echo -e "\033[0;31m[ERR]\033[0m $1"; }

run_ssh() { ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no root@"$PROXMOX_HOST" "$@"; }

normalize_compose() {
  if [[ "$DRY_RUN" == true ]]; then
    log_info "Would normalize /opt/firefly/docker-compose.yml and validate it with docker-compose config -q"
    return 0
  fi

  run_ssh "pct exec $VMID -- bash -lc '
    set -euo pipefail
    test -f /opt/firefly/docker-compose.yml
    if grep -qE \"^version:[[:space:]]*3\\.8[[:space:]]*$\" /opt/firefly/docker-compose.yml; then
      sed -i \"s/^version:[[:space:]]*3\\.8[[:space:]]*$/version: \\\"3.8\\\"/\" /opt/firefly/docker-compose.yml
    fi
    docker-compose -f /opt/firefly/docker-compose.yml config -q
  '"
}

install_firefly_helper() {
  if [[ "$DRY_RUN" == true ]]; then
    log_info "Would install an idempotent FireFly helper and systemd unit in VMID $VMID"
    return 0
  fi

  local helper_tmp unit_tmp
  helper_tmp="$(mktemp)"
  unit_tmp="$(mktemp)"

  cat > "$helper_tmp" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail

COMPOSE_FILE=/opt/firefly/docker-compose.yml
STATUS_URL=http://127.0.0.1:5000/api/v1/status

start_stack() {
  cd /opt/firefly
  test -f "$COMPOSE_FILE"
  if grep -qE '^version:[[:space:]]*3\.8[[:space:]]*$' "$COMPOSE_FILE"; then
    sed -i 's/^version:[[:space:]]*3\.8[[:space:]]*$/version: "3.8"/' "$COMPOSE_FILE"
  fi
  docker-compose -f "$COMPOSE_FILE" config -q
  docker-compose -f "$COMPOSE_FILE" up -d postgres ipfs >/dev/null

  if docker ps -a --format '{{.Names}}' | grep -qx firefly-core; then
    docker start firefly-core >/dev/null 2>&1 || true
  else
    docker-compose -f "$COMPOSE_FILE" up -d firefly-core >/dev/null
  fi

  curl -fsS "$STATUS_URL" >/dev/null
}

stop_stack() {
  docker stop firefly-core firefly-postgres firefly-ipfs >/dev/null 2>&1 || true
}

case "${1:-start}" in
  start)
    start_stack
    ;;
  stop)
    stop_stack
    ;;
  *)
    echo "Usage: $0 [start|stop]" >&2
    exit 64
    ;;
esac
EOF

  cat > "$unit_tmp" <<'EOF'
[Unit]
Description=Ensure Hyperledger FireFly primary stack
After=docker.service network-online.target
Requires=docker.service

[Service]
Type=oneshot
RemainAfterExit=yes
WorkingDirectory=/opt/firefly
User=firefly
Group=firefly
ExecStart=/usr/local/bin/ensure-firefly-primary start
ExecStop=/usr/local/bin/ensure-firefly-primary stop

[Install]
WantedBy=multi-user.target
EOF

  scp -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$helper_tmp" "root@$PROXMOX_HOST:/tmp/ensure-firefly-primary"
  scp -o ConnectTimeout=10 -o StrictHostKeyChecking=no "$unit_tmp" "root@$PROXMOX_HOST:/tmp/firefly.service"
  run_ssh "pct exec $VMID -- rm -f /usr/local/bin/ensure-firefly-primary /etc/systemd/system/firefly.service"
  run_ssh "pct push $VMID /tmp/ensure-firefly-primary /usr/local/bin/ensure-firefly-primary --perms 755"
  run_ssh "pct push $VMID /tmp/firefly.service /etc/systemd/system/firefly.service --perms 644"
  run_ssh "rm -f /tmp/ensure-firefly-primary /tmp/firefly.service"
  rm -f "$helper_tmp" "$unit_tmp"
}

ensure_firefly_service() {
  if [[ "$DRY_RUN" == true ]]; then
    log_info "Would reset-failed and enable/start firefly.service in VMID $VMID"
    return 0
  fi

  run_ssh "pct exec $VMID -- bash -lc '
    set -euo pipefail
    systemctl daemon-reload
    systemctl reset-failed firefly.service || true
    systemctl enable firefly.service >/dev/null 2>&1
    systemctl start firefly.service
  '"
}

verify_firefly_primary() {
  run_ssh "pct exec $VMID -- bash -lc '
    set -euo pipefail
    echo service=\$(systemctl is-active firefly.service)
    docker inspect -f \"{{.HostConfig.RestartPolicy.Name}}\" firefly-core | grep -qx unless-stopped
    docker inspect -f \"{{.HostConfig.RestartPolicy.Name}}\" firefly-postgres | grep -qx unless-stopped
    docker inspect -f \"{{.HostConfig.RestartPolicy.Name}}\" firefly-ipfs | grep -qx unless-stopped
    curl -fsS http://127.0.0.1:5000/api/v1/status
  '" 2>/dev/null
}

echo ""
echo "=== Ensure FireFly primary ==="
echo "  Host: $PROXMOX_HOST  vmid=$VMID  dry-run=$DRY_RUN"
echo ""

status="$(run_ssh "pct status $VMID 2>/dev/null | awk '{print \$2}'" 2>/dev/null || echo "missing")"
if [[ "$status" != "running" ]]; then
  if [[ "$DRY_RUN" == true ]]; then
    log_info "Would start VMID $VMID"
  else
    run_ssh "pct start $VMID"
    sleep 8
  fi
fi

normalize_compose
install_firefly_helper

if [[ "$DRY_RUN" == true ]]; then
  log_info "Would enable/start firefly.service and verify API health"
  exit 0
fi

ensure_firefly_service

if firefly_info="$(verify_firefly_primary)"; then
  log_ok "FireFly primary healthy"
  printf '%s\n' "$firefly_info"
else
  log_err "FireFly primary is still not healthy after normalization"
  exit 1
fi