#!/usr/bin/env bash # # Health watchdog for the BBE site. Probes the frontend's request-time routes and # the backend health endpoint on their local ports. On sustained failure it # restarts bbe-frontend and logs loudly to the journal, turning a silent multi-day # outage (like the /events 500) into a self-healing few-minute blip. # # Meant to be run by bbe-site-healthcheck.timer every few minutes. State is kept # in a tmp file so a single transient blip doesn't trigger a restart — we only act # after FAIL_THRESHOLD consecutive failing runs. set -uo pipefail FRONTEND_BASE="${FRONTEND_BASE:-http://127.0.0.1:4056}" API_BASE="${API_BASE:-http://127.0.0.1:4055}" FAIL_THRESHOLD="${FAIL_THRESHOLD:-2}" STATE_FILE="${STATE_FILE:-/tmp/bbe-healthcheck.fails}" TIMEOUT="${TIMEOUT:-10}" FRONTEND_PATHS=(/ /events /blog) log() { echo "bbe-healthcheck: $*"; } check() { local url="$1" code code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time "${TIMEOUT}" "${url}" 2>/dev/null || echo 000)" if [ "${code}" = "200" ]; then return 0 fi log "FAIL ${url} -> ${code}" return 1 } failed=0 for p in "${FRONTEND_PATHS[@]}"; do check "${FRONTEND_BASE}${p}" || failed=1 done check "${API_BASE}/api/health" || failed=1 if [ "${failed}" -eq 0 ]; then # Healthy: clear the failure counter and exit quietly. rm -f "${STATE_FILE}" log "OK (all probes 200)" exit 0 fi # Unhealthy: increment consecutive-failure counter. count=0 [ -f "${STATE_FILE}" ] && count="$(cat "${STATE_FILE}" 2>/dev/null || echo 0)" count=$((count + 1)) echo "${count}" > "${STATE_FILE}" log "unhealthy run ${count}/${FAIL_THRESHOLD}" if [ "${count}" -ge "${FAIL_THRESHOLD}" ]; then log "threshold reached — restarting bbe-frontend.service" if systemctl restart bbe-frontend 2>/dev/null; then log "restart issued" else log "ERROR: restart failed (needs privileges? see PolicyKit/sudoers note in ops/README.md)" fi rm -f "${STATE_FILE}" fi exit 1