55 lines
2.0 KiB
Bash
55 lines
2.0 KiB
Bash
|
#!/bin/sh
|
||
|
|
set -eu
|
||
|
|
|
||
|
|
# Writes a node-exporter textfile-collector metric so a run of failing
|
||
|
|
# background jobs (e.g. the Threads persona scrape) shows up in Prometheus
|
||
|
|
# instead of only being noticed when a member reports it. Reuses the same
|
||
|
|
# textfile-collector directory as backup.sh's success marker.
|
||
|
|
|
||
|
|
if [ "$(id -u)" -ne 0 ]; then
|
||
|
|
printf '%s\n' "job-health-check must run as root" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
STATUS_DIR=${BACKUP_STATUS_DIR:-/opt/harbor/deploy/backup/status}
|
||
|
|
WINDOW_MINUTES=${JOB_HEALTH_WINDOW_MINUTES:-15}
|
||
|
|
COMPOSE_FILE=${COMPOSE_FILE:-/opt/harbor/deploy/compose/docker-compose.yml}
|
||
|
|
|
||
|
|
set -a
|
||
|
|
# shellcheck disable=SC1091
|
||
|
|
. /etc/harbor/harbor.env
|
||
|
|
set +a
|
||
|
|
: "${MONGO_URI:?MONGO_URI is required}"
|
||
|
|
|
||
|
|
now_s=$(date -u +%s)
|
||
|
|
since_ns=$(( (now_s - WINDOW_MINUTES * 60) * 1000000000 ))
|
||
|
|
|
||
|
|
compose() {
|
||
|
|
docker compose --env-file /etc/harbor/harbor.env -f "$COMPOSE_FILE" "$@"
|
||
|
|
}
|
||
|
|
|
||
|
|
query_count() {
|
||
|
|
# $1 = extra mongosh filter fragment (may be empty)
|
||
|
|
# mongosh --eval resolves the returned promise before printing; top-level
|
||
|
|
# `await` is a syntax error there, so call the collection method directly.
|
||
|
|
eval_js="db.jobs.countDocuments({status:'failed', completed_at:{\$gt: ${since_ns}}${1}})"
|
||
|
|
compose exec -T mongo mongosh "$MONGO_URI" --quiet --eval "$eval_js" | tr -d '[:space:]'
|
||
|
|
}
|
||
|
|
|
||
|
|
scrape_failures=$(query_count ", template_type:'persona_analyze_account'")
|
||
|
|
total_failures=$(query_count "")
|
||
|
|
|
||
|
|
case "$scrape_failures" in ''|*[!0-9]*) scrape_failures=0 ;; esac
|
||
|
|
case "$total_failures" in ''|*[!0-9]*) total_failures=0 ;; esac
|
||
|
|
|
||
|
|
mkdir -p "$STATUS_DIR"
|
||
|
|
tmp="$STATUS_DIR/.haixun_job_failures.prom.tmp"
|
||
|
|
{
|
||
|
|
printf '# HELP haixun_job_failures_recent Jobs that failed in the last %s minutes.\n' "$WINDOW_MINUTES"
|
||
|
|
printf '# TYPE haixun_job_failures_recent gauge\n'
|
||
|
|
printf 'haixun_job_failures_recent{template="persona_analyze_account"} %s\n' "$scrape_failures"
|
||
|
|
printf 'haixun_job_failures_recent{template="_all_"} %s\n' "$total_failures"
|
||
|
|
} > "$tmp"
|
||
|
|
chmod 0644 "$tmp"
|
||
|
|
mv "$tmp" "$STATUS_DIR/haixun_job_failures.prom"
|