prod: throttle persona scrape jobs, cap worker CPU, add job-failure alerting + offsite backup
- job: cap concurrent persona_analyze_account (real headless-Chromium scrape) jobs per owner to 1 in flight via CountActiveByOwnerAndTemplate; rejects with the new ErrTooManyActive (429) instead of letting a member queue up many scrapes and pin the shared 1-vCPU host, or double-schedule the same persona while a previous run is still executing. - harbor-worker@.service: CPUQuota=70% so nginx/gateway keep headroom while a scrape job briefly saturates Chromium. - backup.sh: default BACKUP_DIR now matches the systemd unit (/var/backups/harbor) instead of the stale /var/backups/haixun default. - add harbor-job-health.timer (every 5m): queries Mongo for jobs that failed in the last 15 minutes and writes a node-exporter textfile-collector metric (haixun_job_failures_recent), with new Prometheus alerts (PersonaScrapeJobsFailing / BackgroundJobsFailing / JobHealthMetricStale) so a stuck worker surfaces on its own instead of only via member reports — this would have caught the recent Playwright headless-shell bug directly. - add harbor-offsite-backup.timer + backup/offsite-sync.sh: pushes backup.sh's local backups to an off-host restic repository (client-side encrypted). Installed and enabled by default but a safe no-op until RESTIC_REPOSITORY/RESTIC_PASSWORD are set in harbor.env. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
d1e876629a
commit
8c1be42f4a
|
|
@ -34,6 +34,9 @@ var (
|
|||
ErrNotFound = errors.New("job not found")
|
||||
ErrForbidden = errors.New("job access denied")
|
||||
ErrIllegalStatus = errors.New("illegal job status transition")
|
||||
// ErrTooManyActive — owner already has a CPU-heavy job (e.g. persona account
|
||||
// scrape) in flight; caller should retry once it finishes.
|
||||
ErrTooManyActive = errors.New("too many active jobs")
|
||||
)
|
||||
|
||||
type Job struct {
|
||||
|
|
|
|||
|
|
@ -16,6 +16,10 @@ type Repository interface {
|
|||
ClaimNext(ctx context.Context, workerID string) (*Job, error)
|
||||
// CancelPendingByRef cancels pending/queued jobs of template+ref (e.g. reschedule renew).
|
||||
CancelPendingByRef(ctx context.Context, ownerUID int64, template, refID string) error
|
||||
// CountActiveByOwnerAndTemplate counts an owner's non-terminal (pending/
|
||||
// queued/running) jobs of the given template — used to cap concurrent
|
||||
// CPU-heavy jobs (e.g. Threads profile scrape) per member.
|
||||
CountActiveByOwnerAndTemplate(ctx context.Context, ownerUID int64, template string) (int64, error)
|
||||
// Delete hard-removes a job by id.
|
||||
Delete(ctx context.Context, id string) error
|
||||
// DeleteTerminalBefore removes terminal jobs whose completion time is before beforeNs.
|
||||
|
|
|
|||
|
|
@ -189,6 +189,21 @@ func (s *MemoryStore) CancelPendingByRef(_ context.Context, ownerUID int64, temp
|
|||
return nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) CountActiveByOwnerAndTemplate(_ context.Context, ownerUID int64, template string) (int64, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var n int64
|
||||
for _, j := range s.byID {
|
||||
if j.OwnerUID != ownerUID || j.TemplateType != template {
|
||||
continue
|
||||
}
|
||||
if j.Status == domain.StatusPending || j.Status == domain.StatusQueued || j.Status == domain.StatusRunning {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (s *MemoryStore) Delete(_ context.Context, id string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
|
|
|||
|
|
@ -216,6 +216,15 @@ func (s *MonStore) CancelPendingByRef(ctx context.Context, ownerUID int64, templ
|
|||
return nil
|
||||
}
|
||||
|
||||
func (s *MonStore) CountActiveByOwnerAndTemplate(ctx context.Context, ownerUID int64, template string) (int64, error) {
|
||||
filter := bson.M{
|
||||
"owner_uid": ownerUID,
|
||||
"template_type": template,
|
||||
"status": bson.M{"$in": []string{domain.StatusPending, domain.StatusQueued, domain.StatusRunning}},
|
||||
}
|
||||
return s.jobs.CountDocuments(ctx, filter)
|
||||
}
|
||||
|
||||
func (s *MonStore) Delete(ctx context.Context, id string) error {
|
||||
res, err := s.jobs.DeleteOne(ctx, bson.M{"_id": id})
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -24,16 +24,27 @@ type Service struct {
|
|||
Notifier Notifier
|
||||
LeaseDuration time.Duration
|
||||
HeartbeatInterval time.Duration
|
||||
// MaxActiveAccountScrapes caps how many persona_analyze_account jobs
|
||||
// (real headless-Chromium scrape; CPU-heavy on the shared prod host) a
|
||||
// single owner may have pending/queued/running at once. <=0 disables
|
||||
// the check.
|
||||
MaxActiveAccountScrapes int
|
||||
|
||||
mu sync.Mutex
|
||||
workerID string
|
||||
heartbeats map[string]context.CancelFunc
|
||||
}
|
||||
|
||||
// DefaultMaxActiveAccountScrapes — one in-flight browser scrape per member;
|
||||
// keeps the 1-vCPU prod host from being flooded by many personas being
|
||||
// analyzed back-to-back (see deploy/prod/README.md).
|
||||
const DefaultMaxActiveAccountScrapes = 1
|
||||
|
||||
func New(repo domain.Repository) *Service {
|
||||
return &Service{
|
||||
Repo: repo, LeaseDuration: domain.DefaultLeaseDuration,
|
||||
HeartbeatInterval: domain.DefaultHeartbeat,
|
||||
MaxActiveAccountScrapes: DefaultMaxActiveAccountScrapes,
|
||||
heartbeats: make(map[string]context.CancelFunc),
|
||||
}
|
||||
}
|
||||
|
|
@ -325,6 +336,15 @@ func (s *Service) SchedulePersonaAnalyzeAccount(ctx context.Context, ownerUID in
|
|||
}
|
||||
_ = s.Repo.CancelPendingByRef(ctx, ownerUID, domain.TemplatePersonaAnalyzeAccount, personaID)
|
||||
_ = s.Repo.CancelPendingByRef(ctx, ownerUID, domain.TemplatePersonaAnalyzeText, personaID)
|
||||
if s.MaxActiveAccountScrapes > 0 {
|
||||
active, err := s.Repo.CountActiveByOwnerAndTemplate(ctx, ownerUID, domain.TemplatePersonaAnalyzeAccount)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if active >= int64(s.MaxActiveAccountScrapes) {
|
||||
return nil, domain.ErrTooManyActive
|
||||
}
|
||||
}
|
||||
body, _ := json.Marshal(PersonaAnalyzeAccountPayload{Username: username, Lang: lang})
|
||||
now := domain.NowNano()
|
||||
j := &domain.Job{
|
||||
|
|
|
|||
|
|
@ -226,6 +226,39 @@ func TestJB_15_SchedulePersonaAnalyzeAccount(t *testing.T) {
|
|||
require.Equal(t, domain.StatusRunning, claimed.Status)
|
||||
}
|
||||
|
||||
func TestJB_15b_SchedulePersonaAnalyzeAccount_ThrottlesConcurrentScrapes(t *testing.T) {
|
||||
svc, _ := newJobSvc()
|
||||
uid := int64(1_000_117)
|
||||
// first persona's scrape stays queued (not claimed yet)
|
||||
first, err := svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_one", "acct_one", "zh-TW")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, domain.StatusQueued, first.Status)
|
||||
|
||||
// a different persona for the same owner must be rejected while one is in flight
|
||||
_, err = svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_two", "acct_two", "zh-TW")
|
||||
require.ErrorIs(t, err, domain.ErrTooManyActive)
|
||||
|
||||
// re-scheduling the SAME persona still works (cancels its own pending job first)
|
||||
replaced, err := svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_one", "acct_one_v2", "zh-TW")
|
||||
require.NoError(t, err)
|
||||
require.NotEqual(t, first.ID, replaced.ID)
|
||||
|
||||
// once the in-flight job finishes, a different persona may schedule again
|
||||
claimed, err := svc.ClaimNext(context.Background(), "w1")
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, replaced.ID, claimed.ID)
|
||||
_, err = svc.FailJob(context.Background(), claimed.ID, "boom")
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_two", "acct_two", "zh-TW")
|
||||
require.NoError(t, err)
|
||||
|
||||
// a different owner is never affected by another owner's in-flight scrape
|
||||
otherUID := int64(1_000_118)
|
||||
_, err = svc.SchedulePersonaAnalyzeAccount(context.Background(), otherUID, "pe_other", "acct_other", "zh-TW")
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
func TestJB_16_SchedulePersonaAnalyzeText(t *testing.T) {
|
||||
svc, _ := newJobSvc()
|
||||
uid := int64(1_000_116)
|
||||
|
|
|
|||
|
|
@ -153,6 +153,8 @@ func mapError(err error) (int, Envelope) {
|
|||
return http.StatusBadRequest, Envelope{Code: 400033, Message: "session refresh failed"}
|
||||
case errors.Is(err, jobDomain.ErrIllegalStatus):
|
||||
return http.StatusConflict, Envelope{Code: 409010, Message: "illegal job status transition"}
|
||||
case errors.Is(err, jobDomain.ErrTooManyActive):
|
||||
return http.StatusTooManyRequests, Envelope{Code: 429010, Message: "已有一個人設在爬取公開貼文，請等該任務完成後再試"}
|
||||
case errors.Is(err, usageDomain.ErrNoKey):
|
||||
return http.StatusBadRequest, Envelope{Code: 400040, Message: "no api key configured (platform or byok)"}
|
||||
case errors.Is(err, usageDomain.ErrQuotaExceeded):
|
||||
|
|
|
|||
|
|
@ -135,9 +135,35 @@ ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \
|
|||
journalctl -u 'harbor-gateway@*' -f
|
||||
```
|
||||
|
||||
### Background job health
|
||||
|
||||
`harbor-job-health.timer` runs every 5 minutes and writes a node-exporter
|
||||
textfile-collector metric (`haixun_job_failures_recent`) counting jobs that
|
||||
failed in the last 15 minutes, overall and specifically for
|
||||
`persona_analyze_account` (the Threads profile scrape). Prometheus rules
|
||||
`PersonaScrapeJobsFailing` / `BackgroundJobsFailing` /
|
||||
`JobHealthMetricStale` in `monitoring/prometheus/rules/alerts.yml` alert on
|
||||
this so a stuck worker (e.g. a missing Playwright browser) surfaces on its
|
||||
own instead of only being noticed when a member reports it.
|
||||
|
||||
## Backups
|
||||
|
||||
`harbor-backup.timer` creates daily Mongo and MinIO backups under
|
||||
`/var/backups/harbor` and retains 14 days. This protects against accidental
|
||||
deletion but not loss of the machine or disk. Add an off-host copy before using
|
||||
the service for irreplaceable production data.
|
||||
deletion but not loss of the machine or disk.
|
||||
|
||||
`harbor-offsite-backup.timer` runs `backup/offsite-sync.sh` an hour later and
|
||||
pushes those local backups to an off-host [restic](https://restic.net/)
|
||||
repository (client-side encrypted before upload). It is installed and enabled
|
||||
by default but is a no-op until you set `RESTIC_REPOSITORY` and
|
||||
`RESTIC_PASSWORD` (plus provider credentials, e.g. `AWS_ACCESS_KEY_ID` /
|
||||
`AWS_SECRET_ACCESS_KEY` for an S3-compatible bucket) in
|
||||
`/etc/harbor/harbor.env` — see `harbor.env.example` for the format. Once
|
||||
configured, verify with:
|
||||
|
||||
```bash
|
||||
ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \
|
||||
sudo systemctl start harbor-offsite-backup.service
|
||||
ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \
|
||||
sudo journalctl -u harbor-offsite-backup.service -n 50
|
||||
```
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ set -eu
|
|||
umask 077
|
||||
|
||||
SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
|
||||
BACKUP_DIR=${BACKUP_DIR:-/var/backups/haixun}
|
||||
BACKUP_DIR=${BACKUP_DIR:-/var/backups/harbor}
|
||||
BACKUP_RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-14}
|
||||
BACKUP_STATUS_DIR=${BACKUP_STATUS_DIR:-"$SCRIPT_DIR/status"}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Pushes backup.sh's local backups to an off-host restic repository so
|
||||
# production member data survives loss of the machine or disk itself, not
|
||||
# just accidental deletion (see README "Backups"). restic encrypts
|
||||
# everything client-side before upload, so RESTIC_PASSWORD protects the
|
||||
# data even if the remote bucket credentials leak.
|
||||
#
|
||||
# No-op until RESTIC_REPOSITORY/RESTIC_PASSWORD are configured in
|
||||
# /etc/harbor/harbor.env — safe to enable the timer before picking a
|
||||
# provider.
|
||||
|
||||
BACKUP_DIR=${BACKUP_DIR:-/var/backups/harbor}
|
||||
BACKUP_STATUS_DIR=${BACKUP_STATUS_DIR:-/opt/harbor/deploy/backup/status}
|
||||
OFFSITE_BACKUP_RETENTION_DAYS=${OFFSITE_BACKUP_RETENTION_DAYS:-30}
|
||||
|
||||
if [ -z "${RESTIC_REPOSITORY:-}" ] || { [ -z "${RESTIC_PASSWORD:-}" ] && [ -z "${RESTIC_PASSWORD_FILE:-}" ]; }; then
|
||||
printf '%s\n' "offsite-sync: RESTIC_REPOSITORY/RESTIC_PASSWORD not configured, skipping" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! command -v restic >/dev/null 2>&1; then
|
||||
printf '%s\n' "offsite-sync: restic not installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$BACKUP_DIR" ] || [ -z "$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.*.tmp' -print -quit)" ]; then
|
||||
printf '%s\n' "offsite-sync: no local backup found under $BACKUP_DIR yet, skipping" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
host_tag=$(hostname)
|
||||
|
||||
restic snapshots >/dev/null 2>&1 || restic init
|
||||
|
||||
restic backup --tag harbor-backup --host "$host_tag" "$BACKUP_DIR"
|
||||
restic forget --tag harbor-backup --host "$host_tag" --keep-within "${OFFSITE_BACKUP_RETENTION_DAYS}d" --prune
|
||||
|
||||
mkdir -p "$BACKUP_STATUS_DIR"
|
||||
marker_tmp="$BACKUP_STATUS_DIR/.haixun_offsite_backup.prom.tmp"
|
||||
{
|
||||
printf '# HELP haixun_offsite_backup_last_success_timestamp_seconds Unix timestamp of the last successful offsite backup sync.\n'
|
||||
printf '# TYPE haixun_offsite_backup_last_success_timestamp_seconds gauge\n'
|
||||
printf 'haixun_offsite_backup_last_success_timestamp_seconds %s\n' "$(date -u +%s)"
|
||||
} > "$marker_tmp"
|
||||
chmod 0644 "$marker_tmp"
|
||||
mv "$marker_tmp" "$BACKUP_STATUS_DIR/haixun_offsite_backup.prom"
|
||||
|
||||
printf '%s\n' "offsite-sync: pushed $BACKUP_DIR to $RESTIC_REPOSITORY"
|
||||
|
|
@ -26,3 +26,13 @@ MAIL_SMTP_HOST=
|
|||
MAIL_SMTP_PORT=587
|
||||
MAIL_SMTP_USER=
|
||||
MAIL_SMTP_PASSWORD=
|
||||
|
||||
# Offsite backup sync (backup/offsite-sync.sh, runs from harbor-offsite-backup.timer
|
||||
# daily after the local backup). Leave RESTIC_REPOSITORY empty to keep the sync a
|
||||
# no-op. RESTIC_REPOSITORY/RESTIC_PASSWORD follow restic's own format, e.g. an
|
||||
# S3-compatible bucket: RESTIC_REPOSITORY=s3:https://s3.us-west-000.backblazeb2.com/my-bucket/harbor
|
||||
# plus AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY below for that provider.
|
||||
RESTIC_REPOSITORY=
|
||||
RESTIC_PASSWORD=
|
||||
AWS_ACCESS_KEY_ID=
|
||||
AWS_SECRET_ACCESS_KEY=
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Writes a node-exporter textfile-collector metric so a run of failing
|
||||
# background jobs (e.g. the Threads persona scrape) shows up in Prometheus
|
||||
# instead of only being noticed when a member reports it. Reuses the same
|
||||
# textfile-collector directory as backup.sh's success marker.
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
printf '%s\n' "job-health-check must run as root" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STATUS_DIR=${BACKUP_STATUS_DIR:-/opt/harbor/deploy/backup/status}
|
||||
WINDOW_MINUTES=${JOB_HEALTH_WINDOW_MINUTES:-15}
|
||||
COMPOSE_FILE=${COMPOSE_FILE:-/opt/harbor/deploy/compose/docker-compose.yml}
|
||||
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. /etc/harbor/harbor.env
|
||||
set +a
|
||||
: "${MONGO_URI:?MONGO_URI is required}"
|
||||
|
||||
now_s=$(date -u +%s)
|
||||
since_ns=$(( (now_s - WINDOW_MINUTES * 60) * 1000000000 ))
|
||||
|
||||
compose() {
|
||||
docker compose --env-file /etc/harbor/harbor.env -f "$COMPOSE_FILE" "$@"
|
||||
}
|
||||
|
||||
query_count() {
|
||||
# $1 = extra mongosh filter fragment (may be empty)
|
||||
# mongosh --eval resolves the returned promise before printing; top-level
|
||||
# `await` is a syntax error there, so call the collection method directly.
|
||||
eval_js="db.jobs.countDocuments({status:'failed', completed_at:{\$gt: ${since_ns}}${1}})"
|
||||
compose exec -T mongo mongosh "$MONGO_URI" --quiet --eval "$eval_js" | tr -d '[:space:]'
|
||||
}
|
||||
|
||||
scrape_failures=$(query_count ", template_type:'persona_analyze_account'")
|
||||
total_failures=$(query_count "")
|
||||
|
||||
case "$scrape_failures" in ''|*[!0-9]*) scrape_failures=0 ;; esac
|
||||
case "$total_failures" in ''|*[!0-9]*) total_failures=0 ;; esac
|
||||
|
||||
mkdir -p "$STATUS_DIR"
|
||||
tmp="$STATUS_DIR/.haixun_job_failures.prom.tmp"
|
||||
{
|
||||
printf '# HELP haixun_job_failures_recent Jobs that failed in the last %s minutes.\n' "$WINDOW_MINUTES"
|
||||
printf '# TYPE haixun_job_failures_recent gauge\n'
|
||||
printf 'haixun_job_failures_recent{template="persona_analyze_account"} %s\n' "$scrape_failures"
|
||||
printf 'haixun_job_failures_recent{template="_all_"} %s\n' "$total_failures"
|
||||
} > "$tmp"
|
||||
chmod 0644 "$tmp"
|
||||
mv "$tmp" "$STATUS_DIR/haixun_job_failures.prom"
|
||||
|
|
@ -90,3 +90,30 @@ groups:
|
|||
annotations:
|
||||
summary: "Production backup is missing or stale"
|
||||
description: "No successful MongoDB and MinIO backup has been recorded in the last 26 hours."
|
||||
|
||||
- alert: JobHealthMetricStale
|
||||
expr: absent(haixun_job_failures_recent{template="_all_"})
|
||||
for: 30m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Job health metric missing"
|
||||
description: "harbor-job-health.timer has not reported job failure counts; the metric may be stale or the timer stopped running."
|
||||
|
||||
- alert: PersonaScrapeJobsFailing
|
||||
expr: haixun_job_failures_recent{template="persona_analyze_account"} >= 2
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Threads persona scrape jobs are repeatedly failing"
|
||||
description: "{{ $value }} persona_analyze_account job(s) failed in the last 15 minutes. Check harbor-worker@*.service logs (e.g. missing Playwright browser, Threads layout change)."
|
||||
|
||||
- alert: BackgroundJobsFailing
|
||||
expr: haixun_job_failures_recent{template="_all_"} >= 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "Background jobs are repeatedly failing"
|
||||
description: "{{ $value }} job(s) across all templates failed in the last 15 minutes. Check harbor-worker@*.service logs."
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ fi
|
|||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates certbot curl jq nginx openssl python3-certbot-dns-cloudflare rsync tar gzip ufw fail2ban docker.io docker-compose-v2 nodejs npm
|
||||
apt-get install -y ca-certificates certbot curl jq nginx openssl python3-certbot-dns-cloudflare restic rsync tar gzip ufw fail2ban docker.io docker-compose-v2 nodejs npm
|
||||
systemctl enable --now docker nginx fail2ban
|
||||
|
||||
if ! id harbor >/dev/null 2>&1; then
|
||||
|
|
@ -42,13 +42,17 @@ if [[ ! -d /var/lib/harbor/.cache/ms-playwright ]] \
|
|||
fi
|
||||
rsync -a --delete "$SOURCE_DIR/" /opt/harbor/deploy/
|
||||
chown -R root:root /opt/harbor/deploy
|
||||
chmod +x /opt/harbor/deploy/remote/*.sh /opt/harbor/deploy/backup/backup.sh
|
||||
chmod +x /opt/harbor/deploy/remote/*.sh /opt/harbor/deploy/backup/backup.sh /opt/harbor/deploy/backup/offsite-sync.sh /opt/harbor/deploy/monitoring/job-health-check.sh
|
||||
|
||||
install -m 0644 /opt/harbor/deploy/config/gateway.yaml /etc/harbor/gateway.yaml
|
||||
install -m 0644 /opt/harbor/deploy/systemd/harbor-gateway@.service /etc/systemd/system/harbor-gateway@.service
|
||||
install -m 0644 /opt/harbor/deploy/systemd/harbor-worker@.service /etc/systemd/system/harbor-worker@.service
|
||||
install -m 0644 /opt/harbor/deploy/systemd/harbor-backup.service /etc/systemd/system/harbor-backup.service
|
||||
install -m 0644 /opt/harbor/deploy/systemd/harbor-backup.timer /etc/systemd/system/harbor-backup.timer
|
||||
install -m 0644 /opt/harbor/deploy/systemd/harbor-job-health.service /etc/systemd/system/harbor-job-health.service
|
||||
install -m 0644 /opt/harbor/deploy/systemd/harbor-job-health.timer /etc/systemd/system/harbor-job-health.timer
|
||||
install -m 0644 /opt/harbor/deploy/systemd/harbor-offsite-backup.service /etc/systemd/system/harbor-offsite-backup.service
|
||||
install -m 0644 /opt/harbor/deploy/systemd/harbor-offsite-backup.timer /etc/systemd/system/harbor-offsite-backup.timer
|
||||
|
||||
if [[ ! -f /etc/harbor/harbor.env ]]; then
|
||||
mongo_password=$(openssl rand -hex 24)
|
||||
|
|
@ -173,6 +177,10 @@ ufw --force enable
|
|||
nginx -t
|
||||
systemctl reload nginx
|
||||
systemctl enable --now harbor-backup.timer
|
||||
systemctl enable --now harbor-job-health.timer
|
||||
# offsite-sync.sh no-ops until RESTIC_REPOSITORY/RESTIC_PASSWORD are set in
|
||||
# harbor.env, so it's safe to always enable the timer.
|
||||
systemctl enable --now harbor-offsite-backup.timer
|
||||
|
||||
printf '%s\n' "bootstrap complete"
|
||||
printf '%s\n' "runtime secrets: /etc/harbor/harbor.env"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
[Unit]
|
||||
Description=Harbor Desk background job health metrics
|
||||
After=docker.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
EnvironmentFile=/etc/harbor/harbor.env
|
||||
Environment=BACKUP_STATUS_DIR=/opt/harbor/deploy/backup/status
|
||||
ExecStart=/opt/harbor/deploy/monitoring/job-health-check.sh
|
||||
User=root
|
||||
Group=root
|
||||
Nice=10
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
[Unit]
|
||||
Description=Check background job health every 5 minutes
|
||||
|
||||
[Timer]
|
||||
OnBootSec=2min
|
||||
OnUnitActiveSec=5min
|
||||
RandomizedDelaySec=30
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
[Unit]
|
||||
Description=Harbor Desk offsite backup sync
|
||||
After=docker.service harbor-backup.service
|
||||
Requires=docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
EnvironmentFile=/etc/harbor/harbor.env
|
||||
Environment=BACKUP_DIR=/var/backups/harbor
|
||||
Environment=BACKUP_STATUS_DIR=/opt/harbor/deploy/backup/status
|
||||
ExecStart=/opt/harbor/deploy/backup/offsite-sync.sh
|
||||
User=root
|
||||
Group=root
|
||||
Nice=10
|
||||
IOSchedulingClass=best-effort
|
||||
IOSchedulingPriority=7
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
[Unit]
|
||||
Description=Run Harbor Desk offsite backup sync daily (after the local backup)
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 04:15:00
|
||||
RandomizedDelaySec=900
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -20,6 +20,9 @@ PrivateTmp=true
|
|||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/var/lib/harbor
|
||||
# The host has 1 vCPU; a persona-scrape job briefly saturates Chromium. Cap the
|
||||
# worker so nginx/gateway keep some headroom instead of queueing behind it.
|
||||
CPUQuota=70%
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
SyslogIdentifier=harbor-worker-%i
|
||||
|
|
|
|||
Loading…
Reference in New Issue