prod: throttle persona scrape jobs, cap worker CPU, add job-failure alerting + offsite backup

- job: cap concurrent persona_analyze_account (real headless-Chromium scrape)
  jobs per owner to 1 in flight via CountActiveByOwnerAndTemplate; rejects
  with the new ErrTooManyActive (429) instead of letting a member queue up
  many scrapes and pin the shared 1-vCPU host, or double-schedule the same
  persona while a previous run is still executing.
- harbor-worker@.service: CPUQuota=70% so nginx/gateway keep headroom while
  a scrape job briefly saturates Chromium.
- backup.sh: default BACKUP_DIR now matches the systemd unit (/var/backups/harbor)
  instead of the stale /var/backups/haixun default.
- add harbor-job-health.timer (every 5m): queries Mongo for jobs that failed
  in the last 15 minutes and writes a node-exporter textfile-collector
  metric (haixun_job_failures_recent), with new Prometheus alerts
  (PersonaScrapeJobsFailing / BackgroundJobsFailing / JobHealthMetricStale)
  so a stuck worker surfaces on its own instead of only via member reports —
  this would have caught the recent Playwright headless-shell bug directly.
- add harbor-offsite-backup.timer + backup/offsite-sync.sh: pushes
  backup.sh's local backups to an off-host restic repository (client-side
  encrypted). Installed and enabled by default but a safe no-op until
  RESTIC_REPOSITORY/RESTIC_PASSWORD are set in harbor.env.

Co-authored-by: Cursor <cursoragent@cursor.com>

apps/backend/internal/module/job/domain/job.go

@@ -34,6 +34,9 @@ var (
 	ErrNotFound      = errors.New("job not found")
 	ErrForbidden     = errors.New("job access denied")
 	ErrIllegalStatus = errors.New("illegal job status transition")
+	// ErrTooManyActive — owner already has a CPU-heavy job (e.g. persona account
+	// scrape) in flight; caller should retry once it finishes.
+	ErrTooManyActive = errors.New("too many active jobs")
 )
 
 type Job struct {

apps/backend/internal/module/job/domain/repository.go

@@ -16,6 +16,10 @@ type Repository interface {
 	ClaimNext(ctx context.Context, workerID string) (*Job, error)
 	// CancelPendingByRef cancels pending/queued jobs of template+ref (e.g. reschedule renew).
 	CancelPendingByRef(ctx context.Context, ownerUID int64, template, refID string) error
+	// CountActiveByOwnerAndTemplate counts an owner's non-terminal (pending/
+	// queued/running) jobs of the given template — used to cap concurrent
+	// CPU-heavy jobs (e.g. Threads profile scrape) per member.
+	CountActiveByOwnerAndTemplate(ctx context.Context, ownerUID int64, template string) (int64, error)
 	// Delete hard-removes a job by id.
 	Delete(ctx context.Context, id string) error
 	// DeleteTerminalBefore removes terminal jobs whose completion time is before beforeNs.

apps/backend/internal/module/job/repository/memory.go

@@ -189,6 +189,21 @@ func (s *MemoryStore) CancelPendingByRef(_ context.Context, ownerUID int64, temp
 	return nil
 }
 
+func (s *MemoryStore) CountActiveByOwnerAndTemplate(_ context.Context, ownerUID int64, template string) (int64, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	var n int64
+	for _, j := range s.byID {
+		if j.OwnerUID != ownerUID || j.TemplateType != template {
+			continue
+		}
+		if j.Status == domain.StatusPending || j.Status == domain.StatusQueued || j.Status == domain.StatusRunning {
+			n++
+		}
+	}
+	return n, nil
+}
+
 func (s *MemoryStore) Delete(_ context.Context, id string) error {
 	s.mu.Lock()
 	defer s.mu.Unlock()

apps/backend/internal/module/job/repository/mongo.go

@@ -216,6 +216,15 @@ func (s *MonStore) CancelPendingByRef(ctx context.Context, ownerUID int64, templ
 	return nil
 }
 
+func (s *MonStore) CountActiveByOwnerAndTemplate(ctx context.Context, ownerUID int64, template string) (int64, error) {
+	filter := bson.M{
+		"owner_uid":     ownerUID,
+		"template_type": template,
+		"status":        bson.M{"$in": []string{domain.StatusPending, domain.StatusQueued, domain.StatusRunning}},
+	}
+	return s.jobs.CountDocuments(ctx, filter)
+}
+
 func (s *MonStore) Delete(ctx context.Context, id string) error {
 	res, err := s.jobs.DeleteOne(ctx, bson.M{"_id": id})
 	if err != nil {

apps/backend/internal/module/job/usecase/service.go

@@ -24,16 +24,27 @@ type Service struct {
 	Notifier          Notifier
 	LeaseDuration     time.Duration
 	HeartbeatInterval time.Duration
+	// MaxActiveAccountScrapes caps how many persona_analyze_account jobs
+	// (real headless-Chromium scrape; CPU-heavy on the shared prod host) a
+	// single owner may have pending/queued/running at once. <=0 disables
+	// the check.
+	MaxActiveAccountScrapes int
 
 	mu         sync.Mutex
 	workerID   string
 	heartbeats map[string]context.CancelFunc
 }
 
+// DefaultMaxActiveAccountScrapes — one in-flight browser scrape per member;
+// keeps the 1-vCPU prod host from being flooded by many personas being
+// analyzed back-to-back (see deploy/prod/README.md).
+const DefaultMaxActiveAccountScrapes = 1
+
 func New(repo domain.Repository) *Service {
 	return &Service{
 		Repo: repo, LeaseDuration: domain.DefaultLeaseDuration,
 		HeartbeatInterval:       domain.DefaultHeartbeat,
+		MaxActiveAccountScrapes: DefaultMaxActiveAccountScrapes,
 		heartbeats:              make(map[string]context.CancelFunc),
 	}
 }
@@ -325,6 +336,15 @@ func (s *Service) SchedulePersonaAnalyzeAccount(ctx context.Context, ownerUID in
 	}
 	_ = s.Repo.CancelPendingByRef(ctx, ownerUID, domain.TemplatePersonaAnalyzeAccount, personaID)
 	_ = s.Repo.CancelPendingByRef(ctx, ownerUID, domain.TemplatePersonaAnalyzeText, personaID)
+	if s.MaxActiveAccountScrapes > 0 {
+		active, err := s.Repo.CountActiveByOwnerAndTemplate(ctx, ownerUID, domain.TemplatePersonaAnalyzeAccount)
+		if err != nil {
+			return nil, err
+		}
+		if active >= int64(s.MaxActiveAccountScrapes) {
+			return nil, domain.ErrTooManyActive
+		}
+	}
 	body, _ := json.Marshal(PersonaAnalyzeAccountPayload{Username: username, Lang: lang})
 	now := domain.NowNano()
 	j := &domain.Job{

apps/backend/internal/module/job/usecase/service_test.go

@@ -226,6 +226,39 @@ func TestJB_15_SchedulePersonaAnalyzeAccount(t *testing.T) {
 	require.Equal(t, domain.StatusRunning, claimed.Status)
 }
 
+func TestJB_15b_SchedulePersonaAnalyzeAccount_ThrottlesConcurrentScrapes(t *testing.T) {
+	svc, _ := newJobSvc()
+	uid := int64(1_000_117)
+	// first persona's scrape stays queued (not claimed yet)
+	first, err := svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_one", "acct_one", "zh-TW")
+	require.NoError(t, err)
+	require.Equal(t, domain.StatusQueued, first.Status)
+
+	// a different persona for the same owner must be rejected while one is in flight
+	_, err = svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_two", "acct_two", "zh-TW")
+	require.ErrorIs(t, err, domain.ErrTooManyActive)
+
+	// re-scheduling the SAME persona still works (cancels its own pending job first)
+	replaced, err := svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_one", "acct_one_v2", "zh-TW")
+	require.NoError(t, err)
+	require.NotEqual(t, first.ID, replaced.ID)
+
+	// once the in-flight job finishes, a different persona may schedule again
+	claimed, err := svc.ClaimNext(context.Background(), "w1")
+	require.NoError(t, err)
+	require.Equal(t, replaced.ID, claimed.ID)
+	_, err = svc.FailJob(context.Background(), claimed.ID, "boom")
+	require.NoError(t, err)
+
+	_, err = svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_two", "acct_two", "zh-TW")
+	require.NoError(t, err)
+
+	// a different owner is never affected by another owner's in-flight scrape
+	otherUID := int64(1_000_118)
+	_, err = svc.SchedulePersonaAnalyzeAccount(context.Background(), otherUID, "pe_other", "acct_other", "zh-TW")
+	require.NoError(t, err)
+}
+
 func TestJB_16_SchedulePersonaAnalyzeText(t *testing.T) {
 	svc, _ := newJobSvc()
 	uid := int64(1_000_116)

apps/backend/internal/response/response.go

@@ -153,6 +153,8 @@ func mapError(err error) (int, Envelope) {
 		return http.StatusBadRequest, Envelope{Code: 400033, Message: "session refresh failed"}
 	case errors.Is(err, jobDomain.ErrIllegalStatus):
 		return http.StatusConflict, Envelope{Code: 409010, Message: "illegal job status transition"}
+	case errors.Is(err, jobDomain.ErrTooManyActive):
+		return http.StatusTooManyRequests, Envelope{Code: 429010, Message: "已有一個人設在爬取公開貼文，請等該任務完成後再試"}
 	case errors.Is(err, usageDomain.ErrNoKey):
 		return http.StatusBadRequest, Envelope{Code: 400040, Message: "no api key configured (platform or byok)"}
 	case errors.Is(err, usageDomain.ErrQuotaExceeded):

deploy/prod/README.md

@@ -135,9 +135,35 @@ ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \
   journalctl -u 'harbor-gateway@*' -f
 ```
 
+### Background job health
+
+`harbor-job-health.timer` runs every 5 minutes and writes a node-exporter
+textfile-collector metric (`haixun_job_failures_recent`) counting jobs that
+failed in the last 15 minutes, overall and specifically for
+`persona_analyze_account` (the Threads profile scrape). Prometheus rules
+`PersonaScrapeJobsFailing` / `BackgroundJobsFailing` /
+`JobHealthMetricStale` in `monitoring/prometheus/rules/alerts.yml` alert on
+this so a stuck worker (e.g. a missing Playwright browser) surfaces on its
+own instead of only being noticed when a member reports it.
+
 ## Backups
 
 `harbor-backup.timer` creates daily Mongo and MinIO backups under
 `/var/backups/harbor` and retains 14 days. This protects against accidental
-deletion but not loss of the machine or disk. Add an off-host copy before using
+deletion but not loss of the machine or disk.
-the service for irreplaceable production data.
+
+`harbor-offsite-backup.timer` runs `backup/offsite-sync.sh` an hour later and
+pushes those local backups to an off-host [restic](https://restic.net/)
+repository (client-side encrypted before upload). It is installed and enabled
+by default but is a no-op until you set `RESTIC_REPOSITORY` and
+`RESTIC_PASSWORD` (plus provider credentials, e.g. `AWS_ACCESS_KEY_ID` /
+`AWS_SECRET_ACCESS_KEY` for an S3-compatible bucket) in
+`/etc/harbor/harbor.env` — see `harbor.env.example` for the format. Once
+configured, verify with:
+
+```bash
+ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \
+  sudo systemctl start harbor-offsite-backup.service
+ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \
+  sudo journalctl -u harbor-offsite-backup.service -n 50
+```

deploy/prod/backup/backup.sh

@@ -4,7 +4,7 @@ set -eu
 umask 077
 
 SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
-BACKUP_DIR=${BACKUP_DIR:-/var/backups/haixun}
+BACKUP_DIR=${BACKUP_DIR:-/var/backups/harbor}
 BACKUP_RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-14}
 BACKUP_STATUS_DIR=${BACKUP_STATUS_DIR:-"$SCRIPT_DIR/status"}
 

deploy/prod/backup/offsite-sync.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+set -eu
+
+# Pushes backup.sh's local backups to an off-host restic repository so
+# production member data survives loss of the machine or disk itself, not
+# just accidental deletion (see README "Backups"). restic encrypts
+# everything client-side before upload, so RESTIC_PASSWORD protects the
+# data even if the remote bucket credentials leak.
+#
+# No-op until RESTIC_REPOSITORY/RESTIC_PASSWORD are configured in
+# /etc/harbor/harbor.env — safe to enable the timer before picking a
+# provider.
+
+BACKUP_DIR=${BACKUP_DIR:-/var/backups/harbor}
+BACKUP_STATUS_DIR=${BACKUP_STATUS_DIR:-/opt/harbor/deploy/backup/status}
+OFFSITE_BACKUP_RETENTION_DAYS=${OFFSITE_BACKUP_RETENTION_DAYS:-30}
+
+if [ -z "${RESTIC_REPOSITORY:-}" ] || { [ -z "${RESTIC_PASSWORD:-}" ] && [ -z "${RESTIC_PASSWORD_FILE:-}" ]; }; then
+  printf '%s\n' "offsite-sync: RESTIC_REPOSITORY/RESTIC_PASSWORD not configured, skipping" >&2
+  exit 0
+fi
+
+if ! command -v restic >/dev/null 2>&1; then
+  printf '%s\n' "offsite-sync: restic not installed" >&2
+  exit 1
+fi
+
+if [ ! -d "$BACKUP_DIR" ] || [ -z "$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.*.tmp' -print -quit)" ]; then
+  printf '%s\n' "offsite-sync: no local backup found under $BACKUP_DIR yet, skipping" >&2
+  exit 0
+fi
+
+host_tag=$(hostname)
+
+restic snapshots >/dev/null 2>&1 || restic init
+
+restic backup --tag harbor-backup --host "$host_tag" "$BACKUP_DIR"
+restic forget --tag harbor-backup --host "$host_tag" --keep-within "${OFFSITE_BACKUP_RETENTION_DAYS}d" --prune
+
+mkdir -p "$BACKUP_STATUS_DIR"
+marker_tmp="$BACKUP_STATUS_DIR/.haixun_offsite_backup.prom.tmp"
+{
+  printf '# HELP haixun_offsite_backup_last_success_timestamp_seconds Unix timestamp of the last successful offsite backup sync.\n'
+  printf '# TYPE haixun_offsite_backup_last_success_timestamp_seconds gauge\n'
+  printf 'haixun_offsite_backup_last_success_timestamp_seconds %s\n' "$(date -u +%s)"
+} > "$marker_tmp"
+chmod 0644 "$marker_tmp"
+mv "$marker_tmp" "$BACKUP_STATUS_DIR/haixun_offsite_backup.prom"
+
+printf '%s\n' "offsite-sync: pushed $BACKUP_DIR to $RESTIC_REPOSITORY"

deploy/prod/harbor.env.example

@@ -26,3 +26,13 @@ MAIL_SMTP_HOST=
 MAIL_SMTP_PORT=587
 MAIL_SMTP_USER=
 MAIL_SMTP_PASSWORD=
+
+# Offsite backup sync (backup/offsite-sync.sh, runs from harbor-offsite-backup.timer
+# daily after the local backup). Leave RESTIC_REPOSITORY empty to keep the sync a
+# no-op. RESTIC_REPOSITORY/RESTIC_PASSWORD follow restic's own format, e.g. an
+# S3-compatible bucket: RESTIC_REPOSITORY=s3:https://s3.us-west-000.backblazeb2.com/my-bucket/harbor
+# plus AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY below for that provider.
+RESTIC_REPOSITORY=
+RESTIC_PASSWORD=
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=

deploy/prod/monitoring/job-health-check.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+set -eu
+
+# Writes a node-exporter textfile-collector metric so a run of failing
+# background jobs (e.g. the Threads persona scrape) shows up in Prometheus
+# instead of only being noticed when a member reports it. Reuses the same
+# textfile-collector directory as backup.sh's success marker.
+
+if [ "$(id -u)" -ne 0 ]; then
+  printf '%s\n' "job-health-check must run as root" >&2
+  exit 1
+fi
+
+STATUS_DIR=${BACKUP_STATUS_DIR:-/opt/harbor/deploy/backup/status}
+WINDOW_MINUTES=${JOB_HEALTH_WINDOW_MINUTES:-15}
+COMPOSE_FILE=${COMPOSE_FILE:-/opt/harbor/deploy/compose/docker-compose.yml}
+
+set -a
+# shellcheck disable=SC1091
+. /etc/harbor/harbor.env
+set +a
+: "${MONGO_URI:?MONGO_URI is required}"
+
+now_s=$(date -u +%s)
+since_ns=$(( (now_s - WINDOW_MINUTES * 60) * 1000000000 ))
+
+compose() {
+  docker compose --env-file /etc/harbor/harbor.env -f "$COMPOSE_FILE" "$@"
+}
+
+query_count() {
+  # $1 = extra mongosh filter fragment (may be empty)
+  # mongosh --eval resolves the returned promise before printing; top-level
+  # `await` is a syntax error there, so call the collection method directly.
+  eval_js="db.jobs.countDocuments({status:'failed', completed_at:{\$gt: ${since_ns}}${1}})"
+  compose exec -T mongo mongosh "$MONGO_URI" --quiet --eval "$eval_js" | tr -d '[:space:]'
+}
+
+scrape_failures=$(query_count ", template_type:'persona_analyze_account'")
+total_failures=$(query_count "")
+
+case "$scrape_failures" in ''|*[!0-9]*) scrape_failures=0 ;; esac
+case "$total_failures" in ''|*[!0-9]*) total_failures=0 ;; esac
+
+mkdir -p "$STATUS_DIR"
+tmp="$STATUS_DIR/.haixun_job_failures.prom.tmp"
+{
+  printf '# HELP haixun_job_failures_recent Jobs that failed in the last %s minutes.\n' "$WINDOW_MINUTES"
+  printf '# TYPE haixun_job_failures_recent gauge\n'
+  printf 'haixun_job_failures_recent{template="persona_analyze_account"} %s\n' "$scrape_failures"
+  printf 'haixun_job_failures_recent{template="_all_"} %s\n' "$total_failures"
+} > "$tmp"
+chmod 0644 "$tmp"
+mv "$tmp" "$STATUS_DIR/haixun_job_failures.prom"

deploy/prod/monitoring/prometheus/rules/alerts.yml

@@ -90,3 +90,30 @@ groups:
         annotations:
           summary: "Production backup is missing or stale"
           description: "No successful MongoDB and MinIO backup has been recorded in the last 26 hours."
+
+      - alert: JobHealthMetricStale
+        expr: absent(haixun_job_failures_recent{template="_all_"})
+        for: 30m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Job health metric missing"
+          description: "harbor-job-health.timer has not reported job failure counts; the metric may be stale or the timer stopped running."
+
+      - alert: PersonaScrapeJobsFailing
+        expr: haixun_job_failures_recent{template="persona_analyze_account"} >= 2
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Threads persona scrape jobs are repeatedly failing"
+          description: "{{ $value }} persona_analyze_account job(s) failed in the last 15 minutes. Check harbor-worker@*.service logs (e.g. missing Playwright browser, Threads layout change)."
+
+      - alert: BackgroundJobsFailing
+        expr: haixun_job_failures_recent{template="_all_"} >= 5
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Background jobs are repeatedly failing"
+          description: "{{ $value }} job(s) across all templates failed in the last 15 minutes. Check harbor-worker@*.service logs."

deploy/prod/remote/bootstrap.sh

@@ -14,7 +14,7 @@ fi
 
 export DEBIAN_FRONTEND=noninteractive
 apt-get update
-apt-get install -y ca-certificates certbot curl jq nginx openssl python3-certbot-dns-cloudflare rsync tar gzip ufw fail2ban docker.io docker-compose-v2 nodejs npm
+apt-get install -y ca-certificates certbot curl jq nginx openssl python3-certbot-dns-cloudflare restic rsync tar gzip ufw fail2ban docker.io docker-compose-v2 nodejs npm
 systemctl enable --now docker nginx fail2ban
 
 if ! id harbor >/dev/null 2>&1; then
@@ -42,13 +42,17 @@ if [[ ! -d /var/lib/harbor/.cache/ms-playwright ]] \
 fi
 rsync -a --delete "$SOURCE_DIR/" /opt/harbor/deploy/
 chown -R root:root /opt/harbor/deploy
-chmod +x /opt/harbor/deploy/remote/*.sh /opt/harbor/deploy/backup/backup.sh
+chmod +x /opt/harbor/deploy/remote/*.sh /opt/harbor/deploy/backup/backup.sh /opt/harbor/deploy/backup/offsite-sync.sh /opt/harbor/deploy/monitoring/job-health-check.sh
 
 install -m 0644 /opt/harbor/deploy/config/gateway.yaml /etc/harbor/gateway.yaml
 install -m 0644 /opt/harbor/deploy/systemd/harbor-gateway@.service /etc/systemd/system/harbor-gateway@.service
 install -m 0644 /opt/harbor/deploy/systemd/harbor-worker@.service /etc/systemd/system/harbor-worker@.service
 install -m 0644 /opt/harbor/deploy/systemd/harbor-backup.service /etc/systemd/system/harbor-backup.service
 install -m 0644 /opt/harbor/deploy/systemd/harbor-backup.timer /etc/systemd/system/harbor-backup.timer
+install -m 0644 /opt/harbor/deploy/systemd/harbor-job-health.service /etc/systemd/system/harbor-job-health.service
+install -m 0644 /opt/harbor/deploy/systemd/harbor-job-health.timer /etc/systemd/system/harbor-job-health.timer
+install -m 0644 /opt/harbor/deploy/systemd/harbor-offsite-backup.service /etc/systemd/system/harbor-offsite-backup.service
+install -m 0644 /opt/harbor/deploy/systemd/harbor-offsite-backup.timer /etc/systemd/system/harbor-offsite-backup.timer
 
 if [[ ! -f /etc/harbor/harbor.env ]]; then
   mongo_password=$(openssl rand -hex 24)
@@ -173,6 +177,10 @@ ufw --force enable
 nginx -t
 systemctl reload nginx
 systemctl enable --now harbor-backup.timer
+systemctl enable --now harbor-job-health.timer
+# offsite-sync.sh no-ops until RESTIC_REPOSITORY/RESTIC_PASSWORD are set in
+# harbor.env, so it's safe to always enable the timer.
+systemctl enable --now harbor-offsite-backup.timer
 
 printf '%s\n' "bootstrap complete"
 printf '%s\n' "runtime secrets: /etc/harbor/harbor.env"

deploy/prod/systemd/harbor-job-health.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Harbor Desk background job health metrics
+After=docker.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+EnvironmentFile=/etc/harbor/harbor.env
+Environment=BACKUP_STATUS_DIR=/opt/harbor/deploy/backup/status
+ExecStart=/opt/harbor/deploy/monitoring/job-health-check.sh
+User=root
+Group=root
+Nice=10

deploy/prod/systemd/harbor-job-health.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Check background job health every 5 minutes
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=5min
+RandomizedDelaySec=30
+Persistent=true
+
+[Install]
+WantedBy=timers.target

deploy/prod/systemd/harbor-offsite-backup.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Harbor Desk offsite backup sync
+After=docker.service harbor-backup.service
+Requires=docker.service
+
+[Service]
+Type=oneshot
+EnvironmentFile=/etc/harbor/harbor.env
+Environment=BACKUP_DIR=/var/backups/harbor
+Environment=BACKUP_STATUS_DIR=/opt/harbor/deploy/backup/status
+ExecStart=/opt/harbor/deploy/backup/offsite-sync.sh
+User=root
+Group=root
+Nice=10
+IOSchedulingClass=best-effort
+IOSchedulingPriority=7

deploy/prod/systemd/harbor-offsite-backup.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=Run Harbor Desk offsite backup sync daily (after the local backup)
+
+[Timer]
+OnCalendar=*-*-* 04:15:00
+RandomizedDelaySec=900
+Persistent=true
+
+[Install]
+WantedBy=timers.target

deploy/prod/systemd/harbor-worker@.service

@@ -20,6 +20,9 @@ PrivateTmp=true
 ProtectSystem=strict
 ProtectHome=true
 ReadWritePaths=/var/lib/harbor
+# The host has 1 vCPU; a persona-scrape job briefly saturates Chromium. Cap the
+# worker so nginx/gateway keep some headroom instead of queueing behind it.
+CPUQuota=70%
 StandardOutput=journal
 StandardError=journal
 SyslogIdentifier=harbor-worker-%i