diff --git a/apps/backend/internal/module/job/domain/job.go b/apps/backend/internal/module/job/domain/job.go index 8198d2c..ea622e6 100644 --- a/apps/backend/internal/module/job/domain/job.go +++ b/apps/backend/internal/module/job/domain/job.go @@ -34,6 +34,9 @@ var ( ErrNotFound = errors.New("job not found") ErrForbidden = errors.New("job access denied") ErrIllegalStatus = errors.New("illegal job status transition") + // ErrTooManyActive — owner already has a CPU-heavy job (e.g. persona account + // scrape) in flight; caller should retry once it finishes. + ErrTooManyActive = errors.New("too many active jobs") ) type Job struct { diff --git a/apps/backend/internal/module/job/domain/repository.go b/apps/backend/internal/module/job/domain/repository.go index 48768dc..950d208 100644 --- a/apps/backend/internal/module/job/domain/repository.go +++ b/apps/backend/internal/module/job/domain/repository.go @@ -16,6 +16,10 @@ type Repository interface { ClaimNext(ctx context.Context, workerID string) (*Job, error) // CancelPendingByRef cancels pending/queued jobs of template+ref (e.g. reschedule renew). CancelPendingByRef(ctx context.Context, ownerUID int64, template, refID string) error + // CountActiveByOwnerAndTemplate counts an owner's non-terminal (pending/ + // queued/running) jobs of the given template — used to cap concurrent + // CPU-heavy jobs (e.g. Threads profile scrape) per member. + CountActiveByOwnerAndTemplate(ctx context.Context, ownerUID int64, template string) (int64, error) // Delete hard-removes a job by id. Delete(ctx context.Context, id string) error // DeleteTerminalBefore removes terminal jobs whose completion time is before beforeNs. diff --git a/apps/backend/internal/module/job/repository/memory.go b/apps/backend/internal/module/job/repository/memory.go index d29c841..16c6099 100644 --- a/apps/backend/internal/module/job/repository/memory.go +++ b/apps/backend/internal/module/job/repository/memory.go @@ -189,6 +189,21 @@ func (s *MemoryStore) CancelPendingByRef(_ context.Context, ownerUID int64, temp return nil } +func (s *MemoryStore) CountActiveByOwnerAndTemplate(_ context.Context, ownerUID int64, template string) (int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + var n int64 + for _, j := range s.byID { + if j.OwnerUID != ownerUID || j.TemplateType != template { + continue + } + if j.Status == domain.StatusPending || j.Status == domain.StatusQueued || j.Status == domain.StatusRunning { + n++ + } + } + return n, nil +} + func (s *MemoryStore) Delete(_ context.Context, id string) error { s.mu.Lock() defer s.mu.Unlock() diff --git a/apps/backend/internal/module/job/repository/mongo.go b/apps/backend/internal/module/job/repository/mongo.go index 93d2628..2d2dde9 100644 --- a/apps/backend/internal/module/job/repository/mongo.go +++ b/apps/backend/internal/module/job/repository/mongo.go @@ -216,6 +216,15 @@ func (s *MonStore) CancelPendingByRef(ctx context.Context, ownerUID int64, templ return nil } +func (s *MonStore) CountActiveByOwnerAndTemplate(ctx context.Context, ownerUID int64, template string) (int64, error) { + filter := bson.M{ + "owner_uid": ownerUID, + "template_type": template, + "status": bson.M{"$in": []string{domain.StatusPending, domain.StatusQueued, domain.StatusRunning}}, + } + return s.jobs.CountDocuments(ctx, filter) +} + func (s *MonStore) Delete(ctx context.Context, id string) error { res, err := s.jobs.DeleteOne(ctx, bson.M{"_id": id}) if err != nil { diff --git a/apps/backend/internal/module/job/usecase/service.go b/apps/backend/internal/module/job/usecase/service.go index d35d415..585ed7c 100644 --- a/apps/backend/internal/module/job/usecase/service.go +++ b/apps/backend/internal/module/job/usecase/service.go @@ -24,17 +24,28 @@ type Service struct { Notifier Notifier LeaseDuration time.Duration HeartbeatInterval time.Duration + // MaxActiveAccountScrapes caps how many persona_analyze_account jobs + // (real headless-Chromium scrape; CPU-heavy on the shared prod host) a + // single owner may have pending/queued/running at once. <=0 disables + // the check. + MaxActiveAccountScrapes int mu sync.Mutex workerID string heartbeats map[string]context.CancelFunc } +// DefaultMaxActiveAccountScrapes — one in-flight browser scrape per member; +// keeps the 1-vCPU prod host from being flooded by many personas being +// analyzed back-to-back (see deploy/prod/README.md). +const DefaultMaxActiveAccountScrapes = 1 + func New(repo domain.Repository) *Service { return &Service{ Repo: repo, LeaseDuration: domain.DefaultLeaseDuration, - HeartbeatInterval: domain.DefaultHeartbeat, - heartbeats: make(map[string]context.CancelFunc), + HeartbeatInterval: domain.DefaultHeartbeat, + MaxActiveAccountScrapes: DefaultMaxActiveAccountScrapes, + heartbeats: make(map[string]context.CancelFunc), } } @@ -325,6 +336,15 @@ func (s *Service) SchedulePersonaAnalyzeAccount(ctx context.Context, ownerUID in } _ = s.Repo.CancelPendingByRef(ctx, ownerUID, domain.TemplatePersonaAnalyzeAccount, personaID) _ = s.Repo.CancelPendingByRef(ctx, ownerUID, domain.TemplatePersonaAnalyzeText, personaID) + if s.MaxActiveAccountScrapes > 0 { + active, err := s.Repo.CountActiveByOwnerAndTemplate(ctx, ownerUID, domain.TemplatePersonaAnalyzeAccount) + if err != nil { + return nil, err + } + if active >= int64(s.MaxActiveAccountScrapes) { + return nil, domain.ErrTooManyActive + } + } body, _ := json.Marshal(PersonaAnalyzeAccountPayload{Username: username, Lang: lang}) now := domain.NowNano() j := &domain.Job{ diff --git a/apps/backend/internal/module/job/usecase/service_test.go b/apps/backend/internal/module/job/usecase/service_test.go index 8e72cf1..2561806 100644 --- a/apps/backend/internal/module/job/usecase/service_test.go +++ b/apps/backend/internal/module/job/usecase/service_test.go @@ -226,6 +226,39 @@ func TestJB_15_SchedulePersonaAnalyzeAccount(t *testing.T) { require.Equal(t, domain.StatusRunning, claimed.Status) } +func TestJB_15b_SchedulePersonaAnalyzeAccount_ThrottlesConcurrentScrapes(t *testing.T) { + svc, _ := newJobSvc() + uid := int64(1_000_117) + // first persona's scrape stays queued (not claimed yet) + first, err := svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_one", "acct_one", "zh-TW") + require.NoError(t, err) + require.Equal(t, domain.StatusQueued, first.Status) + + // a different persona for the same owner must be rejected while one is in flight + _, err = svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_two", "acct_two", "zh-TW") + require.ErrorIs(t, err, domain.ErrTooManyActive) + + // re-scheduling the SAME persona still works (cancels its own pending job first) + replaced, err := svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_one", "acct_one_v2", "zh-TW") + require.NoError(t, err) + require.NotEqual(t, first.ID, replaced.ID) + + // once the in-flight job finishes, a different persona may schedule again + claimed, err := svc.ClaimNext(context.Background(), "w1") + require.NoError(t, err) + require.Equal(t, replaced.ID, claimed.ID) + _, err = svc.FailJob(context.Background(), claimed.ID, "boom") + require.NoError(t, err) + + _, err = svc.SchedulePersonaAnalyzeAccount(context.Background(), uid, "pe_two", "acct_two", "zh-TW") + require.NoError(t, err) + + // a different owner is never affected by another owner's in-flight scrape + otherUID := int64(1_000_118) + _, err = svc.SchedulePersonaAnalyzeAccount(context.Background(), otherUID, "pe_other", "acct_other", "zh-TW") + require.NoError(t, err) +} + func TestJB_16_SchedulePersonaAnalyzeText(t *testing.T) { svc, _ := newJobSvc() uid := int64(1_000_116) diff --git a/apps/backend/internal/response/response.go b/apps/backend/internal/response/response.go index b6b73bb..26ff844 100644 --- a/apps/backend/internal/response/response.go +++ b/apps/backend/internal/response/response.go @@ -153,6 +153,8 @@ func mapError(err error) (int, Envelope) { return http.StatusBadRequest, Envelope{Code: 400033, Message: "session refresh failed"} case errors.Is(err, jobDomain.ErrIllegalStatus): return http.StatusConflict, Envelope{Code: 409010, Message: "illegal job status transition"} + case errors.Is(err, jobDomain.ErrTooManyActive): + return http.StatusTooManyRequests, Envelope{Code: 429010, Message: "已有一個人設在爬取公開貼文,請等該任務完成後再試"} case errors.Is(err, usageDomain.ErrNoKey): return http.StatusBadRequest, Envelope{Code: 400040, Message: "no api key configured (platform or byok)"} case errors.Is(err, usageDomain.ErrQuotaExceeded): diff --git a/deploy/prod/README.md b/deploy/prod/README.md index cf36f4f..604b4b7 100644 --- a/deploy/prod/README.md +++ b/deploy/prod/README.md @@ -135,9 +135,35 @@ ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \ journalctl -u 'harbor-gateway@*' -f ``` +### Background job health + +`harbor-job-health.timer` runs every 5 minutes and writes a node-exporter +textfile-collector metric (`haixun_job_failures_recent`) counting jobs that +failed in the last 15 minutes, overall and specifically for +`persona_analyze_account` (the Threads profile scrape). Prometheus rules +`PersonaScrapeJobsFailing` / `BackgroundJobsFailing` / +`JobHealthMetricStale` in `monitoring/prometheus/rules/alerts.yml` alert on +this so a stuck worker (e.g. a missing Playwright browser) surfaces on its +own instead of only being noticed when a member reports it. + ## Backups `harbor-backup.timer` creates daily Mongo and MinIO backups under `/var/backups/harbor` and retains 14 days. This protects against accidental -deletion but not loss of the machine or disk. Add an off-host copy before using -the service for irreplaceable production data. +deletion but not loss of the machine or disk. + +`harbor-offsite-backup.timer` runs `backup/offsite-sync.sh` an hour later and +pushes those local backups to an off-host [restic](https://restic.net/) +repository (client-side encrypted before upload). It is installed and enabled +by default but is a no-op until you set `RESTIC_REPOSITORY` and +`RESTIC_PASSWORD` (plus provider credentials, e.g. `AWS_ACCESS_KEY_ID` / +`AWS_SECRET_ACCESS_KEY` for an S3-compatible bucket) in +`/etc/harbor/harbor.env` — see `harbor.env.example` for the format. Once +configured, verify with: + +```bash +ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \ + sudo systemctl start harbor-offsite-backup.service +ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \ + sudo journalctl -u harbor-offsite-backup.service -n 50 +``` diff --git a/deploy/prod/backup/backup.sh b/deploy/prod/backup/backup.sh index 7b34cb5..e7c6507 100755 --- a/deploy/prod/backup/backup.sh +++ b/deploy/prod/backup/backup.sh @@ -4,7 +4,7 @@ set -eu umask 077 SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -BACKUP_DIR=${BACKUP_DIR:-/var/backups/haixun} +BACKUP_DIR=${BACKUP_DIR:-/var/backups/harbor} BACKUP_RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-14} BACKUP_STATUS_DIR=${BACKUP_STATUS_DIR:-"$SCRIPT_DIR/status"} diff --git a/deploy/prod/backup/offsite-sync.sh b/deploy/prod/backup/offsite-sync.sh new file mode 100755 index 0000000..3287d19 --- /dev/null +++ b/deploy/prod/backup/offsite-sync.sh @@ -0,0 +1,50 @@ +#!/bin/sh +set -eu + +# Pushes backup.sh's local backups to an off-host restic repository so +# production member data survives loss of the machine or disk itself, not +# just accidental deletion (see README "Backups"). restic encrypts +# everything client-side before upload, so RESTIC_PASSWORD protects the +# data even if the remote bucket credentials leak. +# +# No-op until RESTIC_REPOSITORY/RESTIC_PASSWORD are configured in +# /etc/harbor/harbor.env — safe to enable the timer before picking a +# provider. + +BACKUP_DIR=${BACKUP_DIR:-/var/backups/harbor} +BACKUP_STATUS_DIR=${BACKUP_STATUS_DIR:-/opt/harbor/deploy/backup/status} +OFFSITE_BACKUP_RETENTION_DAYS=${OFFSITE_BACKUP_RETENTION_DAYS:-30} + +if [ -z "${RESTIC_REPOSITORY:-}" ] || { [ -z "${RESTIC_PASSWORD:-}" ] && [ -z "${RESTIC_PASSWORD_FILE:-}" ]; }; then + printf '%s\n' "offsite-sync: RESTIC_REPOSITORY/RESTIC_PASSWORD not configured, skipping" >&2 + exit 0 +fi + +if ! command -v restic >/dev/null 2>&1; then + printf '%s\n' "offsite-sync: restic not installed" >&2 + exit 1 +fi + +if [ ! -d "$BACKUP_DIR" ] || [ -z "$(find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.*.tmp' -print -quit)" ]; then + printf '%s\n' "offsite-sync: no local backup found under $BACKUP_DIR yet, skipping" >&2 + exit 0 +fi + +host_tag=$(hostname) + +restic snapshots >/dev/null 2>&1 || restic init + +restic backup --tag harbor-backup --host "$host_tag" "$BACKUP_DIR" +restic forget --tag harbor-backup --host "$host_tag" --keep-within "${OFFSITE_BACKUP_RETENTION_DAYS}d" --prune + +mkdir -p "$BACKUP_STATUS_DIR" +marker_tmp="$BACKUP_STATUS_DIR/.haixun_offsite_backup.prom.tmp" +{ + printf '# HELP haixun_offsite_backup_last_success_timestamp_seconds Unix timestamp of the last successful offsite backup sync.\n' + printf '# TYPE haixun_offsite_backup_last_success_timestamp_seconds gauge\n' + printf 'haixun_offsite_backup_last_success_timestamp_seconds %s\n' "$(date -u +%s)" +} > "$marker_tmp" +chmod 0644 "$marker_tmp" +mv "$marker_tmp" "$BACKUP_STATUS_DIR/haixun_offsite_backup.prom" + +printf '%s\n' "offsite-sync: pushed $BACKUP_DIR to $RESTIC_REPOSITORY" diff --git a/deploy/prod/harbor.env.example b/deploy/prod/harbor.env.example index 9ed10fc..be75429 100644 --- a/deploy/prod/harbor.env.example +++ b/deploy/prod/harbor.env.example @@ -26,3 +26,13 @@ MAIL_SMTP_HOST= MAIL_SMTP_PORT=587 MAIL_SMTP_USER= MAIL_SMTP_PASSWORD= + +# Offsite backup sync (backup/offsite-sync.sh, runs from harbor-offsite-backup.timer +# daily after the local backup). Leave RESTIC_REPOSITORY empty to keep the sync a +# no-op. RESTIC_REPOSITORY/RESTIC_PASSWORD follow restic's own format, e.g. an +# S3-compatible bucket: RESTIC_REPOSITORY=s3:https://s3.us-west-000.backblazeb2.com/my-bucket/harbor +# plus AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY below for that provider. +RESTIC_REPOSITORY= +RESTIC_PASSWORD= +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= diff --git a/deploy/prod/monitoring/job-health-check.sh b/deploy/prod/monitoring/job-health-check.sh new file mode 100755 index 0000000..e922b7e --- /dev/null +++ b/deploy/prod/monitoring/job-health-check.sh @@ -0,0 +1,54 @@ +#!/bin/sh +set -eu + +# Writes a node-exporter textfile-collector metric so a run of failing +# background jobs (e.g. the Threads persona scrape) shows up in Prometheus +# instead of only being noticed when a member reports it. Reuses the same +# textfile-collector directory as backup.sh's success marker. + +if [ "$(id -u)" -ne 0 ]; then + printf '%s\n' "job-health-check must run as root" >&2 + exit 1 +fi + +STATUS_DIR=${BACKUP_STATUS_DIR:-/opt/harbor/deploy/backup/status} +WINDOW_MINUTES=${JOB_HEALTH_WINDOW_MINUTES:-15} +COMPOSE_FILE=${COMPOSE_FILE:-/opt/harbor/deploy/compose/docker-compose.yml} + +set -a +# shellcheck disable=SC1091 +. /etc/harbor/harbor.env +set +a +: "${MONGO_URI:?MONGO_URI is required}" + +now_s=$(date -u +%s) +since_ns=$(( (now_s - WINDOW_MINUTES * 60) * 1000000000 )) + +compose() { + docker compose --env-file /etc/harbor/harbor.env -f "$COMPOSE_FILE" "$@" +} + +query_count() { + # $1 = extra mongosh filter fragment (may be empty) + # mongosh --eval resolves the returned promise before printing; top-level + # `await` is a syntax error there, so call the collection method directly. + eval_js="db.jobs.countDocuments({status:'failed', completed_at:{\$gt: ${since_ns}}${1}})" + compose exec -T mongo mongosh "$MONGO_URI" --quiet --eval "$eval_js" | tr -d '[:space:]' +} + +scrape_failures=$(query_count ", template_type:'persona_analyze_account'") +total_failures=$(query_count "") + +case "$scrape_failures" in ''|*[!0-9]*) scrape_failures=0 ;; esac +case "$total_failures" in ''|*[!0-9]*) total_failures=0 ;; esac + +mkdir -p "$STATUS_DIR" +tmp="$STATUS_DIR/.haixun_job_failures.prom.tmp" +{ + printf '# HELP haixun_job_failures_recent Jobs that failed in the last %s minutes.\n' "$WINDOW_MINUTES" + printf '# TYPE haixun_job_failures_recent gauge\n' + printf 'haixun_job_failures_recent{template="persona_analyze_account"} %s\n' "$scrape_failures" + printf 'haixun_job_failures_recent{template="_all_"} %s\n' "$total_failures" +} > "$tmp" +chmod 0644 "$tmp" +mv "$tmp" "$STATUS_DIR/haixun_job_failures.prom" diff --git a/deploy/prod/monitoring/prometheus/rules/alerts.yml b/deploy/prod/monitoring/prometheus/rules/alerts.yml index 008d97f..ab55fc7 100644 --- a/deploy/prod/monitoring/prometheus/rules/alerts.yml +++ b/deploy/prod/monitoring/prometheus/rules/alerts.yml @@ -90,3 +90,30 @@ groups: annotations: summary: "Production backup is missing or stale" description: "No successful MongoDB and MinIO backup has been recorded in the last 26 hours." + + - alert: JobHealthMetricStale + expr: absent(haixun_job_failures_recent{template="_all_"}) + for: 30m + labels: + severity: warning + annotations: + summary: "Job health metric missing" + description: "harbor-job-health.timer has not reported job failure counts; the metric may be stale or the timer stopped running." + + - alert: PersonaScrapeJobsFailing + expr: haixun_job_failures_recent{template="persona_analyze_account"} >= 2 + for: 5m + labels: + severity: warning + annotations: + summary: "Threads persona scrape jobs are repeatedly failing" + description: "{{ $value }} persona_analyze_account job(s) failed in the last 15 minutes. Check harbor-worker@*.service logs (e.g. missing Playwright browser, Threads layout change)." + + - alert: BackgroundJobsFailing + expr: haixun_job_failures_recent{template="_all_"} >= 5 + for: 5m + labels: + severity: warning + annotations: + summary: "Background jobs are repeatedly failing" + description: "{{ $value }} job(s) across all templates failed in the last 15 minutes. Check harbor-worker@*.service logs." diff --git a/deploy/prod/remote/bootstrap.sh b/deploy/prod/remote/bootstrap.sh index 660e655..a0cd1e3 100755 --- a/deploy/prod/remote/bootstrap.sh +++ b/deploy/prod/remote/bootstrap.sh @@ -14,7 +14,7 @@ fi export DEBIAN_FRONTEND=noninteractive apt-get update -apt-get install -y ca-certificates certbot curl jq nginx openssl python3-certbot-dns-cloudflare rsync tar gzip ufw fail2ban docker.io docker-compose-v2 nodejs npm +apt-get install -y ca-certificates certbot curl jq nginx openssl python3-certbot-dns-cloudflare restic rsync tar gzip ufw fail2ban docker.io docker-compose-v2 nodejs npm systemctl enable --now docker nginx fail2ban if ! id harbor >/dev/null 2>&1; then @@ -42,13 +42,17 @@ if [[ ! -d /var/lib/harbor/.cache/ms-playwright ]] \ fi rsync -a --delete "$SOURCE_DIR/" /opt/harbor/deploy/ chown -R root:root /opt/harbor/deploy -chmod +x /opt/harbor/deploy/remote/*.sh /opt/harbor/deploy/backup/backup.sh +chmod +x /opt/harbor/deploy/remote/*.sh /opt/harbor/deploy/backup/backup.sh /opt/harbor/deploy/backup/offsite-sync.sh /opt/harbor/deploy/monitoring/job-health-check.sh install -m 0644 /opt/harbor/deploy/config/gateway.yaml /etc/harbor/gateway.yaml install -m 0644 /opt/harbor/deploy/systemd/harbor-gateway@.service /etc/systemd/system/harbor-gateway@.service install -m 0644 /opt/harbor/deploy/systemd/harbor-worker@.service /etc/systemd/system/harbor-worker@.service install -m 0644 /opt/harbor/deploy/systemd/harbor-backup.service /etc/systemd/system/harbor-backup.service install -m 0644 /opt/harbor/deploy/systemd/harbor-backup.timer /etc/systemd/system/harbor-backup.timer +install -m 0644 /opt/harbor/deploy/systemd/harbor-job-health.service /etc/systemd/system/harbor-job-health.service +install -m 0644 /opt/harbor/deploy/systemd/harbor-job-health.timer /etc/systemd/system/harbor-job-health.timer +install -m 0644 /opt/harbor/deploy/systemd/harbor-offsite-backup.service /etc/systemd/system/harbor-offsite-backup.service +install -m 0644 /opt/harbor/deploy/systemd/harbor-offsite-backup.timer /etc/systemd/system/harbor-offsite-backup.timer if [[ ! -f /etc/harbor/harbor.env ]]; then mongo_password=$(openssl rand -hex 24) @@ -173,6 +177,10 @@ ufw --force enable nginx -t systemctl reload nginx systemctl enable --now harbor-backup.timer +systemctl enable --now harbor-job-health.timer +# offsite-sync.sh no-ops until RESTIC_REPOSITORY/RESTIC_PASSWORD are set in +# harbor.env, so it's safe to always enable the timer. +systemctl enable --now harbor-offsite-backup.timer printf '%s\n' "bootstrap complete" printf '%s\n' "runtime secrets: /etc/harbor/harbor.env" diff --git a/deploy/prod/systemd/harbor-job-health.service b/deploy/prod/systemd/harbor-job-health.service new file mode 100644 index 0000000..b38ee7b --- /dev/null +++ b/deploy/prod/systemd/harbor-job-health.service @@ -0,0 +1,13 @@ +[Unit] +Description=Harbor Desk background job health metrics +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/harbor/harbor.env +Environment=BACKUP_STATUS_DIR=/opt/harbor/deploy/backup/status +ExecStart=/opt/harbor/deploy/monitoring/job-health-check.sh +User=root +Group=root +Nice=10 diff --git a/deploy/prod/systemd/harbor-job-health.timer b/deploy/prod/systemd/harbor-job-health.timer new file mode 100644 index 0000000..fa01799 --- /dev/null +++ b/deploy/prod/systemd/harbor-job-health.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Check background job health every 5 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=5min +RandomizedDelaySec=30 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/deploy/prod/systemd/harbor-offsite-backup.service b/deploy/prod/systemd/harbor-offsite-backup.service new file mode 100644 index 0000000..275d5fa --- /dev/null +++ b/deploy/prod/systemd/harbor-offsite-backup.service @@ -0,0 +1,16 @@ +[Unit] +Description=Harbor Desk offsite backup sync +After=docker.service harbor-backup.service +Requires=docker.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/harbor/harbor.env +Environment=BACKUP_DIR=/var/backups/harbor +Environment=BACKUP_STATUS_DIR=/opt/harbor/deploy/backup/status +ExecStart=/opt/harbor/deploy/backup/offsite-sync.sh +User=root +Group=root +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 diff --git a/deploy/prod/systemd/harbor-offsite-backup.timer b/deploy/prod/systemd/harbor-offsite-backup.timer new file mode 100644 index 0000000..86af4d6 --- /dev/null +++ b/deploy/prod/systemd/harbor-offsite-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run Harbor Desk offsite backup sync daily (after the local backup) + +[Timer] +OnCalendar=*-*-* 04:15:00 +RandomizedDelaySec=900 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/deploy/prod/systemd/harbor-worker@.service b/deploy/prod/systemd/harbor-worker@.service index 07a9726..1dfc273 100644 --- a/deploy/prod/systemd/harbor-worker@.service +++ b/deploy/prod/systemd/harbor-worker@.service @@ -20,6 +20,9 @@ PrivateTmp=true ProtectSystem=strict ProtectHome=true ReadWritePaths=/var/lib/harbor +# The host has 1 vCPU; a persona-scrape job briefly saturates Chromium. Cap the +# worker so nginx/gateway keep some headroom instead of queueing behind it. +CPUQuota=70% StandardOutput=journal StandardError=journal SyslogIdentifier=harbor-worker-%i