diff --git a/deploy/.env.example b/deploy/.env.example index 32404cc..a9facec 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -31,6 +31,14 @@ HAIXUN_MONGO_DB=haixun REDIS_PASSWORD=CHANGE_ME_REDIS HAIXUN_REDIS_ADDR=127.0.0.1:6379 HAIXUN_REDIS_PASSWORD=CHANGE_ME_REDIS +REDIS_NAMESPACE=haixun:dev:v1 +# openssl rand -hex 32;不可與 JWT/BYOK encryption key 共用 +AI_MODEL_CACHE_FINGERPRINT_SECRET=CHANGE_ME_AI_MODEL_CACHE_FINGERPRINT_SECRET + +# ---------- Member BYOK AES-256-GCM(openssl rand -base64 32)---------- +# 上線後需永久保留;gateway/worker 必須使用相同 key ID 與 key +MEMBER_SETTINGS_ENCRYPTION_KEY_ID=v1 +MEMBER_SETTINGS_ENCRYPTION_KEY=CHANGE_ME_MEMBER_SETTINGS_ENCRYPTION_KEY # ---------- MinIO(AccessKey 建議與 ROOT 相同)---------- MINIO_ROOT_USER=haixun @@ -49,6 +57,17 @@ AUTH_REFRESH_SECRET=CHANGE_ME_JWT_REFRESH HAIXUN_JWT_ACCESS_SECRET=CHANGE_ME_JWT_ACCESS HAIXUN_JWT_REFRESH_SECRET=CHANGE_ME_JWT_REFRESH +# ---------- Stripe subscriptions ---------- +# Disabled is startup-safe; billing endpoints return 503 until fully configured. +STRIPE_ENABLED=false +STRIPE_SECRET_KEY= +STRIPE_WEBHOOK_SECRET= +STRIPE_STARTER_PRICE_ID= +STRIPE_PRO_PRICE_ID= +STRIPE_SUCCESS_URL=https://threads-tool-dev.30cm.net/app/usage/checkout?result=success&checkout_id={CHECKOUT_ID}&plan={PLAN_ID} +STRIPE_CANCEL_URL=https://threads-tool-dev.30cm.net/app/usage/checkout?result=cancel&plan={PLAN_ID} +STRIPE_PORTAL_RETURN_URL=https://threads-tool-dev.30cm.net/app/usage/plans + # ---------- Mail(dev → Mailpit)---------- HAIXUN_SMTP_HOST=127.0.0.1 HAIXUN_SMTP_PORT=1025 diff --git a/deploy/README.md b/deploy/README.md index 92458ca..73f335d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -93,22 +93,21 @@ make -C deploy redeploy # = redeploy.sh --yes --kill-orphan(清資料) | `REDIS_PASSWORD` | monc 快取 | | `HAIXUN_STORAGE_S3_*` | MinIO 上傳與 `PublicBaseURL` | | `AUTH_*` / `HAIXUN_JWT_*` | JWT | +| `REDIS_NAMESPACE` | Redis 環境隔離前綴 | +| `AI_MODEL_CACHE_FINGERPRINT_SECRET` | AI model cache credential HMAC | +| `MEMBER_SETTINGS_ENCRYPTION_KEY_ID` / `MEMBER_SETTINGS_ENCRYPTION_KEY` | BYOK AES-256-GCM | +| `STRIPE_ENABLED` / `STRIPE_SECRET_KEY` / `STRIPE_WEBHOOK_SECRET` | Stripe subscription billing; disabled is startup-safe | +| `STRIPE_STARTER_PRICE_ID` / `STRIPE_PRO_PRICE_ID` | Trusted recurring Price IDs mapped by the server | +| `STRIPE_SUCCESS_URL` / `STRIPE_CANCEL_URL` / `STRIPE_PORTAL_RETURN_URL` | Trusted Checkout and portal redirects | | `PUBLIC_WEB_BASE` | 重設信連結、郵件 logo | -## Seed 管理員 +## 首次管理員 -migration `000003_seed_admin`: +Migration 不建立預設管理員,也不包含固定帳密。首次部署請設定 runtime secret 後執行: -| 欄位 | 值 | -|------|-----| -| Email | `admin@30cm.net`(以 seed JSON 為準) | -| 密碼 | `admin123`(seed 註解;若登入失敗見下方) | -| uid | `1000000` | - -密碼政策若為 **≥12 碼**,seed 舊密可能無法當「新密碼」使用,但 **登入仍可比對 hash**。 -若無法登入:用 `register` 開新帳或改 seed 後 wipe 再 migrate。 - -前端:**設定 → 資料來源 → Live**。 +```bash +make -C apps/backend seeder +``` ## 埠與 URL diff --git a/deploy/prod/.gitignore b/deploy/prod/.gitignore new file mode 100644 index 0000000..d4f588e --- /dev/null +++ b/deploy/prod/.gitignore @@ -0,0 +1 @@ +artifacts/ diff --git a/deploy/prod/README.md b/deploy/prod/README.md new file mode 100644 index 0000000..bbf1424 --- /dev/null +++ b/deploy/prod/README.md @@ -0,0 +1,110 @@ +# Harbor Desk Production Deployment + +This deployment targets Ubuntu amd64 at `daniel@10.0.0.33` and serves +`threads-tool-dev.30cm.net`. + +## Layout + +- Host Nginx serves immutable static web releases. +- Gateway uses blue/green systemd slots on `127.0.0.1:8888` and `:8889`. +- Worker uses matching systemd slots and guarded Mongo/Redis leases. +- Docker Compose runs MongoDB, Redis, MinIO, Prometheus, Grafana, Loki, + Alertmanager, Alloy, node-exporter, cAdvisor, and blackbox-exporter. +- Runtime secrets stay in `/etc/harbor/harbor.env` with mode `0600`. +- Releases stay under `/opt/harbor/releases`; `/opt/harbor/current` changes + atomically. + +## First Install + +The dedicated SSH key is expected at `~/.ssh/harbor_deploy`. + +```bash +./deploy/prod/bootstrap-server.sh +./deploy/prod/release.sh +``` + +Bootstrap asks for remote sudo once. It then grants `daniel` passwordless sudo +only for the validated activate, rollback, and status scripts. + +Edit external provider credentials after bootstrap: + +```bash +ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 +sudoedit /etc/harbor/harbor.env +``` + +Restart the active services after changing runtime env values. + +## Normal Release + +```bash +./deploy/prod/release.sh +``` + +The command runs backend tests/vet, frontend tests, builds Linux amd64 binaries +and static assets, uploads a checksummed archive, migrates forward, starts the +inactive gateway, checks health, reloads Nginx, then hands work to the new +worker. The old gateway remains available for Nginx connection draining. + +If an earlier inactive gateway still has established connections, activation +refuses to replace it. Retry after those requests finish. + +To deploy an already built artifact: + +```bash +./deploy/prod/release.sh deploy/prod/artifacts/harbor-*.tar.gz +``` + +## Rollback + +```bash +./deploy/prod/rollback.sh +``` + +Rollback switches application code only. Never automatically run a down +migration; production migrations must remain backward compatible. + +## TLS + +The public hostname is Cloudflare-proxied while the origin has a private IP. +Use a restricted Cloudflare DNS API token to obtain a DNS-01 certificate: + +```bash +sudo install -m 0600 /dev/null /etc/letsencrypt/cloudflare.ini +sudoedit /etc/letsencrypt/cloudflare.ini +# dns_cloudflare_api_token = REPLACE_WITH_RESTRICTED_TOKEN +sudo certbot certonly --dns-cloudflare \ + --dns-cloudflare-credentials /etc/letsencrypt/cloudflare.ini \ + -d threads-tool-dev.30cm.net +sudo /opt/harbor/deploy/remote/enable-tls.sh +``` + +Ensure Cloudflare Tunnel, router forwarding, or split DNS sends origin traffic +to `10.0.0.33`. A public Cloudflare DNS record alone cannot reach a private IP. + +## Monitoring + +All monitoring ports bind to localhost. Open Grafana through an SSH tunnel: + +```bash +ssh -i ~/.ssh/harbor_deploy -L 3000:127.0.0.1:3000 daniel@10.0.0.33 +``` + +Then browse `http://127.0.0.1:3000`. The generated Grafana password is in +`/etc/harbor/harbor.env` and should be read with sudo. + +Useful commands: + +```bash +ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \ + sudo /opt/harbor/deploy/remote/status.sh +ssh -i ~/.ssh/harbor_deploy daniel@10.0.0.33 \ + journalctl -u 'harbor-gateway@*' -f +``` + +## Backups + +`harbor-backup.timer` creates daily Mongo and MinIO backups under +`/var/backups/harbor` and retains 14 days. This protects against accidental +deletion but not loss of the machine or disk. Add an off-host copy before using +the service for irreplaceable production data. diff --git a/deploy/prod/backup/backup.sh b/deploy/prod/backup/backup.sh new file mode 100755 index 0000000..7b34cb5 --- /dev/null +++ b/deploy/prod/backup/backup.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -eu + +umask 077 + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +BACKUP_DIR=${BACKUP_DIR:-/var/backups/haixun} +BACKUP_RETENTION_DAYS=${BACKUP_RETENTION_DAYS:-14} +BACKUP_STATUS_DIR=${BACKUP_STATUS_DIR:-"$SCRIPT_DIR/status"} + +: "${MONGO_URI:?MONGO_URI is required}" +: "${MINIO_ENDPOINT:?MINIO_ENDPOINT is required}" +: "${MINIO_ACCESS_KEY:?MINIO_ACCESS_KEY is required}" +: "${MINIO_SECRET_KEY:?MINIO_SECRET_KEY is required}" +: "${MINIO_BUCKET:?MINIO_BUCKET is required}" + +for command_name in docker find; do + if ! command -v "$command_name" >/dev/null 2>&1; then + printf '%s\n' "Required command not found: $command_name" >&2 + exit 1 + fi +done + +case "$BACKUP_RETENTION_DAYS" in + ''|*[!0-9]*) + printf '%s\n' "BACKUP_RETENTION_DAYS must be a non-negative integer" >&2 + exit 1 + ;; +esac + +mkdir -p "$BACKUP_DIR" "$BACKUP_STATUS_DIR" +chmod 0755 "$BACKUP_STATUS_DIR" +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +work_dir="$BACKUP_DIR/.${timestamp}.tmp" +final_dir="$BACKUP_DIR/$timestamp" + +cleanup() { + rm -rf "$work_dir" +} +trap cleanup EXIT HUP INT TERM + +mkdir -p "$work_dir/minio" +docker compose --env-file /etc/harbor/harbor.env -f /opt/harbor/deploy/compose/docker-compose.yml \ + exec -T mongo mongodump --uri="$MONGO_URI" --archive --gzip --quiet > "$work_dir/mongodb.archive.gz" + +docker run --rm --network haixun-prod_data \ + -e MINIO_ACCESS_KEY -e MINIO_SECRET_KEY -e MINIO_BUCKET \ + -v "$work_dir/minio:/backup" \ + --entrypoint /bin/sh minio/mc:RELEASE.2025-04-16T18-13-26Z -ec \ + 'mc alias set backup-source http://minio:9000 "$MINIO_ACCESS_KEY" "$MINIO_SECRET_KEY" >/dev/null; mc mirror --overwrite --retry "backup-source/$MINIO_BUCKET" /backup >/dev/null' + +mv "$work_dir" "$final_dir" +trap - EXIT HUP INT TERM + +marker_tmp="$BACKUP_STATUS_DIR/.haixun_backup.prom.tmp" +printf '# HELP haixun_backup_last_success_timestamp_seconds Unix timestamp of the last successful production backup.\n' > "$marker_tmp" +printf '# TYPE haixun_backup_last_success_timestamp_seconds gauge\n' >> "$marker_tmp" +printf 'haixun_backup_last_success_timestamp_seconds %s\n' "$(date -u +%s)" >> "$marker_tmp" +chmod 0644 "$marker_tmp" +mv "$marker_tmp" "$BACKUP_STATUS_DIR/haixun_backup.prom" + +find "$BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d ! -name '.*.tmp' -mtime "+$BACKUP_RETENTION_DAYS" -exec rm -rf -- {} + +printf '%s\n' "Backup completed: $final_dir" diff --git a/deploy/prod/backup/status/.gitkeep b/deploy/prod/backup/status/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/deploy/prod/bootstrap-server.sh b/deploy/prod/bootstrap-server.sh new file mode 100755 index 0000000..1d18ca5 --- /dev/null +++ b/deploy/prod/bootstrap-server.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +TARGET=${DEPLOY_TARGET:-daniel@10.0.0.33} +SSH_KEY=${DEPLOY_SSH_KEY:-$HOME/.ssh/harbor_deploy} +REMOTE_STAGE=/tmp/harbor-prod-bootstrap +ssh_opts=(-i "$SSH_KEY" -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes) + +ssh "${ssh_opts[@]}" "$TARGET" "rm -rf '$REMOTE_STAGE' && mkdir -p '$REMOTE_STAGE'" +rsync -az --delete -e "ssh -i $SSH_KEY -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" \ + --exclude artifacts "$SCRIPT_DIR/" "$TARGET:$REMOTE_STAGE/" +printf '%s\n' "The remote host may request the sudo password once." +ssh -t "${ssh_opts[@]}" "$TARGET" "sudo '$REMOTE_STAGE/remote/bootstrap.sh' '$REMOTE_STAGE'" diff --git a/deploy/prod/build-release.sh b/deploy/prod/build-release.sh new file mode 100755 index 0000000..9b8fc10 --- /dev/null +++ b/deploy/prod/build-release.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ROOT_DIR=$(cd "$SCRIPT_DIR/../.." && pwd) +BACKEND_DIR="$ROOT_DIR/apps/backend" +WEB_DIR="$ROOT_DIR/apps/web" +ARTIFACT_DIR="$SCRIPT_DIR/artifacts" +timestamp=$(date -u +%Y%m%dT%H%M%SZ) +revision=$(git -C "$ROOT_DIR" rev-parse --short HEAD 2>/dev/null || printf unknown) +if ! git -C "$ROOT_DIR" diff --quiet --ignore-submodules -- 2>/dev/null || [[ -n $(git -C "$ROOT_DIR" ls-files --others --exclude-standard 2>/dev/null) ]]; then + revision="$revision-dirty" +fi +release_id="harbor-$timestamp-$revision" +stage=$(mktemp -d "/tmp/${release_id}.XXXXXX") +trap 'rm -rf "$stage"' EXIT + +mkdir -p "$stage/bin" "$stage/web" "$stage/migrations" "$ARTIFACT_DIR" + +if [[ ${SKIP_TESTS:-0} != 1 ]]; then + (cd "$BACKEND_DIR" && go test ./... && go vet ./...) + (cd "$WEB_DIR" && npm test) +fi + +printf '%s\n' "building Linux amd64 gateway and worker" +(cd "$BACKEND_DIR" && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o "$stage/bin/gateway" .) +(cd "$BACKEND_DIR" && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o "$stage/bin/worker" ./cmd/worker) +GOBIN="$stage/bin" CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go install -tags mongodb github.com/golang-migrate/migrate/v4/cmd/migrate@v4.18.3 + +printf '%s\n' "building static web" +(cd "$WEB_DIR" && npm ci && VITE_API_BASE= npm run build) +cp -a "$WEB_DIR/dist/." "$stage/web/" +cp -a "$BACKEND_DIR/generate/database/mongo/." "$stage/migrations/" + +cat > "$stage/release.txt" < manifest.sha256) +find "$stage" -type d -exec chmod 0755 {} + +find "$stage" -type f -exec chmod 0644 {} + +chmod 0755 "$stage/bin/gateway" "$stage/bin/worker" "$stage/bin/migrate" + +artifact="$ARTIFACT_DIR/$release_id.tar.gz" +tar -C "$stage" -czf "$artifact" . +sha256sum "$artifact" > "$artifact.sha256" +printf '%s\n' "$artifact" diff --git a/deploy/prod/compose/.env.example b/deploy/prod/compose/.env.example new file mode 100644 index 0000000..d8c2f74 --- /dev/null +++ b/deploy/prod/compose/.env.example @@ -0,0 +1,28 @@ +MONGO_ROOT_USER=haixun +MONGO_ROOT_PASSWORD=CHANGE_ME +MONGO_DATABASE=haixun +MONGO_PORT=27017 + +REDIS_PASSWORD=CHANGE_ME +REDIS_PORT=6379 + +MINIO_ROOT_USER=haixun +MINIO_ROOT_PASSWORD=CHANGE_ME +MINIO_SERVER_URL=https://assets.example.com +MINIO_BROWSER_REDIRECT_URL=https://minio-admin.example.com +MINIO_API_PORT=9000 +MINIO_CONSOLE_PORT=9001 + +GRAFANA_ADMIN_USER=admin +GRAFANA_ADMIN_PASSWORD=CHANGE_ME +GRAFANA_ROOT_URL=https://grafana.example.com +GRAFANA_PORT=3000 + +PROMETHEUS_PORT=9090 +PROMETHEUS_RETENTION=30d +LOKI_PORT=3100 +ALLOY_PORT=12345 +ALERTMANAGER_PORT=9093 +NODE_EXPORTER_PORT=9100 +CADVISOR_PORT=8080 +BLACKBOX_EXPORTER_PORT=9115 diff --git a/deploy/prod/compose/docker-compose.yml b/deploy/prod/compose/docker-compose.yml new file mode 100644 index 0000000..b1e2c50 --- /dev/null +++ b/deploy/prod/compose/docker-compose.yml @@ -0,0 +1,253 @@ +name: haixun-prod + +services: + mongo: + image: mongo:7.0.24 + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: ${MONGO_ROOT_USER:?MONGO_ROOT_USER is required} + MONGO_INITDB_ROOT_PASSWORD: ${MONGO_ROOT_PASSWORD:?MONGO_ROOT_PASSWORD is required} + MONGO_INITDB_DATABASE: ${MONGO_DATABASE:-haixun} + command: ["mongod", "--bind_ip_all", "--auth"] + ports: + - "127.0.0.1:${MONGO_PORT:-27017}:27017" + volumes: + - mongo_data:/data/db + - mongo_config:/data/configdb + healthcheck: + test: ["CMD-SHELL", "mongosh --quiet --host 127.0.0.1 --username \"$$MONGO_INITDB_ROOT_USERNAME\" --password \"$$MONGO_INITDB_ROOT_PASSWORD\" --authenticationDatabase admin --eval \"db.adminCommand('ping').ok\" | grep 1"] + interval: 15s + timeout: 10s + retries: 10 + start_period: 40s + stop_grace_period: 60s + networks: [data] + + redis: + image: redis:7.4.5-alpine + restart: unless-stopped + environment: + REDIS_PASSWORD: ${REDIS_PASSWORD:?REDIS_PASSWORD is required} + command: + - sh + - -ec + - exec redis-server --appendonly yes --appendfsync everysec --requirepass "$$REDIS_PASSWORD" + ports: + - "127.0.0.1:${REDIS_PORT:-6379}:6379" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli ping | grep PONG"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 15s + stop_grace_period: 30s + networks: [data] + + minio: + image: minio/minio:RELEASE.2025-04-22T22-12-26Z + restart: unless-stopped + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER:?MINIO_ROOT_USER is required} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?MINIO_ROOT_PASSWORD is required} + MINIO_BROWSER_REDIRECT_URL: ${MINIO_BROWSER_REDIRECT_URL:?MINIO_BROWSER_REDIRECT_URL is required} + MINIO_SERVER_URL: ${MINIO_SERVER_URL:?MINIO_SERVER_URL is required} + ports: + - "127.0.0.1:${MINIO_API_PORT:-9000}:9000" + - "127.0.0.1:${MINIO_CONSOLE_PORT:-9001}:9001" + volumes: + - minio_data:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"] + interval: 15s + timeout: 10s + retries: 10 + start_period: 30s + stop_grace_period: 60s + networks: [data] + + prometheus: + image: prom/prometheus:v3.5.0 + restart: unless-stopped + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=${PROMETHEUS_RETENTION:-30d} + - --storage.tsdb.wal-compression + - --web.enable-lifecycle + ports: + - "127.0.0.1:${PROMETHEUS_PORT:-9090}:9090" + volumes: + - ../monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - ../monitoring/prometheus/rules:/etc/prometheus/rules:ro + - ../monitoring/prometheus/targets:/etc/prometheus/targets:ro + - prometheus_data:/prometheus + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:9090/-/healthy"] + interval: 15s + timeout: 5s + retries: 10 + start_period: 20s + depends_on: + alertmanager: + condition: service_started + node-exporter: + condition: service_started + cadvisor: + condition: service_started + blackbox-exporter: + condition: service_started + networks: [monitoring] + + grafana: + image: grafana/grafana:12.0.2 + restart: unless-stopped + environment: + GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER:?GRAFANA_ADMIN_USER is required} + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD:?GRAFANA_ADMIN_PASSWORD is required} + GF_SERVER_ROOT_URL: ${GRAFANA_ROOT_URL:?GRAFANA_ROOT_URL is required} + GF_USERS_ALLOW_SIGN_UP: "false" + GF_AUTH_ANONYMOUS_ENABLED: "false" + # Grafana is localhost-only and reached through an SSH tunnel. + GF_SECURITY_COOKIE_SECURE: "false" + GF_SECURITY_COOKIE_SAMESITE: strict + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + GF_LOG_MODE: console + ports: + - "127.0.0.1:${GRAFANA_PORT:-3000}:3000" + volumes: + - ../monitoring/grafana/provisioning:/etc/grafana/provisioning:ro + - grafana_data:/var/lib/grafana + healthcheck: + test: ["CMD-SHELL", "wget --spider -q http://127.0.0.1:3000/api/health"] + interval: 15s + timeout: 5s + retries: 10 + start_period: 30s + depends_on: + prometheus: + condition: service_healthy + loki: + condition: service_started + networks: [monitoring] + + loki: + image: grafana/loki:3.5.1 + restart: unless-stopped + command: ["-config.file=/etc/loki/loki.yml"] + ports: + - "127.0.0.1:${LOKI_PORT:-3100}:3100" + volumes: + - ../monitoring/loki/loki.yml:/etc/loki/loki.yml:ro + - loki_data:/loki + networks: [monitoring] + + alloy: + image: grafana/alloy:v1.9.1 + restart: unless-stopped + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + ports: + - "127.0.0.1:${ALLOY_PORT:-12345}:12345" + volumes: + - ../monitoring/alloy/config.alloy:/etc/alloy/config.alloy:ro + - alloy_data:/var/lib/alloy/data + - /var/run/docker.sock:/var/run/docker.sock:ro + - /var/log/journal:/var/log/journal:ro + - /run/log/journal:/run/log/journal:ro + - /etc/machine-id:/etc/machine-id:ro + depends_on: + loki: + condition: service_started + networks: [monitoring] + + alertmanager: + image: prom/alertmanager:v0.28.1 + restart: unless-stopped + command: + - --config.file=/etc/alertmanager/alertmanager.yml + - --storage.path=/alertmanager + ports: + - "127.0.0.1:${ALERTMANAGER_PORT:-9093}:9093" + volumes: + - ../monitoring/alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml:ro + - ../monitoring/alertmanager/templates:/etc/alertmanager/templates:ro + - alertmanager_data:/alertmanager + healthcheck: + test: ["CMD", "wget", "--spider", "-q", "http://127.0.0.1:9093/-/healthy"] + interval: 15s + timeout: 5s + retries: 10 + start_period: 20s + networks: [monitoring] + + node-exporter: + image: prom/node-exporter:v1.9.1 + restart: unless-stopped + command: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + - --path.rootfs=/rootfs + - --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/) + - --collector.textfile.directory=/var/lib/node_exporter/textfile_collector + ports: + - "127.0.0.1:${NODE_EXPORTER_PORT:-9100}:9100" + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro,rslave + - ../backup/status:/var/lib/node_exporter/textfile_collector:ro + read_only: true + security_opt: [no-new-privileges:true] + networks: [monitoring] + + cadvisor: + image: gcr.io/cadvisor/cadvisor:v0.52.1 + restart: unless-stopped + command: ["--docker_only=true", "--housekeeping_interval=30s"] + ports: + - "127.0.0.1:${CADVISOR_PORT:-8080}:8080" + volumes: + - /:/rootfs:ro + - /var/run:/var/run:ro + - /sys:/sys:ro + - /var/lib/docker:/var/lib/docker:ro + devices: + - /dev/kmsg:/dev/kmsg + privileged: true + networks: [monitoring] + + blackbox-exporter: + image: prom/blackbox-exporter:v0.27.0 + restart: unless-stopped + command: ["--config.file=/etc/blackbox_exporter/blackbox.yml"] + ports: + - "127.0.0.1:${BLACKBOX_EXPORTER_PORT:-9115}:9115" + volumes: + - ../monitoring/blackbox/blackbox.yml:/etc/blackbox_exporter/blackbox.yml:ro + read_only: true + security_opt: [no-new-privileges:true] + extra_hosts: + - host.docker.internal:host-gateway + networks: [monitoring] + +volumes: + mongo_data: + mongo_config: + redis_data: + minio_data: + prometheus_data: + grafana_data: + loki_data: + alloy_data: + alertmanager_data: + +networks: + data: + monitoring: diff --git a/deploy/prod/config/gateway.yaml b/deploy/prod/config/gateway.yaml new file mode 100644 index 0000000..f30df61 --- /dev/null +++ b/deploy/prod/config/gateway.yaml @@ -0,0 +1,99 @@ +Name: haixun-gateway +Host: 127.0.0.1 +Port: 8888 +Timeout: 120000 +MaxBytes: 104857600 + +Log: + Mode: console + Level: info + +CacheRedis: + - Host: 127.0.0.1:6379 + Type: node + Pass: "" + +Redis: + Namespace: "haixun:prod:v1" + AIModelCacheFingerprintSecret: "" + +Mongo: + URI: "mongodb://127.0.0.1:27017" + Database: "haixun" + +Auth: + AccessSecret: "" + AccessExpire: 86400 + RefreshSecret: "" + RefreshExpire: 604800 + +MemberSettingsEncryption: + CurrentKeyID: "v1" + Key: "" + +Platform: + AIKey: "" + XAIKey: "" + OpenCodeKey: "" + ExaKey: "" + ThreadsAppId: "" + ThreadsAppSecret: "" + +Scout: + SessionSecret: "" + CrawlerEndpoint: "" + CrawlerToken: "" + +Worker: + ID: "" + PollIntervalMs: 2000 + +Seed: + AdminPassword: "" + +Bcrypt: + Cost: 12 + +OAuth: + GoogleClientID: "" + LineClientID: "" + LineClientSecret: "" + LineRedirectURI: "" + +Stripe: + Enabled: false + SecretKey: "" + WebhookSecret: "" + StarterPriceID: "" + ProPriceID: "" + SuccessURL: "" + CancelURL: "" + PortalReturnURL: "" + +Mail: + Sender: "Harbor Desk " + SMTP: + Host: "" + Port: 587 + User: "" + Password: "" + DevExposeCode: false + +PublicWebBase: "https://threads-tool-dev.30cm.net" +PublicAPIBase: "https://threads-tool-dev.30cm.net" +CORSAllowedOrigins: + - "https://threads-tool-dev.30cm.net" + +Brand: + Name: "Harbor Desk" + LogoURL: "" + Copyright: "Harbor Desk" + +ObjectStorage: + Endpoint: "http://127.0.0.1:9000" + Region: "us-east-1" + Bucket: "haixun-assets" + AccessKey: "" + SecretKey: "" + UsePathStyle: true + PublicBaseURL: "https://threads-tool-dev.30cm.net/haixun-assets" diff --git a/deploy/prod/harbor.env.example b/deploy/prod/harbor.env.example new file mode 100644 index 0000000..9ed10fc --- /dev/null +++ b/deploy/prod/harbor.env.example @@ -0,0 +1,28 @@ +# Installed as /etc/harbor/harbor.env by remote/bootstrap.sh. +# Bootstrap generates all first-party secrets. Add external provider credentials +# directly on the server and keep this file free of real secrets. +PUBLIC_HOST=threads-tool-dev.30cm.net +PUBLIC_WEB_BASE=https://threads-tool-dev.30cm.net +PUBLIC_API_BASE=https://threads-tool-dev.30cm.net +CORS_ALLOWED_ORIGINS=https://threads-tool-dev.30cm.net + +PLATFORM_XAI_KEY= +PLATFORM_OPENCODE_KEY= +PLATFORM_EXA_KEY= +THREADS_APP_ID= +THREADS_APP_SECRET= + +STRIPE_ENABLED=false +STRIPE_SECRET_KEY= +STRIPE_WEBHOOK_SECRET= +STRIPE_STARTER_PRICE_ID= +STRIPE_PRO_PRICE_ID= +STRIPE_SUCCESS_URL="https://threads-tool-dev.30cm.net/app/usage/checkout?result=success&checkout_id={CHECKOUT_ID}&plan={PLAN_ID}" +STRIPE_CANCEL_URL="https://threads-tool-dev.30cm.net/app/usage/checkout?result=cancel&plan={PLAN_ID}" +STRIPE_PORTAL_RETURN_URL=https://threads-tool-dev.30cm.net/app/usage/plans + +MAIL_SENDER="Harbor Desk " +MAIL_SMTP_HOST= +MAIL_SMTP_PORT=587 +MAIL_SMTP_USER= +MAIL_SMTP_PASSWORD= diff --git a/deploy/prod/monitoring/alertmanager/alertmanager.yml b/deploy/prod/monitoring/alertmanager/alertmanager.yml new file mode 100644 index 0000000..58416b5 --- /dev/null +++ b/deploy/prod/monitoring/alertmanager/alertmanager.yml @@ -0,0 +1,22 @@ +global: + resolve_timeout: 5m + +templates: + - /etc/alertmanager/templates/*.tmpl + +route: + receiver: local-ui + group_by: [alertname, service] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + +receivers: + - name: local-ui + +inhibit_rules: + - source_matchers: + - severity="critical" + target_matchers: + - severity="warning" + equal: [alertname, instance] diff --git a/deploy/prod/monitoring/alertmanager/templates/default.tmpl b/deploy/prod/monitoring/alertmanager/templates/default.tmpl new file mode 100644 index 0000000..d4fe417 --- /dev/null +++ b/deploy/prod/monitoring/alertmanager/templates/default.tmpl @@ -0,0 +1,9 @@ +{{ define "haixun.title" }}[{{ .Status | toUpper }}] {{ .CommonLabels.alertname }}{{ end }} +{{ define "haixun.message" }} +{{ range .Alerts -}} +Summary: {{ .Annotations.summary }} +Description: {{ .Annotations.description }} +Instance: {{ .Labels.instance }} +Started: {{ .StartsAt }} +{{ end -}} +{{ end }} diff --git a/deploy/prod/monitoring/alloy/config.alloy b/deploy/prod/monitoring/alloy/config.alloy new file mode 100644 index 0000000..29d9547 --- /dev/null +++ b/deploy/prod/monitoring/alloy/config.alloy @@ -0,0 +1,43 @@ +discovery.docker "local" { + host = "unix:///var/run/docker.sock" +} + +discovery.relabel "docker" { + targets = discovery.docker.local.targets + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "service" + } + + rule { + source_labels = ["__meta_docker_container_log_stream"] + target_label = "stream" + } +} + +loki.source.docker "local" { + host = "unix:///var/run/docker.sock" + targets = discovery.relabel.docker.output + forward_to = [loki.write.local.receiver] + labels = { job = "docker" } +} + +loki.source.journal "local" { + path = "/var/log/journal" + max_age = "24h" + forward_to = [loki.write.local.receiver] + labels = { job = "journald" } +} + +loki.write "local" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/deploy/prod/monitoring/blackbox/blackbox.yml b/deploy/prod/monitoring/blackbox/blackbox.yml new file mode 100644 index 0000000..90728e3 --- /dev/null +++ b/deploy/prod/monitoring/blackbox/blackbox.yml @@ -0,0 +1,13 @@ +modules: + http_2xx: + prober: http + timeout: 10s + http: + method: GET + preferred_ip_protocol: ip4 + follow_redirects: true + fail_if_not_ssl: false + + tcp_connect: + prober: tcp + timeout: 5s diff --git a/deploy/prod/monitoring/grafana/provisioning/datasources/datasources.yml b/deploy/prod/monitoring/grafana/provisioning/datasources/datasources.yml new file mode 100644 index 0000000..6b79bbb --- /dev/null +++ b/deploy/prod/monitoring/grafana/provisioning/datasources/datasources.yml @@ -0,0 +1,40 @@ +apiVersion: 1 + +deleteDatasources: + - name: Prometheus + orgId: 1 + - name: Loki + orgId: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + jsonData: + timeInterval: 30s + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + derivedFields: + - name: TraceID + matcherRegex: 'trace[_-]?id["=: ]+([a-fA-F0-9]+)' + url: '' + + - name: Alertmanager + uid: alertmanager + type: alertmanager + access: proxy + url: http://alertmanager:9093 + editable: false + jsonData: + implementation: prometheus + handleGrafanaManagedAlerts: false diff --git a/deploy/prod/monitoring/loki/loki.yml b/deploy/prod/monitoring/loki/loki.yml new file mode 100644 index 0000000..37799ce --- /dev/null +++ b/deploy/prod/monitoring/loki/loki.yml @@ -0,0 +1,41 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + +schema_config: + configs: + - from: "2024-01-01" + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +storage_config: + filesystem: + directory: /loki/chunks + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +limits_config: + retention_period: 720h + allow_structured_metadata: true + volume_enabled: true + +analytics: + reporting_enabled: false diff --git a/deploy/prod/monitoring/prometheus/prometheus.yml b/deploy/prod/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..e13f7da --- /dev/null +++ b/deploy/prod/monitoring/prometheus/prometheus.yml @@ -0,0 +1,56 @@ +global: + scrape_interval: 30s + evaluation_interval: 30s + external_labels: + environment: production + +rule_files: + - /etc/prometheus/rules/*.yml + +alerting: + alertmanagers: + - static_configs: + - targets: [alertmanager:9093] + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: [prometheus:9090] + + - job_name: node + static_configs: + - targets: [node-exporter:9100] + + - job_name: cadvisor + static_configs: + - targets: [cadvisor:8080] + + - job_name: blackbox-http + metrics_path: /probe + params: + module: [http_2xx] + file_sd_configs: + - files: [/etc/prometheus/targets/blackbox.yml] + refresh_interval: 1m + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 + + - job_name: blackbox-tls + metrics_path: /probe + params: + module: [http_2xx] + file_sd_configs: + - files: [/etc/prometheus/targets/tls.yml] + refresh_interval: 1m + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 diff --git a/deploy/prod/monitoring/prometheus/rules/alerts.yml b/deploy/prod/monitoring/prometheus/rules/alerts.yml new file mode 100644 index 0000000..008d97f --- /dev/null +++ b/deploy/prod/monitoring/prometheus/rules/alerts.yml @@ -0,0 +1,92 @@ +groups: + - name: monitoring + rules: + - alert: PrometheusTargetDown + expr: up == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Prometheus target is down: {{ $labels.instance }}" + description: "Job {{ $labels.job }} has been unreachable for more than five minutes." + + - alert: HostHighCPU + expr: 100 * (1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > 90 + for: 15m + labels: + severity: warning + annotations: + summary: "Host CPU usage is above 90%" + description: "CPU usage on {{ $labels.instance }} has remained high for 15 minutes." + + - alert: HostLowMemory + expr: 100 * node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < 10 + for: 10m + labels: + severity: warning + annotations: + summary: "Host available memory is below 10%" + description: "Available memory on {{ $labels.instance }} has remained low for 10 minutes." + + - alert: HostDiskSpaceLow + expr: 100 * node_filesystem_avail_bytes{fstype!~"tmpfs|overlay|squashfs"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay|squashfs"} < 10 + for: 15m + labels: + severity: critical + annotations: + summary: "Host filesystem has less than 10% free space" + description: "{{ $labels.mountpoint }} on {{ $labels.instance }} is nearly full." + + - alert: HostFilesystemReadOnly + expr: node_filesystem_readonly{fstype!~"tmpfs|squashfs"} == 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Host filesystem became read-only" + description: "{{ $labels.mountpoint }} on {{ $labels.instance }} is read-only." + + - alert: ContainerMissing + expr: time() - container_last_seen{name!=""} > 180 + for: 5m + labels: + severity: warning + annotations: + summary: "Container disappeared: {{ $labels.name }}" + description: "cAdvisor has not seen the container for more than three minutes." + + - alert: ContainerHighCPU + expr: sum by (name) (rate(container_cpu_usage_seconds_total{name!=""}[5m])) * 100 > 90 + for: 15m + labels: + severity: warning + annotations: + summary: "Container CPU usage is high: {{ $labels.name }}" + description: "Container CPU consumption has exceeded 90% of one core for 15 minutes." + + - alert: BlackboxProbeFailed + expr: probe_success == 0 + for: 5m + labels: + severity: critical + annotations: + summary: "Endpoint probe failed: {{ $labels.instance }}" + description: "The endpoint has failed blackbox probes for five minutes." + + - alert: CertificateExpiringSoon + expr: probe_ssl_earliest_cert_expiry - time() < 14 * 24 * 60 * 60 + for: 1h + labels: + severity: warning + annotations: + summary: "TLS certificate expires within 14 days" + description: "The certificate for {{ $labels.instance }} expires soon." + + - alert: ProductionBackupMissing + expr: absent(haixun_backup_last_success_timestamp_seconds) or (time() - haixun_backup_last_success_timestamp_seconds > 26 * 60 * 60) + for: 30m + labels: + severity: critical + annotations: + summary: "Production backup is missing or stale" + description: "No successful MongoDB and MinIO backup has been recorded in the last 26 hours." diff --git a/deploy/prod/monitoring/prometheus/targets/blackbox.yml b/deploy/prod/monitoring/prometheus/targets/blackbox.yml new file mode 100644 index 0000000..eb9459f --- /dev/null +++ b/deploy/prod/monitoring/prometheus/targets/blackbox.yml @@ -0,0 +1,3 @@ +- targets: [http://host.docker.internal/health] + labels: + service: origin-health diff --git a/deploy/prod/monitoring/prometheus/targets/tls.yml b/deploy/prod/monitoring/prometheus/targets/tls.yml new file mode 100644 index 0000000..7adfe59 --- /dev/null +++ b/deploy/prod/monitoring/prometheus/targets/tls.yml @@ -0,0 +1,3 @@ +- targets: [https://threads-tool-dev.30cm.net/health] + labels: + service: public-tls diff --git a/deploy/prod/nginx/harbor-http.conf b/deploy/prod/nginx/harbor-http.conf new file mode 100644 index 0000000..91221b4 --- /dev/null +++ b/deploy/prod/nginx/harbor-http.conf @@ -0,0 +1,66 @@ +upstream harbor_active { + include /etc/nginx/harbor-active-server.conf; + keepalive 32; +} + +upstream harbor_minio { + server 127.0.0.1:9000; + keepalive 16; +} + +server { + listen 80; + listen [::]:80; + server_name threads-tool-dev.30cm.net 10.0.0.33; + + root /opt/harbor/current/web; + index index.html; + client_max_body_size 100m; + + location = /health { + proxy_pass http://harbor_active/api/v1/health; + proxy_set_header Host $host; + } + + location /api/ { + proxy_pass http://harbor_active; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Origin ""; + proxy_set_header Authorization $http_authorization; + proxy_set_header X-Member-Authorization $http_x_member_authorization; + proxy_set_header Connection ""; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + } + + location /haixun-assets/ { + proxy_pass http://harbor_minio; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + expires 7d; + add_header Cache-Control "public" always; + } + + location /assets/ { + try_files $uri =404; + expires 1y; + add_header Cache-Control "public, immutable" always; + } + + location = /index.html { + add_header Cache-Control "no-cache, no-store, must-revalidate" always; + } + + location / { + try_files $uri $uri/ /index.html; + } +} diff --git a/deploy/prod/nginx/harbor-tls.conf b/deploy/prod/nginx/harbor-tls.conf new file mode 100644 index 0000000..bde22e4 --- /dev/null +++ b/deploy/prod/nginx/harbor-tls.conf @@ -0,0 +1,82 @@ +upstream harbor_active { + include /etc/nginx/harbor-active-server.conf; + keepalive 32; +} + +upstream harbor_minio { + server 127.0.0.1:9000; + keepalive 16; +} + +server { + listen 80; + listen [::]:80; + server_name threads-tool-dev.30cm.net; + return 301 https://$host$request_uri; +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name threads-tool-dev.30cm.net; + + ssl_certificate /etc/letsencrypt/live/threads-tool-dev.30cm.net/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/threads-tool-dev.30cm.net/privkey.pem; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 1d; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Content-Type-Options nosniff always; + add_header Referrer-Policy strict-origin-when-cross-origin always; + + root /opt/harbor/current/web; + index index.html; + client_max_body_size 100m; + + location = /health { + proxy_pass http://harbor_active/api/v1/health; + proxy_set_header Host $host; + } + + location /api/ { + proxy_pass http://harbor_active; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header Origin ""; + proxy_set_header Authorization $http_authorization; + proxy_set_header X-Member-Authorization $http_x_member_authorization; + proxy_set_header Connection ""; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 3600s; + proxy_send_timeout 3600s; + } + + location /haixun-assets/ { + proxy_pass http://harbor_minio; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto https; + expires 7d; + add_header Cache-Control "public" always; + } + + location /assets/ { + try_files $uri =404; + expires 1y; + add_header Cache-Control "public, immutable" always; + } + + location = /index.html { + add_header Cache-Control "no-cache, no-store, must-revalidate" always; + } + + location / { + try_files $uri $uri/ /index.html; + } +} diff --git a/deploy/prod/release.sh b/deploy/prod/release.sh new file mode 100755 index 0000000..d6ce7ae --- /dev/null +++ b/deploy/prod/release.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +TARGET=${DEPLOY_TARGET:-daniel@10.0.0.33} +SSH_KEY=${DEPLOY_SSH_KEY:-$HOME/.ssh/harbor_deploy} +artifact=${1:-} + +if [[ -z "$artifact" ]]; then + "$SCRIPT_DIR/build-release.sh" + artifact=$(find "$SCRIPT_DIR/artifacts" -maxdepth 1 -type f -name 'harbor-*.tar.gz' -printf '%T@ %p\n' | sort -nr | cut -d' ' -f2- | sed -n '1p') +fi +test -f "$artifact" +test -f "$artifact.sha256" + +name=$(basename "$artifact") +checksum=$(cut -d' ' -f1 "$artifact.sha256") +ssh_opts=(-i "$SSH_KEY" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes) + +printf '%s\n' "uploading $name" +rsync -az --partial -e "ssh -i $SSH_KEY -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" "$artifact" "$TARGET:/opt/harbor/incoming/$name" +ssh "${ssh_opts[@]}" "$TARGET" sudo /opt/harbor/deploy/remote/activate-release.sh "/opt/harbor/incoming/$name" "$checksum" +ssh "${ssh_opts[@]}" "$TARGET" sudo /opt/harbor/deploy/remote/status.sh diff --git a/deploy/prod/remote/activate-release.sh b/deploy/prod/remote/activate-release.sh new file mode 100755 index 0000000..6e6bd24 --- /dev/null +++ b/deploy/prod/remote/activate-release.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${EUID} -ne 0 ]]; then + printf '%s\n' "activate-release must run as root" >&2 + exit 1 +fi + +archive=${1:-} +checksum=${2:-} +if [[ -z "$archive" || -z "$checksum" || ! -f "$archive" ]]; then + printf '%s\n' "usage: activate-release.sh " >&2 + exit 1 +fi +if [[ "$archive" != /opt/harbor/incoming/* ]]; then + printf '%s\n' "archive must be under /opt/harbor/incoming" >&2 + exit 1 +fi + +actual=$(sha256sum "$archive" | cut -d' ' -f1) +if [[ "$actual" != "$checksum" ]]; then + printf '%s\n' "release checksum mismatch" >&2 + exit 1 +fi + +release_id=$(basename "$archive" .tar.gz) +if [[ ! "$release_id" =~ ^harbor-[0-9]{8}T[0-9]{6}Z-[a-zA-Z0-9._-]+$ ]]; then + printf '%s\n' "invalid release archive name" >&2 + exit 1 +fi +release_dir=/opt/harbor/releases/$release_id +if [[ ! -d "$release_dir" ]]; then + install -d -m 0755 "$release_dir" + tar -xzf "$archive" -C "$release_dir" +fi +test -x "$release_dir/bin/gateway" +test -x "$release_dir/bin/worker" +test -x "$release_dir/bin/migrate" +test -f "$release_dir/web/index.html" +chown -R root:harbor "$release_dir" +find "$release_dir" -type d -exec chmod 0755 {} + +find "$release_dir" -type f -exec chmod 0644 {} + +chmod 0755 "$release_dir/bin/gateway" "$release_dir/bin/worker" "$release_dir/bin/migrate" + +active=blue +if [[ -f /etc/harbor/active-slot ]]; then + active=$(cat /etc/harbor/active-slot) +fi +if [[ "$active" == blue ]]; then + inactive=green + inactive_port=8889 +else + inactive=blue + inactive_port=8888 +fi + +if systemctl is-active --quiet "harbor-gateway@$inactive.service"; then + connections=$(ss -Htn state established "( sport = :$inactive_port )" | wc -l) + if (( connections > 0 )); then + printf '%s\n' "inactive slot still has $connections established connection(s); retry later" >&2 + exit 1 + fi + systemctl stop "harbor-gateway@$inactive.service" +fi +systemctl stop "harbor-worker@$inactive.service" 2>/dev/null || true + +set -a +source /etc/harbor/harbor.env +set +a +printf '%s\n' "running forward migrations" +"$release_dir/bin/migrate" -path "$release_dir/migrations" -database "$MONGO_URL" up || status=$? +if [[ ${status:-0} -ne 0 && ${status:-0} -ne 2 ]]; then + exit "$status" +fi + +ln -sfn "$release_dir" "/opt/harbor/slots/$inactive" +chown -h harbor:harbor "/opt/harbor/slots/$inactive" +systemctl start "harbor-gateway@$inactive.service" +for _ in $(seq 1 60); do + if curl -fsS "http://127.0.0.1:$inactive_port/api/v1/health" >/dev/null; then + break + fi + sleep 1 +done +if ! curl -fsS "http://127.0.0.1:$inactive_port/api/v1/health" >/dev/null; then + journalctl -u "harbor-gateway@$inactive.service" -n 80 --no-pager >&2 + systemctl stop "harbor-gateway@$inactive.service" + exit 1 +fi + +if [[ -L /opt/harbor/current ]]; then + readlink -f /opt/harbor/current > /etc/harbor/previous-release + printf '%s\n' "$active" > /etc/harbor/previous-slot +fi +ln -sfn "$release_dir" /opt/harbor/current.new +mv -Tf /opt/harbor/current.new /opt/harbor/current +printf 'server 127.0.0.1:%s;\n' "$inactive_port" > /etc/nginx/harbor-active-server.conf.new +mv -f /etc/nginx/harbor-active-server.conf.new /etc/nginx/harbor-active-server.conf +nginx -t +systemctl reload nginx +printf '%s\n' "$inactive" > /etc/harbor/active-slot + +systemctl start "harbor-worker@$inactive.service" +sleep 3 +if ! systemctl is-active --quiet "harbor-worker@$inactive.service"; then + journalctl -u "harbor-worker@$inactive.service" -n 80 --no-pager >&2 + /opt/harbor/deploy/remote/rollback.sh || true + exit 1 +fi +systemctl stop --no-block "harbor-worker@$active.service" 2>/dev/null || true + +curl -fsS http://127.0.0.1/health >/dev/null +rm -f "$archive" +printf '%s\n' "activated $release_id on $inactive ($inactive_port)" diff --git a/deploy/prod/remote/bootstrap.sh b/deploy/prod/remote/bootstrap.sh new file mode 100755 index 0000000..4ec1c9f --- /dev/null +++ b/deploy/prod/remote/bootstrap.sh @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${EUID} -ne 0 ]]; then + printf '%s\n' "bootstrap must run as root" >&2 + exit 1 +fi + +SOURCE_DIR=${1:-} +if [[ -z "$SOURCE_DIR" || ! -d "$SOURCE_DIR" ]]; then + printf '%s\n' "usage: bootstrap.sh " >&2 + exit 1 +fi + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install -y ca-certificates certbot curl jq nginx openssl python3-certbot-dns-cloudflare rsync tar gzip ufw fail2ban docker.io docker-compose-v2 +systemctl enable --now docker nginx fail2ban + +if ! id harbor >/dev/null 2>&1; then + useradd --system --home /var/lib/harbor --shell /usr/sbin/nologin harbor +fi + +install -d -m 0755 /opt/harbor /opt/harbor/releases /opt/harbor/slots /opt/harbor/deploy +install -d -m 2770 -o daniel -g harbor /opt/harbor/incoming +install -d -m 0750 -o harbor -g harbor /var/lib/harbor +install -d -m 0700 /var/backups/harbor +install -d -m 0750 -o root -g harbor /etc/harbor +rsync -a --delete "$SOURCE_DIR/" /opt/harbor/deploy/ +chown -R root:root /opt/harbor/deploy +chmod +x /opt/harbor/deploy/remote/*.sh /opt/harbor/deploy/backup/backup.sh + +install -m 0644 /opt/harbor/deploy/config/gateway.yaml /etc/harbor/gateway.yaml +install -m 0644 /opt/harbor/deploy/systemd/harbor-gateway@.service /etc/systemd/system/harbor-gateway@.service +install -m 0644 /opt/harbor/deploy/systemd/harbor-worker@.service /etc/systemd/system/harbor-worker@.service +install -m 0644 /opt/harbor/deploy/systemd/harbor-backup.service /etc/systemd/system/harbor-backup.service +install -m 0644 /opt/harbor/deploy/systemd/harbor-backup.timer /etc/systemd/system/harbor-backup.timer + +if [[ ! -f /etc/harbor/harbor.env ]]; then + mongo_password=$(openssl rand -hex 24) + redis_password=$(openssl rand -hex 24) + minio_password=$(openssl rand -hex 24) + grafana_password=$(openssl rand -base64 24 | tr -d '/+=') + cat > /etc/harbor/harbor.env < /etc/harbor/gateway-blue.env <<'EOF' +PORT=8888 +EOF +cat > /etc/harbor/gateway-green.env <<'EOF' +PORT=8889 +EOF +chmod 0644 /etc/harbor/gateway-blue.env /etc/harbor/gateway-green.env + +printf '%s\n' 'server 127.0.0.1:8888;' > /etc/nginx/harbor-active-server.conf +install -m 0644 /opt/harbor/deploy/nginx/harbor-http.conf /etc/nginx/sites-available/harbor.conf +ln -sfn /etc/nginx/sites-available/harbor.conf /etc/nginx/sites-enabled/harbor.conf +rm -f /etc/nginx/sites-enabled/default + +cat > /etc/sudoers.d/harbor-deploy <<'EOF' +daniel ALL=(root) NOPASSWD: /opt/harbor/deploy/remote/activate-release.sh * +daniel ALL=(root) NOPASSWD: /opt/harbor/deploy/remote/rollback.sh +daniel ALL=(root) NOPASSWD: /opt/harbor/deploy/remote/status.sh +EOF +chmod 0440 /etc/sudoers.d/harbor-deploy +visudo -cf /etc/sudoers.d/harbor-deploy + +systemctl daemon-reload +docker compose --env-file /etc/harbor/harbor.env -f /opt/harbor/deploy/compose/docker-compose.yml pull +docker compose --env-file /etc/harbor/harbor.env -f /opt/harbor/deploy/compose/docker-compose.yml up -d + +ufw allow OpenSSH +ufw allow 80/tcp +ufw allow 443/tcp +ufw --force enable +nginx -t +systemctl reload nginx +systemctl enable --now harbor-backup.timer + +printf '%s\n' "bootstrap complete" +printf '%s\n' "runtime secrets: /etc/harbor/harbor.env" diff --git a/deploy/prod/remote/enable-tls.sh b/deploy/prod/remote/enable-tls.sh new file mode 100755 index 0000000..72a2991 --- /dev/null +++ b/deploy/prod/remote/enable-tls.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${EUID} -ne 0 ]]; then + printf '%s\n' "enable-tls must run as root" >&2 + exit 1 +fi +domain=threads-tool-dev.30cm.net +if [[ ! -f "/etc/letsencrypt/live/$domain/fullchain.pem" || ! -f "/etc/letsencrypt/live/$domain/privkey.pem" ]]; then + printf '%s\n' "certificate not found; obtain a DNS-01 certificate for $domain first" >&2 + exit 1 +fi +install -m 0644 /opt/harbor/deploy/nginx/harbor-tls.conf /etc/nginx/sites-available/harbor.conf +nginx -t +systemctl reload nginx +printf '%s\n' "TLS enabled for $domain" diff --git a/deploy/prod/remote/rollback.sh b/deploy/prod/remote/rollback.sh new file mode 100755 index 0000000..a940fa2 --- /dev/null +++ b/deploy/prod/remote/rollback.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ ${EUID} -ne 0 ]]; then + printf '%s\n' "rollback must run as root" >&2 + exit 1 +fi +test -f /etc/harbor/previous-release +test -f /etc/harbor/previous-slot + +previous_release=$(cat /etc/harbor/previous-release) +previous_slot=$(cat /etc/harbor/previous-slot) +test -d "$previous_release" +if [[ "$previous_slot" == blue ]]; then + previous_port=8888 +else + previous_port=8889 +fi + +ln -sfn "$previous_release" "/opt/harbor/slots/$previous_slot" +systemctl start "harbor-gateway@$previous_slot.service" +for _ in $(seq 1 60); do + curl -fsS "http://127.0.0.1:$previous_port/api/v1/health" >/dev/null && break + sleep 1 +done +curl -fsS "http://127.0.0.1:$previous_port/api/v1/health" >/dev/null + +current_release=$(readlink -f /opt/harbor/current) +current_slot=$(cat /etc/harbor/active-slot) +printf '%s\n' "$current_release" > /etc/harbor/previous-release +printf '%s\n' "$current_slot" > /etc/harbor/previous-slot +ln -sfn "$previous_release" /opt/harbor/current.new +mv -Tf /opt/harbor/current.new /opt/harbor/current +printf 'server 127.0.0.1:%s;\n' "$previous_port" > /etc/nginx/harbor-active-server.conf.new +mv -f /etc/nginx/harbor-active-server.conf.new /etc/nginx/harbor-active-server.conf +nginx -t +systemctl reload nginx +printf '%s\n' "$previous_slot" > /etc/harbor/active-slot +systemctl start "harbor-worker@$previous_slot.service" +systemctl stop --no-block "harbor-worker@$current_slot.service" 2>/dev/null || true +curl -fsS http://127.0.0.1/health >/dev/null +printf '%s\n' "rolled back to $(basename "$previous_release") on $previous_slot" diff --git a/deploy/prod/remote/status.sh b/deploy/prod/remote/status.sh new file mode 100755 index 0000000..c67e40e --- /dev/null +++ b/deploy/prod/remote/status.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +active=none +[[ -f /etc/harbor/active-slot ]] && active=$(cat /etc/harbor/active-slot) +printf 'active_slot=%s\n' "$active" +printf 'current_release=%s\n' "$(readlink -f /opt/harbor/current 2>/dev/null || printf none)" +systemctl --no-pager --full status "harbor-gateway@$active.service" "harbor-worker@$active.service" 2>/dev/null || true +docker compose --env-file /etc/harbor/harbor.env -f /opt/harbor/deploy/compose/docker-compose.yml ps +curl -fsS http://127.0.0.1/health +printf '\n' diff --git a/deploy/prod/rollback.sh b/deploy/prod/rollback.sh new file mode 100755 index 0000000..9ff9b98 --- /dev/null +++ b/deploy/prod/rollback.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET=${DEPLOY_TARGET:-daniel@10.0.0.33} +SSH_KEY=${DEPLOY_SSH_KEY:-$HOME/.ssh/harbor_deploy} +ssh -i "$SSH_KEY" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes "$TARGET" sudo /opt/harbor/deploy/remote/rollback.sh diff --git a/deploy/prod/systemd/harbor-backup.service b/deploy/prod/systemd/harbor-backup.service new file mode 100644 index 0000000..25ff502 --- /dev/null +++ b/deploy/prod/systemd/harbor-backup.service @@ -0,0 +1,16 @@ +[Unit] +Description=Harbor Desk local backup +After=docker.service +Requires=docker.service + +[Service] +Type=oneshot +EnvironmentFile=/etc/harbor/harbor.env +Environment=BACKUP_DIR=/var/backups/harbor +Environment=BACKUP_STATUS_DIR=/opt/harbor/deploy/backup/status +ExecStart=/opt/harbor/deploy/backup/backup.sh +User=root +Group=root +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 diff --git a/deploy/prod/systemd/harbor-backup.timer b/deploy/prod/systemd/harbor-backup.timer new file mode 100644 index 0000000..3b02387 --- /dev/null +++ b/deploy/prod/systemd/harbor-backup.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Run Harbor Desk backup daily + +[Timer] +OnCalendar=*-*-* 03:15:00 +RandomizedDelaySec=900 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/deploy/prod/systemd/harbor-gateway@.service b/deploy/prod/systemd/harbor-gateway@.service new file mode 100644 index 0000000..66693c8 --- /dev/null +++ b/deploy/prod/systemd/harbor-gateway@.service @@ -0,0 +1,28 @@ +[Unit] +Description=Harbor Desk gateway (%i) +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service + +[Service] +Type=simple +User=harbor +Group=harbor +EnvironmentFile=/etc/harbor/harbor.env +EnvironmentFile=/etc/harbor/gateway-%i.env +ExecStart=/opt/harbor/slots/%i/bin/gateway -f /etc/harbor/gateway.yaml +Restart=on-failure +RestartSec=3 +TimeoutStopSec=300 +KillSignal=SIGTERM +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/harbor +StandardOutput=journal +StandardError=journal +SyslogIdentifier=harbor-gateway-%i + +[Install] +WantedBy=multi-user.target diff --git a/deploy/prod/systemd/harbor-worker@.service b/deploy/prod/systemd/harbor-worker@.service new file mode 100644 index 0000000..07a9726 --- /dev/null +++ b/deploy/prod/systemd/harbor-worker@.service @@ -0,0 +1,28 @@ +[Unit] +Description=Harbor Desk worker (%i) +After=network-online.target docker.service +Wants=network-online.target +Requires=docker.service + +[Service] +Type=simple +User=harbor +Group=harbor +EnvironmentFile=/etc/harbor/harbor.env +Environment=WORKER_ID=harbor-worker-%i +ExecStart=/opt/harbor/slots/%i/bin/worker -f /etc/harbor/gateway.yaml +Restart=on-failure +RestartSec=5 +TimeoutStopSec=1800 +KillSignal=SIGTERM +NoNewPrivileges=true +PrivateTmp=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/harbor +StandardOutput=journal +StandardError=journal +SyslogIdentifier=harbor-worker-%i + +[Install] +WantedBy=multi-user.target diff --git a/deploy/scripts/build.sh b/deploy/scripts/build.sh index e34eaff..316d57e 100755 --- a/deploy/scripts/build.sh +++ b/deploy/scripts/build.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# 建置 gateway + web 依賴 +# 建置 gateway、worker + web 依賴 set -euo pipefail # shellcheck source=_common.sh source "$(cd "$(dirname "$0")" && pwd)/_common.sh" @@ -10,7 +10,7 @@ require_cmd npm log "build backend…" cd "$BACKEND_DIR" go build -o bin/gateway . -go build -o bin/worker ./cmd/worker 2>/dev/null || log "worker build skipped/failed (optional)" +go build -o bin/worker ./cmd/worker log "npm install web…" cd "$WEB_DIR" diff --git a/deploy/scripts/migrate.sh b/deploy/scripts/migrate.sh index 3fd2ef4..e8d8532 100755 --- a/deploy/scripts/migrate.sh +++ b/deploy/scripts/migrate.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Mongo migration(golang-migrate + seed admin) +# Mongo migration(golang-migrate;不建立預設帳號) set -euo pipefail # shellcheck source=_common.sh source "$(cd "$(dirname "$0")" && pwd)/_common.sh" @@ -25,13 +25,11 @@ if [[ -z "${MONGO_URL:-}" ]]; then die "MONGO_URL 未設定" fi -log "migrate up → $MONGO_URL" -# 隱藏密碼於 log:只顯示 host +# Never print MONGO_URL because it commonly contains database credentials. +log "migrate up → configured MongoDB" cd "$BACKEND_DIR" migrate -path generate/database/mongo -database "$MONGO_URL" up migrate -path generate/database/mongo -database "$MONGO_URL" version || true -log "seed admin(migration 000003):" -log " email: 見 generate/database/mongo/000003_seed_admin.up.json(admin@30cm.net / admin123)" -log " 若密碼政策已改為 ≥12 碼,登入後請立刻改密或重設" +log "migration 不建立預設管理員;首次部署請以 runtime secret 執行 seeder" log "migrate done" diff --git a/deploy/scripts/redeploy.sh b/deploy/scripts/redeploy.sh index 722a2b4..c264ce3 100755 --- a/deploy/scripts/redeploy.sh +++ b/deploy/scripts/redeploy.sh @@ -28,7 +28,7 @@ for a in "$@"; do 1) stop apps 2) wipe infra volumes(除非 --keep-data) 3) up-infra - 4) migrate(seed admin) + 4) migrate(不建立預設帳號) 5) build 6) start gateway + vite EOF diff --git a/deploy/scripts/start.sh b/deploy/scripts/start.sh index 4a10e09..c618508 100755 --- a/deploy/scripts/start.sh +++ b/deploy/scripts/start.sh @@ -80,4 +80,4 @@ log " Assets: ${HAIXUN_STORAGE_S3_PUBLIC_BASE_URL:-}/…" log " Mailpit: http://127.0.0.1:${MAILPIT_UI_PORT:-8025}" log " MinIO: http://127.0.0.1:${MINIO_CONSOLE_PORT:-9001}" log " logs: $RUN_DIR/" -log " FE 請切 Live 資料來源;seed: admin@30cm.net / admin123(見 migration)" +log " 管理員帳號不由 migration 建立;首次部署請使用 runtime seeder"