prod: bundle threads-profile scrape release, fix headless-shell install, add reset-data tooling
- release layout now ships scripts/threads-profile/scrape.mjs + node_modules;
activate-release.sh verifies it exists before activating a release
- profile_scrape.go resolves the script path relative to the release dir
(bin/{gateway,worker} -> ../scripts/threads-profile/scrape.mjs) in prod,
falling back to the dev-tree relative paths for local runs
- bootstrap.sh: Playwright 1.49+ needs a separate chromium-headless-shell
build for headless launches; the idempotency check only looked for
chromium-*, so a stalled/partial headless-shell download from an earlier
run was never retried. Now checks and installs both builds.
- add reset-data.sh (local wrapper + remote script) to wipe Mongo/Redis/MinIO
and reseed the admin account for a clean-slate redeploy
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
parent
fb8100825f
commit
d1e876629a
|
|
@ -112,12 +112,23 @@ func defaultFetchProfilePostTexts(ctx context.Context, username, storageStateJSO
|
|||
}
|
||||
|
||||
func findProfileScrapeScript() (string, error) {
|
||||
// relative to process cwd and source file location
|
||||
candidates := []string{
|
||||
var candidates []string
|
||||
// release layout: <release>/bin/{gateway,worker} + <release>/scripts/threads-profile/scrape.mjs
|
||||
if exe, err := os.Executable(); err == nil {
|
||||
if resolved, rerr := filepath.EvalSymlinks(exe); rerr == nil {
|
||||
exe = resolved
|
||||
}
|
||||
releaseDir := filepath.Dir(filepath.Dir(exe)) // <release>/bin/worker -> <release>
|
||||
candidates = append(candidates,
|
||||
filepath.Join(releaseDir, "scripts", "threads-profile", "scrape.mjs"),
|
||||
)
|
||||
}
|
||||
// relative to process cwd and source file location (local dev)
|
||||
candidates = append(candidates,
|
||||
"scripts/threads-profile/scrape.mjs",
|
||||
"apps/backend/scripts/threads-profile/scrape.mjs",
|
||||
filepath.Join("..", "scripts", "threads-profile", "scrape.mjs"),
|
||||
}
|
||||
)
|
||||
// from this source file: internal/module/studio/usecase -> ../../../../scripts/...
|
||||
if _, file, _, ok := runtime.Caller(0); ok {
|
||||
base := filepath.Dir(file)
|
||||
|
|
|
|||
|
|
@ -71,6 +71,32 @@ To deploy an already built artifact:
|
|||
Rollback switches application code only. Never automatically run a down
|
||||
migration; production migrations must remain backward compatible.
|
||||
|
||||
## Threads Profile Scrape (Playwright)
|
||||
|
||||
Persona analysis shells out to `node scripts/threads-profile/scrape.mjs` (a
|
||||
headless Chromium scrape of a public Threads profile). `build-release.sh`
|
||||
bundles the script with its production `node_modules`; the Chromium browser
|
||||
binary itself is installed once on the host by `bootstrap.sh` (cached under
|
||||
`/var/lib/harbor/.cache/ms-playwright`, shared across blue/green releases).
|
||||
|
||||
This host only has 1 vCPU. A scrape briefly saturates the CPU and can slow
|
||||
down other requests while it runs; keep usage infrequent until the box is
|
||||
upsized.
|
||||
|
||||
## Data Reset
|
||||
|
||||
To wipe Mongo, Redis, and MinIO (irreversible; monitoring volumes are untouched):
|
||||
|
||||
```bash
|
||||
./deploy/prod/reset-data.sh --yes
|
||||
```
|
||||
|
||||
This stops both app slots, recreates empty `mongo`/`redis`/`minio` containers using
|
||||
the existing credentials in `harbor.env`, and clears the admin-seeded marker so the
|
||||
next `release.sh` run creates a fresh `admin@haixun.local`. Take a backup first with
|
||||
`backup.sh` if the data might still be worth keeping. Run `release.sh` immediately
|
||||
after to bring the app back up with forward migrations and a reseeded admin.
|
||||
|
||||
## TLS
|
||||
|
||||
The public hostname is Cloudflare-proxied while the origin has a private IP.
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ release_id="harbor-$timestamp-$revision"
|
|||
stage=$(mktemp -d "/tmp/${release_id}.XXXXXX")
|
||||
trap 'rm -rf "$stage"' EXIT
|
||||
|
||||
mkdir -p "$stage/bin" "$stage/web" "$stage/migrations" "$ARTIFACT_DIR"
|
||||
mkdir -p "$stage/bin" "$stage/web" "$stage/migrations" "$stage/scripts/threads-profile" "$ARTIFACT_DIR"
|
||||
|
||||
if [[ ${SKIP_TESTS:-0} != 1 ]]; then
|
||||
(cd "$BACKEND_DIR" && go test ./... && go vet ./...)
|
||||
|
|
@ -37,6 +37,12 @@ printf '%s\n' "building static web"
|
|||
cp -a "$WEB_DIR/dist/." "$stage/web/"
|
||||
cp -a "$BACKEND_DIR/generate/database/mongo/." "$stage/migrations/"
|
||||
|
||||
printf '%s\n' "bundling threads-profile scrape script"
|
||||
(cd "$BACKEND_DIR/scripts/threads-profile" && npm ci --omit=dev)
|
||||
cp -a "$BACKEND_DIR/scripts/threads-profile/scrape.mjs" "$stage/scripts/threads-profile/"
|
||||
cp -a "$BACKEND_DIR/scripts/threads-profile/package.json" "$stage/scripts/threads-profile/"
|
||||
cp -a "$BACKEND_DIR/scripts/threads-profile/node_modules" "$stage/scripts/threads-profile/"
|
||||
|
||||
cat > "$stage/release.txt" <<EOF
|
||||
release=$release_id
|
||||
revision=$revision
|
||||
|
|
|
|||
|
|
@ -48,6 +48,7 @@ test -x "$release_dir/bin/worker"
|
|||
test -x "$release_dir/bin/seeder"
|
||||
test -x "$release_dir/bin/migrate"
|
||||
test -f "$release_dir/web/index.html"
|
||||
test -f "$release_dir/scripts/threads-profile/scrape.mjs"
|
||||
chown -R root:harbor "$release_dir"
|
||||
find "$release_dir" -type d -exec chmod 0755 {} +
|
||||
find "$release_dir" -type f -exec chmod 0644 {} +
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ fi
|
|||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update
|
||||
apt-get install -y ca-certificates certbot curl jq nginx openssl python3-certbot-dns-cloudflare rsync tar gzip ufw fail2ban docker.io docker-compose-v2
|
||||
apt-get install -y ca-certificates certbot curl jq nginx openssl python3-certbot-dns-cloudflare rsync tar gzip ufw fail2ban docker.io docker-compose-v2 nodejs npm
|
||||
systemctl enable --now docker nginx fail2ban
|
||||
|
||||
if ! id harbor >/dev/null 2>&1; then
|
||||
|
|
@ -26,6 +26,20 @@ install -d -m 2770 -o daniel -g harbor /opt/harbor/incoming
|
|||
install -d -m 0750 -o harbor -g harbor /var/lib/harbor
|
||||
install -d -m 0700 /var/backups/harbor
|
||||
install -d -m 0750 -o root -g harbor /etc/harbor
|
||||
|
||||
# threads-profile scrape script needs a real Chromium; keep the browser cache
|
||||
# under the harbor user's home so releases (which only ship node_modules) can
|
||||
# reuse it across blue/green swaps.
|
||||
playwright_version=1.49.1
|
||||
# since Playwright 1.49, headless launches need the separate chromium-headless-shell
|
||||
# build in addition to the regular chromium build; check both so a partial/older
|
||||
# install (or a headless-shell-only download that timed out) doesn't get skipped.
|
||||
if [[ ! -d /var/lib/harbor/.cache/ms-playwright ]] \
|
||||
|| ! find /var/lib/harbor/.cache/ms-playwright -maxdepth 1 -iname 'chromium-*' -print -quit | grep -q . \
|
||||
|| ! find /var/lib/harbor/.cache/ms-playwright -maxdepth 1 -iname 'chromium_headless_shell-*' -print -quit | grep -q .; then
|
||||
npx --yes "playwright@$playwright_version" install-deps chromium
|
||||
runuser -u harbor -- env HOME=/var/lib/harbor npx --yes "playwright@$playwright_version" install chromium chromium-headless-shell
|
||||
fi
|
||||
rsync -a --delete "$SOURCE_DIR/" /opt/harbor/deploy/
|
||||
chown -R root:root /opt/harbor/deploy
|
||||
chmod +x /opt/harbor/deploy/remote/*.sh /opt/harbor/deploy/backup/backup.sh
|
||||
|
|
|
|||
|
|
@ -0,0 +1,42 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
if [[ ${EUID} -ne 0 ]]; then
|
||||
printf '%s\n' "reset-data must run as root" >&2
|
||||
exit 1
|
||||
fi
|
||||
exec 9>/run/lock/harbor-release.lock
|
||||
if ! flock -n 9; then
|
||||
printf '%s\n' "another release or rollback is in progress" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ "${1:-}" != "--yes" ]]; then
|
||||
printf '%s\n' "usage: reset-data.sh --yes" >&2
|
||||
printf '%s\n' "this permanently deletes all Mongo, Redis, and MinIO data" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
compose=(docker compose --env-file /etc/harbor/harbor.env -f /opt/harbor/deploy/compose/docker-compose.yml)
|
||||
|
||||
printf '%s\n' "stopping app slots"
|
||||
for slot in blue green; do
|
||||
systemctl stop "harbor-gateway@$slot.service" 2>/dev/null || true
|
||||
systemctl stop "harbor-worker@$slot.service" 2>/dev/null || true
|
||||
done
|
||||
|
||||
printf '%s\n' "stopping data services"
|
||||
"${compose[@]}" stop mongo redis minio
|
||||
"${compose[@]}" rm -f mongo redis minio
|
||||
|
||||
printf '%s\n' "removing data volumes"
|
||||
docker volume rm haixun-prod_mongo_data haixun-prod_mongo_config haixun-prod_redis_data haixun-prod_minio_data
|
||||
|
||||
printf '%s\n' "recreating empty data services"
|
||||
"${compose[@]}" up -d --wait --wait-timeout 180 mongo redis minio
|
||||
|
||||
rm -f /etc/harbor/admin-seeded /etc/harbor/initial-admin-credentials
|
||||
|
||||
printf '%s\n' "data reset complete: mongo/redis/minio are now empty"
|
||||
printf '%s\n' "monitoring volumes (prometheus/grafana/loki/alertmanager) were left untouched"
|
||||
printf '%s\n' "next: run release.sh to redeploy app code, run forward migrations, and reseed the admin account"
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
TARGET=${DEPLOY_TARGET:-daniel@10.0.0.33}
|
||||
SSH_KEY=${DEPLOY_SSH_KEY:-$HOME/.ssh/harbor_deploy}
|
||||
ssh_opts=(-i "$SSH_KEY" -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes)
|
||||
|
||||
if [[ "${1:-}" != "--yes" ]]; then
|
||||
printf '%s\n' "usage: reset-data.sh --yes" >&2
|
||||
printf '%s\n' "this permanently deletes all Mongo, Redis, and MinIO data on $TARGET" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf '%s\n' "uploading reset-data.sh"
|
||||
rsync -az -e "ssh -i $SSH_KEY -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes" \
|
||||
"$SCRIPT_DIR/remote/reset-data.sh" "$TARGET:/tmp/harbor-reset-data.sh"
|
||||
|
||||
printf '%s\n' "The remote host may request the sudo password once."
|
||||
ssh -t "${ssh_opts[@]}" "$TARGET" \
|
||||
"sudo install -o root -g root -m 0755 /tmp/harbor-reset-data.sh /opt/harbor/deploy/remote/reset-data.sh && rm -f /tmp/harbor-reset-data.sh && sudo /opt/harbor/deploy/remote/reset-data.sh --yes"
|
||||
Loading…
Reference in New Issue