From 8ef6fe470b75c5585ba35a75de5a3dc0e42106cd Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 13 Jul 2026 08:59:13 +0000 Subject: [PATCH] fix frontend --- .gitignore | 2 + apps/backend/Makefile | 48 +- apps/backend/README.md | 30 +- apps/backend/cmd/init/main.go | 55 ++ apps/backend/cmd/seeder/main.go | 70 +++ apps/backend/cmd/worker/main.go | 82 ++- apps/backend/crawler/README.md | 48 ++ apps/backend/crawler/package-lock.json | 547 ++++++++++++++++++ apps/backend/crawler/package.json | 13 + apps/backend/crawler/src/server.ts | 159 +++++ .../{gateway.yaml => gateway.example.yaml} | 42 +- apps/backend/gateway.go | 7 +- apps/backend/generate/api/m5.api | 7 +- .../mongo/000008_worker_indexes.down.json | 6 + .../mongo/000008_worker_indexes.up.json | 28 + apps/backend/internal/config/config.go | 77 ++- apps/backend/internal/handler/routes.go | 10 +- .../internal/lib/redislock/redislock.go | 47 ++ .../logic/auth/o_auth_callback_logic.go | 34 +- .../internal/logic/auth/o_auth_start_logic.go | 25 +- .../internal/logic/scout/run_scan_logic.go | 18 +- apps/backend/internal/middleware/cors.go | 68 ++- apps/backend/internal/middleware/cors_test.go | 42 ++ .../internal/middleware/m1_t119_auth_test.go | 30 +- .../backend/internal/module/job/domain/job.go | 4 + .../internal/module/job/repository/memory.go | 10 +- .../internal/module/job/repository/mongo.go | 13 +- .../internal/module/job/usecase/service.go | 16 + .../internal/module/member/usecase/account.go | 16 +- .../module/member/usecase/auth_test.go | 9 +- .../module/member/usecase/m1_t119_test.go | 22 +- .../internal/module/scout/domain/domain.go | 60 +- .../module/scout/repository/memory.go | 2 +- .../internal/module/scout/repository/mongo.go | 14 +- .../scout/usecase/chrome_crawler_provider.go | 126 ++++ .../scout/usecase/exa_threads_provider.go | 163 ++++++ .../usecase/exa_threads_provider_test.go | 36 ++ .../module/scout/usecase/m5_scout_test.go | 126 +++- .../internal/module/scout/usecase/planner.go | 128 ++++ .../module/scout/usecase/planner_test.go | 120 ++++ .../internal/module/scout/usecase/service.go | 275 ++++++--- .../module/studio/publish/unavailable.go | 24 + .../internal/module/studio/usecase/m4_test.go | 37 ++ .../internal/module/studio/usecase/service.go | 37 ++ .../module/threads/provider/unavailable.go | 33 ++ .../module/threads/usecase/service.go | 3 + .../module/usage/usecase/platform_gate.go | 67 +++ apps/backend/internal/svc/service_context.go | 60 +- apps/backend/internal/types/m5_convert.go | 3 +- apps/backend/internal/types/types.go | 6 + .../haixun-threads-sync/service-worker.js | 13 +- apps/web/.env.example | 3 +- apps/web/src/data/live/m5Repos.ts | 28 +- apps/web/src/data/repos.ts | 6 +- apps/web/src/domain/types.ts | 10 +- apps/web/src/lib/i18n/messages.ts | 50 +- apps/web/src/pages/ScoutPage.tsx | 501 ++++++++-------- apps/web/src/pages/SettingsPage.tsx | 2 + apps/web/src/styles/global.css | 41 +- apps/web/vite.config.ts | 36 +- deploy/nginx/haixun-dev.conf | 2 + 61 files changed, 3005 insertions(+), 592 deletions(-) create mode 100644 apps/backend/cmd/init/main.go create mode 100644 apps/backend/cmd/seeder/main.go create mode 100644 apps/backend/crawler/README.md create mode 100644 apps/backend/crawler/package-lock.json create mode 100644 apps/backend/crawler/package.json create mode 100644 apps/backend/crawler/src/server.ts rename apps/backend/etc/{gateway.yaml => gateway.example.yaml} (71%) create mode 100644 apps/backend/generate/database/mongo/000008_worker_indexes.down.json create mode 100644 apps/backend/generate/database/mongo/000008_worker_indexes.up.json create mode 100644 apps/backend/internal/lib/redislock/redislock.go create mode 100644 apps/backend/internal/middleware/cors_test.go create mode 100644 apps/backend/internal/module/scout/usecase/chrome_crawler_provider.go create mode 100644 apps/backend/internal/module/scout/usecase/exa_threads_provider.go create mode 100644 apps/backend/internal/module/scout/usecase/exa_threads_provider_test.go create mode 100644 apps/backend/internal/module/scout/usecase/planner.go create mode 100644 apps/backend/internal/module/scout/usecase/planner_test.go create mode 100644 apps/backend/internal/module/studio/publish/unavailable.go create mode 100644 apps/backend/internal/module/threads/provider/unavailable.go diff --git a/.gitignore b/.gitignore index 2242334..323e094 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ deploy/.env deploy/.env.dev deploy/.env.prod infra/.env +# Local runtime credentials. Commit the adjacent gateway.example.yaml only. +apps/backend/etc/gateway.yaml old/deploy/.env old/deploy/.env.dev old/deploy/.env.prod diff --git a/apps/backend/Makefile b/apps/backend/Makefile index b787659..834597b 100644 --- a/apps/backend/Makefile +++ b/apps/backend/Makefile @@ -1,10 +1,15 @@ # 只保留會重複打的指令；其餘直接 go / migrate。 -.PHONY: build test test-integration gen-api tidy migrate-up migrate-down +.PHONY: build test test-integration gen-api tidy migrate-up migrate-down init seeder \ + start stop restart crawler-install crawler crawler-start crawler-stop crawler-restart + +RUNTIME_DIR ?= /tmp/haixun build: go build -o bin/gateway . go build -o bin/worker ./cmd/worker + go build -o bin/init ./cmd/init + go build -o bin/seeder ./cmd/seeder test: go test ./... @@ -35,3 +40,44 @@ migrate-up: migrate-down: migrate -path generate/database/mongo -database "$(MONGO_URL)" down 1 + +init: + go run ./cmd/init -f etc/gateway.yaml + +seeder: + go run ./cmd/seeder -f etc/gateway.yaml + +crawler-install: + cd crawler && npm install && npx playwright install --with-deps chromium + +# Requires SCOUT_CRAWLER_TOKEN equal to Scout.CrawlerToken in gateway.yaml. +crawler: + cd crawler && npm run start + +# Local service lifecycle. Logs and PID files stay outside the worktree. +start: build + @mkdir -p "$(RUNTIME_DIR)" + @nohup "$(CURDIR)/bin/gateway" -f "$(CURDIR)/etc/gateway.yaml" >"$(RUNTIME_DIR)/gateway.log" 2>&1 & printf '%s\n' $$! >"$(RUNTIME_DIR)/gateway.pid" + @nohup "$(CURDIR)/bin/worker" -f "$(CURDIR)/etc/gateway.yaml" >"$(RUNTIME_DIR)/worker.log" 2>&1 & printf '%s\n' $$! >"$(RUNTIME_DIR)/worker.pid" + +stop: + @for service in gateway worker; do \ + pidfile="$(RUNTIME_DIR)/$$service.pid"; \ + if [ -f "$$pidfile" ] && kill -0 "$$(cat "$$pidfile")" 2>/dev/null; then kill "$$(cat "$$pidfile")"; fi; \ + rm -f "$$pidfile"; \ + done + +restart: stop start + +# The crawler token must come from the private runtime environment, never this file. +crawler-start: + @test -n "$(SCOUT_CRAWLER_TOKEN)" || (printf '%s\n' 'SCOUT_CRAWLER_TOKEN is required' >&2; exit 1) + @mkdir -p "$(RUNTIME_DIR)" + @cd crawler && SCOUT_CRAWLER_TOKEN="$(SCOUT_CRAWLER_TOKEN)" nohup npm run start >"$(RUNTIME_DIR)/crawler.log" 2>&1 & printf '%s\n' $$! >"$(RUNTIME_DIR)/crawler.pid" + +crawler-stop: + @pidfile="$(RUNTIME_DIR)/crawler.pid"; \ + if [ -f "$$pidfile" ] && kill -0 "$$(cat "$$pidfile")" 2>/dev/null; then kill "$$(cat "$$pidfile")"; fi; \ + rm -f "$$pidfile" + +crawler-restart: crawler-stop crawler-start diff --git a/apps/backend/README.md b/apps/backend/README.md index 138a10a..319e0fb 100644 --- a/apps/backend/README.md +++ b/apps/backend/README.md @@ -7,7 +7,7 @@ Harbor Desk Phase C（go-zero）。 | 做 | 不做 | |----|------| | `generate/api` + `make gen-api` | 手寫 handler／routes | -| `generate/database/mongo` + `make migrate-up` | `cmd/init`、`cmd/seeder` 程式遷移 | +| `generate/database/mongo` + `make migrate-up` | 在 gateway 啟動時隱式建 index／seed | | `monc` + `CacheRedis` | 自己半套 mongo cache | | `internal/logic` + `module/*/domain|repository|usecase` | 業務塞進 handler | @@ -16,20 +16,40 @@ Harbor Desk Phase C（go-zero）。 ```bash cd apps/backend make build # gateway + worker +make start # build 後背景啟動 gateway + worker（PID/log 在 /tmp/haixun） +make stop +make restart make test make gen-api # 改 .api 後 make migrate-up # 改 generate/database 後 +make init # 只建立可重複執行的 operational indexes +make seeder # 讀 etc/gateway.yaml 的 Seed.AdminPassword +make crawler-install # 安裝 dev_mode Chrome crawler 的 Playwright/Chromium +SCOUT_CRAWLER_TOKEN='...' make crawler-start +SCOUT_CRAWLER_TOKEN='...' make crawler-restart make tidy ``` +### Scout Chrome Crawler + +`dev_mode=true` 時，Scout 只走私有 Playwright crawler，沒有同步有效 Chrome session 就會要求使用者先同步。 + +1. 在私有 `etc/gateway.yaml` 填 `Scout.SessionSecret`、`Scout.CrawlerEndpoint`、`Scout.CrawlerToken`。 +2. 執行 `make crawler-install` 一次。 +3. 執行 `SCOUT_CRAWLER_TOKEN='與 Scout.CrawlerToken 相同的值' make crawler-start`。 +4. crawler 必須維持 `127.0.0.1` 私網綁定，禁止公開反向代理。 + +詳見 [`crawler/README.md`](crawler/README.md)。 + ```bash # 需要 Mongo + Redis（go-zero monc / redis） +# 私有設定只放在 etc/gateway.yaml（此檔不會被 git 追蹤）： +# cp etc/gateway.example.yaml etc/gateway.yaml +# 填入 Mongo/Redis/MinIO 密碼與兩組不同的 JWT secret。 # Mongo 若有帳密（本機 infra 常見），一定要帶進 URI： -export MONGO_URI='mongodb://USER:PASS@127.0.0.1:27017/?authSource=admin' -export MONGO_URL="$MONGO_URI" # migrate 用 -make migrate-up +MONGO_URL='mongodb://USER:PASS@127.0.0.1:27017/haixun?authSource=admin' make migrate-up go run . -f etc/gateway.yaml -# admin@haixun.local / admin123（seed，uid=1000000 → 顯示 01000000） +# Seed.AdminPassword 填好後執行 make seeder（uid=1000000 → 顯示 01000000） ``` ### 日誌與密碼（安全） diff --git a/apps/backend/cmd/init/main.go b/apps/backend/cmd/init/main.go new file mode 100644 index 0000000..a1522d2 --- /dev/null +++ b/apps/backend/cmd/init/main.go @@ -0,0 +1,55 @@ +package main + +import ( + "context" + "flag" + "fmt" + "time" + + "apps/backend/internal/config" + + "github.com/zeromicro/go-zero/core/conf" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" +) + +// init creates idempotent operational indexes only. It intentionally never +// inserts members or other business records. +func main() { + configFile := flag.String("f", "etc/gateway.yaml", "config file") + flag.Parse() + + var c config.Config + conf.MustLoad(*configFile, &c) + c.ApplyEnv() + if c.Mongo.URI == "" || c.Mongo.Database == "" { + panic("Mongo.URI and Mongo.Database are required") + } + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(c.Mongo.URI)) + if err != nil { + panic(err) + } + defer client.Disconnect(context.Background()) + + db := client.Database(c.Mongo.Database) + indexes := map[string][]mongo.IndexModel{ + "jobs": { + {Keys: bson.D{{Key: "status", Value: 1}, {Key: "run_after", Value: 1}, {Key: "created_at", Value: 1}}, Options: options.Index().SetName("claim_due_jobs")}, + {Keys: bson.D{{Key: "status", Value: 1}, {Key: "completed_at", Value: 1}}, Options: options.Index().SetName("purge_terminal_jobs")}, + }, + "studio_outbox": { + {Keys: bson.D{{Key: "status", Value: 1}, {Key: "updated_at", Value: 1}}, Options: options.Index().SetName("worker_outbox_status_updated")}, + {Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "updated_at", Value: -1}}, Options: options.Index().SetName("owner_outbox_updated")}, + }, + } + for collection, models := range indexes { + if _, err := db.Collection(collection).Indexes().CreateMany(ctx, models); err != nil { + panic(fmt.Errorf("create %s indexes: %w", collection, err)) + } + } + fmt.Println("database indexes initialized") +} diff --git a/apps/backend/cmd/seeder/main.go b/apps/backend/cmd/seeder/main.go new file mode 100644 index 0000000..8a71cfa --- /dev/null +++ b/apps/backend/cmd/seeder/main.go @@ -0,0 +1,70 @@ +package main + +import ( + "context" + "flag" + "fmt" + "strings" + "time" + + "apps/backend/internal/config" + + "github.com/zeromicro/go-zero/core/conf" + "go.mongodb.org/mongo-driver/bson" + "go.mongodb.org/mongo-driver/mongo" + "go.mongodb.org/mongo-driver/mongo/options" + "golang.org/x/crypto/bcrypt" +) + +// seeder is intentionally explicit: a deployment must provide its own admin +// password rather than receiving a committed default credential. +func main() { + configFile := flag.String("f", "etc/gateway.yaml", "config file") + flag.Parse() + + var c config.Config + conf.MustLoad(*configFile, &c) + c.ApplyEnv() + password := c.Seed.AdminPassword + if len(password) < 12 || strings.Contains(password, "REPLACE_WITH") { + panic("Seed.AdminPassword in gateway.yaml must be at least 12 characters") + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + client, err := mongo.Connect(ctx, options.Client().ApplyURI(c.Mongo.URI)) + if err != nil { + panic(err) + } + defer client.Disconnect(context.Background()) + + hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + if err != nil { + panic(err) + } + now := time.Now().UTC().UnixNano() + admin := bson.M{ + "uid": int64(1_000_000), + "tenant_id": "default", + "email": "admin@haixun.local", + "display_name": "Harbor Admin", + "roles": []string{"admin", "member"}, + "status": "active", + "email_verified": true, + "email_verified_at": now, + "invite_code": "SEEDADMIN", + "password_hash": string(hash), + "timezone": "Asia/Taipei", + "created_at": now, + "updated_at": now, + } + db := client.Database(c.Mongo.Database) + _, err = db.Collection("members").UpdateOne(ctx, bson.M{"uid": admin["uid"]}, bson.M{"$setOnInsert": admin}, options.Update().SetUpsert(true)) + if err != nil { + panic(err) + } + _, err = db.Collection("counters").UpdateOne(ctx, bson.M{"_id": "member_uid"}, bson.M{"$setOnInsert": bson.M{"seq": int64(1_000_000)}}, options.Update().SetUpsert(true)) + if err != nil { + panic(err) + } + fmt.Printf("seeded admin %s\n", strings.TrimSpace(admin["email"].(string))) +} diff --git a/apps/backend/cmd/worker/main.go b/apps/backend/cmd/worker/main.go index d20ae6a..7d4b909 100644 --- a/apps/backend/cmd/worker/main.go +++ b/apps/backend/cmd/worker/main.go @@ -12,6 +12,7 @@ import ( "time" "apps/backend/internal/config" + redislock "apps/backend/internal/lib/redislock" "apps/backend/internal/module/ai" appnotifRepo "apps/backend/internal/module/appnotif/repository" appnotifUC "apps/backend/internal/module/appnotif/usecase" @@ -23,6 +24,7 @@ import ( jobUC "apps/backend/internal/module/job/usecase" memberDomain "apps/backend/internal/module/member/domain" memberRepo "apps/backend/internal/module/member/repository" + scoutDomain "apps/backend/internal/module/scout/domain" scoutRepo "apps/backend/internal/module/scout/repository" scoutUC "apps/backend/internal/module/scout/usecase" studioPublish "apps/backend/internal/module/studio/publish" @@ -51,10 +53,17 @@ func main() { var c config.Config conf.MustLoad(*configFile, &c) c.ApplyEnv() + if err := c.ValidateProductionDependencies(); err != nil { + logx.Must(err) + } workerID := c.Worker.ID - if workerID == "" || workerID == "worker-1" || workerID == "worker-local-1" { - workerID = "demo-worker-1" + if workerID == "" || workerID == "worker-1" || workerID == "worker-local-1" || workerID == "demo-worker-1" { + host, _ := os.Hostname() + if host == "" { + host = "worker" + } + workerID = fmt.Sprintf("%s-%d", host, os.Getpid()) } interval := time.Duration(c.Worker.PollIntervalMs) * time.Millisecond if interval <= 0 { @@ -65,19 +74,15 @@ func main() { jobs := jobUC.New(jobRepo.NewMonStore(c.Mongo.URI, c.Mongo.Database)) jobs.Notifier = appN - secret := c.Auth.AccessSecret - if secret == "" { - secret = "dev-token-secret" - } var provider threadsDomain.Provider if strings.TrimSpace(c.Platform.ThreadsAppId) != "" && strings.TrimSpace(c.Platform.ThreadsAppSecret) != "" { provider = threadsProv.NewMeta(c.Platform.ThreadsAppId, c.Platform.ThreadsAppSecret) logx.Info("worker threads: meta provider") } else { - provider = &threadsProv.FakeProvider{} - logx.Info("worker threads: fake provider") + provider = threadsProv.NewUnavailable("set THREADS_APP_ID and THREADS_APP_SECRET") + logx.Info("worker threads: Threads operations disabled until credentials are configured") } - threadsSvc := threadsUC.New(threadsRepo.NewMonStore(c.Mongo.URI, c.Mongo.Database), provider, secret) + threadsSvc := threadsUC.New(threadsRepo.NewMonStore(c.Mongo.URI, c.Mongo.Database), provider, c.Auth.AccessSecret) threadsSvc.Renew = jobs // usage + AI keys（與 gateway 對齊，persona LLM 需真 key） @@ -89,21 +94,15 @@ func main() { PlatformOpenCode: c.Platform.OpenCodeKey, PlatformExa: c.Platform.ExaKey, } - if keyRes.PlatformAI == "" && keyRes.PlatformXAI == "" && keyRes.PlatformOpenCode == "" { - keyRes.PlatformAI = "fake-platform-ai" - logx.Info("worker usage: no platform AI keys — fake-platform-ai for quota only") - } usageSvc := usageUC.New(usageRepo.NewMonStore(c.Mongo.URI, c.Mongo.Database), keyRes) aiRegistry := ai.NewRegistry() - // Outbox／scout 發文：有 Meta 憑證則真 Threads Graph，否則 Fake（單元測試用） var pub studioPublish.Transport if strings.TrimSpace(c.Platform.ThreadsAppId) != "" && strings.TrimSpace(c.Platform.ThreadsAppSecret) != "" { pub = studioPublish.NewMeta("https://graph.threads.net") logx.Info("worker studio: meta publish transport (real Threads)") } else { - pub = studioPublish.NewFake() - logx.Info("worker studio: fake publish transport (no Threads app credentials)") + pub = studioPublish.NewUnavailable("set THREADS_APP_ID and THREADS_APP_SECRET") } studio := studioUC.New(studioRepo.NewMonStore(c.Mongo.URI, c.Mongo.Database), pub) studio.Accounts = threadsSvc @@ -118,8 +117,9 @@ func main() { scoutSvc := scoutUC.New(scoutRepo.NewMonStore(c.Mongo.URI, c.Mongo.Database)) scoutSvc.Settings = &workerDevMode{Members: members} - scoutSvc.Transport = pub - scoutSvc.Accounts = threadsSvc + scoutSvc.Provider = scoutUC.NewExaThreadsProvider(c.Platform.ExaKey) + scoutSvc.SessionSecret = c.Scout.SessionSecret + scoutSvc.Crawler = scoutUC.NewHTTPCrawlerProvider(c.Scout.CrawlerEndpoint, c.Scout.CrawlerToken) studio.Crawler = scoutSvc // worker 執行 job 本體，不經 API 再入列 studio.Jobs = nil @@ -133,6 +133,7 @@ func main() { defer tick.Stop() ctx := context.Background() + outboxLock := redislock.New(c.CacheRedis[0].RedisConf, "haixun:worker:outbox", workerID, 60*time.Second) for { select { case <-sig: @@ -183,17 +184,31 @@ func main() { } else { logx.Infof("worker %s play_generate_script %s ok", workerID, j.ID) } + case jobDomain.TemplateScoutScan: + if err := runScoutScan(ctx, jobs, scoutSvc, j); err != nil { + logx.Errorf("worker %s scout scan %s failed: %v", workerID, j.ID, err) + _, _ = jobs.FailJob(ctx, j.ID, err.Error()) + } default: logx.Infof("worker %s unknown template %s job %s — fail", workerID, j.TemplateType, j.ID) _, _ = jobs.FailJob(ctx, j.ID, "unknown template: "+j.TemplateType) } } - // 2) outbox due steps - n, err := studio.ProcessDueSteps(ctx, 0) - if err != nil { - logx.Errorf("worker %s outbox tick: %v", workerID, err) - } else if n > 0 { - logx.Infof("worker %s outbox published %d step(s)", workerID, n) + // 2) One worker owns an outbox tick. The Redis value is workerID and + // release is compare-and-delete, so a departing worker cannot unlock a + // successor's lease. + if locked, lerr := outboxLock.Acquire(ctx); lerr != nil { + logx.Errorf("worker %s outbox lock: %v", workerID, lerr) + } else if locked { + n, err := studio.ProcessDueSteps(ctx, 0) + if rerr := outboxLock.Release(ctx); rerr != nil { + logx.Errorf("worker %s outbox unlock: %v", workerID, rerr) + } + if err != nil { + logx.Errorf("worker %s outbox tick: %v", workerID, err) + } else if n > 0 { + logx.Infof("worker %s outbox published %d step(s)", workerID, n) + } } // 3) 終態任務超過 2 天自動清除 if purged, err := jobs.PurgeExpiredTerminal(ctx); err != nil { @@ -205,6 +220,25 @@ func main() { } } +func runScoutScan(ctx context.Context, jobs *jobUC.Service, scout *scoutUC.Service, j *jobDomain.Job) error { + var brief scoutDomain.RunBrief + if err := json.Unmarshal([]byte(j.Payload), &brief); err != nil { + return fmt.Errorf("invalid scout scan payload: %w", err) + } + if _, err := jobs.MarkRunningProgress(ctx, j.ID, 15, "海巡 · 準備搜尋來源"); err != nil { + return err + } + posts, err := scout.RunScanFromBrief(ctx, j.OwnerUID, &brief) + if err != nil { + return err + } + if _, err := jobs.MarkRunningProgress(ctx, j.ID, 90, fmt.Sprintf("海巡 · 已寫入 %d 筆候選", len(posts))); err != nil { + return err + } + _, err = jobs.SucceedJob(ctx, j.ID, fmt.Sprintf("海巡完成 · 命中 %d 筆", len(posts))) + return err +} + func runTokenRenew(ctx context.Context, jobs *jobUC.Service, threads *threadsUC.Service, j *jobDomain.Job) error { if j.RefID == "" { return fmt.Errorf("缺少帳號 ref_id") diff --git a/apps/backend/crawler/README.md b/apps/backend/crawler/README.md new file mode 100644 index 0000000..d2790db --- /dev/null +++ b/apps/backend/crawler/README.md @@ -0,0 +1,48 @@ +# Scout Chrome Crawler + +Private Playwright service for Scout `dev_mode=true`. It is not a public API. +It binds to `127.0.0.1` only, requires a bearer token, and receives decrypted +Chrome storage state only from the backend worker/gateway process. + +## Install + +```bash +cd apps/backend/crawler +npm install +npx playwright install --with-deps chromium +``` + +## Configure + +Set the same private token in the local-only `apps/backend/etc/gateway.yaml`: + +```yaml +Scout: + SessionSecret: "a dedicated long random secret" + CrawlerEndpoint: "http://127.0.0.1:8891" + CrawlerToken: "a different long random token" +``` + +Start the crawler with the matching token: + +```bash +cd apps/backend/crawler +SCOUT_CRAWLER_TOKEN='the CrawlerToken value' npm run start +``` + +Optional: `SCOUT_CRAWLER_PORT=8891` changes the local port. + +## Runtime Rules + +- Never expose this port through nginx, Docker host publishing, or the public internet. +- Never place `CrawlerToken`, `SessionSecret`, or storage state in a tracked file or log. +- `dev_mode=false` does not use this service; it uses the configured API provider. +- `dev_mode=true` requires a freshly synchronized Chrome session. Missing or expired sessions fail with a user-actionable error. +- The service supports `/v1/threads/search` and `/v1/threads/resolve`; both require `Authorization: Bearer `. + +## Operations + +1. Start Mongo, Redis, gateway, worker, and this crawler service. +2. In Harbor Desk settings, enable dev mode and synchronize Chrome Threads session. +3. Start a Scout run. The worker chooses crawler mode and contacts only `CrawlerEndpoint`. +4. If Threads redirects to login, sync the session again. No fallback to API occurs in dev mode. diff --git a/apps/backend/crawler/package-lock.json b/apps/backend/crawler/package-lock.json new file mode 100644 index 0000000..ae3255b --- /dev/null +++ b/apps/backend/crawler/package-lock.json @@ -0,0 +1,547 @@ +{ + "name": "@harbor/scout-crawler", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@harbor/scout-crawler", + "dependencies": { + "playwright": "^1.58.0", + "tsx": "^4.20.6" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx": { + "version": "4.23.1", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.1.tgz", + "integrity": "sha512-GQHnkIfxyx1wYCOS/wonik5MVRZU9hi1TEZmzGZSCJB1y9YgoZ8H6itNE/u4suE+yLmOzuE4E5S4TZ/ZX2wcWQ==", + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + } + } +} diff --git a/apps/backend/crawler/package.json b/apps/backend/crawler/package.json new file mode 100644 index 0000000..39692b4 --- /dev/null +++ b/apps/backend/crawler/package.json @@ -0,0 +1,13 @@ +{ + "name": "@harbor/scout-crawler", + "private": true, + "type": "module", + "scripts": { + "dev": "tsx src/server.ts", + "start": "tsx src/server.ts" + }, + "dependencies": { + "playwright": "^1.58.0", + "tsx": "^4.20.6" + } +} diff --git a/apps/backend/crawler/src/server.ts b/apps/backend/crawler/src/server.ts new file mode 100644 index 0000000..30de428 --- /dev/null +++ b/apps/backend/crawler/src/server.ts @@ -0,0 +1,159 @@ +import { chromium, type Page } from "playwright"; +import { createServer } from "node:http"; + +type SearchRequest = { storage_state?: string; terms?: string[]; limit?: number }; +type ResolveRequest = { storage_state?: string; permalink?: string }; +type Post = { permalink: string; author: string; text: string }; + +const port = Number(process.env.SCOUT_CRAWLER_PORT || 8891); +const token = process.env.SCOUT_CRAWLER_TOKEN || ""; + +function isThreadsURL(value: string): boolean { + try { + const host = new URL(value).hostname.toLowerCase(); + return host === "threads.com" || host.endsWith(".threads.com") || host === "threads.net" || host.endsWith(".threads.net"); + } catch { + return false; + } +} + +async function readPosts(page: Page, limit: number): Promise { + const posts = new Map(); + const links = page.locator('a[href*="/post/"]'); + const count = Math.min(await links.count(), 50); + for (let i = 0; i < count && posts.size < limit; i++) { + const link = links.nth(i); + const href = await link.getAttribute("href").catch(() => null); + if (!href) continue; + const permalink = href.startsWith("http") ? href : `https://www.threads.com${href}`; + if (!isThreadsURL(permalink)) continue; + const author = href.match(/@([^/]+)\/post/)?.[1] || ""; + const scope = link.locator("xpath=ancestor::div[position()<=6]").first(); + const text = (await scope.innerText().catch(() => "")).trim(); + if (text.length < 5) continue; + posts.set(permalink, { permalink, author, text: text.slice(0, 2000) }); + } + return [...posts.values()]; +} + +async function search(storageState: string, terms: string[], limit: number): Promise { + const state = JSON.parse(storageState) as { cookies?: unknown[] }; + if (!Array.isArray(state.cookies) || state.cookies.length === 0) throw new Error("crawler session is invalid"); + const browser = await chromium.launch({ headless: true }); + try { + const context = await browser.newContext({ storageState: state, locale: "zh-TW", timezoneId: "Asia/Taipei" }); + const page = await context.newPage(); + const query = terms.filter(Boolean).join(" ").slice(0, 180); + await page.goto(`https://www.threads.com/search?q=${encodeURIComponent(query)}&serp_type=default`, { waitUntil: "domcontentloaded", timeout: 45_000 }); + const body = await page.locator("body").innerText().catch(() => ""); + if (page.url().includes("/login") || body.includes("登入")) throw new Error("crawler session expired"); + await page.waitForSelector('a[href*="/post/"]', { timeout: 12_000 }).catch(() => undefined); + await page.mouse.wheel(0, 900); + await page.waitForTimeout(1000); + const posts = await readPosts(page, limit); + await context.close(); + return posts; + } finally { + await browser.close(); + } +} + +function shortcodeFromPermalink(permalink: string): string { + return permalink.match(/\/post\/([^/?]+)/)?.[1] || ""; +} + +function findMediaID(value: unknown, shortcode: string): string { + if (!value || typeof value !== "object") return ""; + if (Array.isArray(value)) { for (const item of value) { const id = findMediaID(item, shortcode); if (id) return id; } return ""; } + const record = value as Record; + const code = String(record.code || record.shortcode || ""); + const id = String(record.id || record.pk || record.media_id || ""); + if (code === shortcode && /^\d{10,}$/.test(id)) return id; + for (const child of Object.values(record)) { const found = findMediaID(child, shortcode); if (found) return found; } + return ""; +} + +function findMediaIDInText(raw: string, shortcode: string): string { + const escaped = shortcode.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const patterns = [ + new RegExp(`(?:code|shortcode)["']?\\s*[:=]\\s*["']${escaped}["'][\\s\\S]{0,800}?(?:id|pk|media_id)["']?\\s*[:=]\\s*["']?(\\d{10,})`, "i"), + new RegExp(`(?:id|pk|media_id)["']?\\s*[:=]\\s*["']?(\\d{10,})[\\s\\S]{0,800}?(?:code|shortcode)["']?\\s*[:=]\\s*["']${escaped}["']`, "i"), + ]; + for (const pattern of patterns) { + const match = raw.match(pattern); + if (match?.[1]) return match[1]; + } + return ""; +} + +async function resolve(storageState: string, permalink: string): Promise { + if (!isThreadsURL(permalink)) throw new Error("invalid Threads permalink"); + const state = JSON.parse(storageState) as { cookies?: unknown[] }; + if (!Array.isArray(state.cookies) || state.cookies.length === 0) throw new Error("crawler session is invalid"); + const shortcode = shortcodeFromPermalink(permalink); + if (!shortcode) throw new Error("permalink has no Threads shortcode"); + const browser = await chromium.launch({ headless: true }); + try { + const context = await browser.newContext({ storageState: state, locale: "zh-TW", timezoneId: "Asia/Taipei" }); + const page = await context.newPage(); + let mediaID = ""; + const responseReads: Promise[] = []; + page.on("response", async (response) => { + if (mediaID || !/graphql|threads|instagram/.test(response.url())) return; + responseReads.push((async () => { + try { + const raw = await response.text(); + if (!mediaID) { + try { mediaID = findMediaID(JSON.parse(raw), shortcode); } catch { /* non-JSON response */ } + } + if (!mediaID) mediaID = findMediaIDInText(raw, shortcode); + } catch { /* ignored */ } + })()); + }); + await page.goto(permalink, { waitUntil: "domcontentloaded", timeout: 45_000 }); + const body = await page.locator("body").innerText().catch(() => ""); + if (page.url().includes("/login") || body.includes("登入")) throw new Error("crawler session expired"); + await page.waitForTimeout(2500); + await Promise.allSettled(responseReads); + if (!mediaID) { + mediaID = findMediaIDInText(await page.content(), shortcode); + } + await context.close(); + if (!mediaID) throw new Error("Threads media ID could not be resolved") + return mediaID; + } finally { await browser.close(); } +} + +if (!token) throw new Error("SCOUT_CRAWLER_TOKEN is required"); + +createServer(async (req, res) => { + const path = new URL(req.url || "/", "http://127.0.0.1").pathname; + if (req.method !== "POST" || (path !== "/v1/threads/search" && path !== "/v1/threads/resolve")) { + res.writeHead(404).end(); + return; + } + if (req.headers.authorization !== `Bearer ${token}`) { + res.writeHead(401).end(); + return; + } + const body = await new Promise((resolve, reject) => { + let raw = ""; + req.setEncoding("utf8"); + req.on("data", (chunk) => { raw += chunk; if (raw.length > 300_000) req.destroy(); }); + req.on("end", () => resolve(raw)); + req.on("error", reject); + }); + try { + if (path === "/v1/threads/resolve") { + const input = JSON.parse(body) as ResolveRequest; + const mediaID = await resolve(String(input.storage_state || ""), String(input.permalink || "")); + res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ media_id: mediaID })); + } else { + const input = JSON.parse(body) as SearchRequest; + const posts = await search(String(input.storage_state || ""), Array.isArray(input.terms) ? input.terms.slice(0, 12) : [], Math.min(Math.max(Number(input.limit) || 10, 1), 30)); + res.writeHead(200, { "content-type": "application/json" }).end(JSON.stringify({ posts })); + } + } catch (error) { + res.writeHead(422, { "content-type": "application/json" }).end(JSON.stringify({ error: error instanceof Error ? error.message : "crawler failed" })); + } +}).listen(port, "127.0.0.1"); diff --git a/apps/backend/etc/gateway.yaml b/apps/backend/etc/gateway.example.yaml similarity index 71% rename from apps/backend/etc/gateway.yaml rename to apps/backend/etc/gateway.example.yaml index 09007bc..087a908 100644 --- a/apps/backend/etc/gateway.yaml +++ b/apps/backend/etc/gateway.example.yaml @@ -1,5 +1,5 @@ -# 巡樓 gateway — 唯一主設定檔（A-02） -# Secret 請用環境變數覆寫。 +# 巡樓 gateway — 可提交的設定範本。 +# 複製為 gateway.yaml 後填入私有密鑰；gateway.yaml 已被 .gitignore 排除。 Name: haixun-gateway Host: 0.0.0.0 Port: 8888 @@ -28,34 +28,46 @@ Mongo: Database: "haixun" Auth: - AccessSecret: "change-me-access-secret" + # 至少 32 bytes，且不得與 RefreshSecret 相同。 + AccessSecret: "REPLACE_WITH_A_LONG_RANDOM_ACCESS_SECRET" AccessExpire: 86400 - RefreshSecret: "change-me-refresh-secret" + RefreshSecret: "REPLACE_WITH_A_DIFFERENT_LONG_RANDOM_REFRESH_SECRET" RefreshExpire: 604800 # 平台預設 key（所有會員可用；會員在設定頁填的 BYOK 優先） # 也可用環境變數覆寫：PLATFORM_XAI_KEY / PLATFORM_OPENCODE_KEY / PLATFORM_EXA_KEY / PLATFORM_AI_KEY Platform: # xAI（api.x.ai）— 人設 LLM、文案等 - XAIKey: "xai-TtmPWVAz1CdssptRetj9iOFSdSG2LZDffEOTKzIiDmVSDqUL62lVdDladlQIXAwoeHtowWraVqYuHzl1" + XAIKey: "" # 相容舊名：XAIKey 空時會用 AIKey AIKey: "" # OpenCode Go（opencode.ai/zen/go） - OpenCodeKey: "sk-sJkJVG2Q3tQEGqgl4cCy10QCVnrHpvuzR8RuE2eTuvruXiVNromsQ7zo6db0Ssj1" + OpenCodeKey: "" # Exa 網搜 - ExaKey: "5d9af9b2-a847-4295-a110-f666049e1073" - ThreadsAppId: "2733369680379930" - ThreadsAppSecret: "31db4afdd8e348d75d76dee8fb9a42d3" + ExaKey: "" + ThreadsAppId: "" + ThreadsAppSecret: "" + +Scout: + # Dedicated encryption key for Chrome storage state; do not reuse JWT keys. + SessionSecret: "REPLACE_WITH_A_DEDICATED_SCOUT_SESSION_SECRET" + # Private Playwright crawler service, called only by cmd/worker in dev_mode. + CrawlerEndpoint: "" + CrawlerToken: "" Worker: - # demo-worker 預設 id（cmd/worker 若見 worker-local-1 會改成 demo-worker-1） - ID: "demo-worker-1" + # 留空時 worker 會用 hostname + pid 產生唯一 owner id；多 worker 不可共用固定值。 + ID: "" PollIntervalMs: 2000 +# 僅 cmd/seeder 使用，gateway/worker 不會讀取或寫入 log。 +Seed: + AdminPassword: "REPLACE_WITH_A_STRONG_SEED_ADMIN_PASSWORD" + Bcrypt: Cost: 10 -# OAuth（空則 callback 走 mock / 開發路徑） +# OAuth（未設定 provider 時相關 API 明確拒絕，不提供 mock 路徑） OAuth: GoogleClientID: "" LineClientID: "" @@ -88,6 +100,12 @@ Mail: # 前端 origin：重設信連結 + 預設 logo URL（/brand-mark.jpg） # 遠端 dev：https://threads-tool-dev.30cm.net PublicWebBase: "https://threads-tool-dev.30cm.net" +# 瀏覽器直連 gateway 時的 allowlist；同源 proxy 不需要列在此處。 +# CORS_ALLOWED_ORIGINS 可用逗號附加額外的明確 origin，不能使用 *。 +CORSAllowedOrigins: + - "https://threads-tool-dev.30cm.net" + - "http://127.0.0.1:5173" + - "http://localhost:5173" # 郵件品牌（hermes 模板；LogoURL 空 = PublicWebBase/brand-mark.jpg） Brand: diff --git a/apps/backend/gateway.go b/apps/backend/gateway.go index b30ca93..d347adc 100644 --- a/apps/backend/gateway.go +++ b/apps/backend/gateway.go @@ -27,15 +27,18 @@ func main() { var c config.Config conf.MustLoad(*configFile, &c) c.ApplyEnv() + if err := c.ValidateProductionDependencies(); err != nil { + logx.Must(err) + } // 禁止 Verbose（否則可能印 request body 含 password） c.RestConf.Verbose = false server := rest.MustNewServer(c.RestConf, - rest.WithNotAllowedHandler(middleware.NewCorsMiddleware().Handler()), + rest.WithNotAllowedHandler(middleware.NewCorsMiddleware(c.CORSAllowedOrigins).Handler()), ) defer server.Stop() - server.Use(middleware.NewCorsMiddleware().Handle) + server.Use(middleware.NewCorsMiddleware(c.CORSAllowedOrigins).Handle) ctx := svc.NewServiceContext(c) handler.RegisterHandlers(server, ctx) diff --git a/apps/backend/generate/api/m5.api b/apps/backend/generate/api/m5.api index 9064346..d437bad 100644 --- a/apps/backend/generate/api/m5.api +++ b/apps/backend/generate/api/m5.api @@ -241,6 +241,9 @@ type ( ScoutScanReq { Brief ScoutBriefPublic `json:"brief"` } + ScoutScanJobData { + Job JobPublic `json:"job"` + } ScoutPostPublic { Id string `json:"id"` BrandId string `json:"brand_id,optional"` @@ -259,6 +262,8 @@ type ( ThemeKey string `json:"theme_key,optional"` ThemeLabel string `json:"theme_label,optional"` ScanPath string `json:"scan_path,optional"` + Classification string `json:"classification,optional"` + Permalink string `json:"permalink,optional"` CreatedAt int64 `json:"created_at"` } ScoutPostListData { @@ -422,7 +427,7 @@ service gateway { post /brief (ScoutBriefReq) returns (ScoutBriefPublic) @handler RunScan - post /scan (ScoutScanReq) returns (ScoutPostListData) + post /scan (ScoutScanReq) returns (ScoutScanJobData) @handler ListScoutPosts get /posts (ScoutPostListReq) returns (ScoutPostListData) diff --git a/apps/backend/generate/database/mongo/000008_worker_indexes.down.json b/apps/backend/generate/database/mongo/000008_worker_indexes.down.json new file mode 100644 index 0000000..a460d64 --- /dev/null +++ b/apps/backend/generate/database/mongo/000008_worker_indexes.down.json @@ -0,0 +1,6 @@ +[ + { "dropIndexes": "jobs", "index": "claim_due_jobs" }, + { "dropIndexes": "jobs", "index": "purge_terminal_jobs" }, + { "dropIndexes": "studio_outbox", "index": "worker_outbox_status_updated" }, + { "dropIndexes": "studio_outbox", "index": "owner_outbox_updated" } +] diff --git a/apps/backend/generate/database/mongo/000008_worker_indexes.up.json b/apps/backend/generate/database/mongo/000008_worker_indexes.up.json new file mode 100644 index 0000000..0c9b531 --- /dev/null +++ b/apps/backend/generate/database/mongo/000008_worker_indexes.up.json @@ -0,0 +1,28 @@ +[ + { + "createIndexes": "jobs", + "indexes": [ + { + "key": { "status": 1, "run_after": 1, "created_at": 1 }, + "name": "claim_due_jobs" + }, + { + "key": { "status": 1, "completed_at": 1 }, + "name": "purge_terminal_jobs" + } + ] + }, + { + "createIndexes": "studio_outbox", + "indexes": [ + { + "key": { "status": 1, "updated_at": 1 }, + "name": "worker_outbox_status_updated" + }, + { + "key": { "owner_uid": 1, "updated_at": -1 }, + "name": "owner_outbox_updated" + } + ] + } +] diff --git a/apps/backend/internal/config/config.go b/apps/backend/internal/config/config.go index 86d0b03..70c09ab 100644 --- a/apps/backend/internal/config/config.go +++ b/apps/backend/internal/config/config.go @@ -1,6 +1,7 @@ package config import ( + "fmt" "os" "strings" @@ -26,7 +27,9 @@ type Config struct { RefreshSecret string RefreshExpire int64 } - Platform struct { + // CORSAllowedOrigins is an explicit browser-origin allowlist. + CORSAllowedOrigins []string `json:",optional"` + Platform struct { // AIKey — legacy alias for XAIKey AIKey string `json:",optional"` // XAIKey — platform BYOK for xAI (api.x.ai) @@ -37,15 +40,25 @@ type Config struct { ThreadsAppId string `json:",optional"` ThreadsAppSecret string `json:",optional"` } + // Scout keeps browser-session credentials separate from JWT and public APIs. + Scout struct { + SessionSecret string `json:",optional"` + CrawlerEndpoint string `json:",optional"` + CrawlerToken string `json:",optional"` + } Worker struct { ID string `json:",default=worker-1"` PollIntervalMs int `json:",default=2000"` } + // Seed is used only by cmd/seeder; gateway and worker never read or log it. + Seed struct { + AdminPassword string `json:",optional"` + } // Bcrypt cost (stand-alone Bcrypt.Cost) Bcrypt struct { Cost int `json:",default=10"` } - // OAuth third-party (optional; empty → mock path in callback) + // OAuth third-party. Empty provider config disables that provider. OAuth struct { GoogleClientID string `json:",optional"` LineClientID string `json:",optional"` @@ -183,6 +196,9 @@ func (c *Config) ApplyEnv() { if v := firstEnv("PUBLIC_API_BASE", "HAIXUN_PUBLIC_API_BASE"); v != "" { c.PublicAPIBase = strings.TrimRight(v, "/") } + if v := os.Getenv("CORS_ALLOWED_ORIGINS"); v != "" { + c.CORSAllowedOrigins = appendUnique(c.CORSAllowedOrigins, splitCSV(v)) + } if v := os.Getenv("MAIL_BRAND_NAME"); v != "" { c.Brand.Name = v } @@ -218,6 +234,35 @@ func (c *Config) ApplyEnv() { } } +// ValidateProductionDependencies prevents a successful-looking deployment that +// silently falls back to test credentials or fake external integrations. +func (c Config) ValidateProductionDependencies() error { + if invalidSecret(c.Auth.AccessSecret) { + return fmt.Errorf("AUTH_ACCESS_SECRET must be at least 32 bytes") + } + if invalidSecret(c.Auth.RefreshSecret) { + return fmt.Errorf("AUTH_REFRESH_SECRET must be at least 32 bytes") + } + if c.Auth.AccessSecret == c.Auth.RefreshSecret { + return fmt.Errorf("AUTH_ACCESS_SECRET and AUTH_REFRESH_SECRET must differ") + } + if len(c.CacheRedis) == 0 || strings.TrimSpace(c.CacheRedis[0].Host) == "" { + return fmt.Errorf("CacheRedis is required") + } + if strings.TrimSpace(c.Mongo.URI) == "" || strings.TrimSpace(c.Mongo.Database) == "" { + return fmt.Errorf("Mongo.URI and Mongo.Database are required") + } + if len(c.CORSAllowedOrigins) == 0 { + return fmt.Errorf("CORSAllowedOrigins is required") + } + return nil +} + +func invalidSecret(value string) bool { + value = strings.TrimSpace(value) + return len(value) < 32 || strings.Contains(value, "REPLACE_WITH") || strings.Contains(value, "CHANGE_ME") +} + func firstEnv(keys ...string) string { for _, k := range keys { if v := os.Getenv(k); v != "" { @@ -227,6 +272,34 @@ func firstEnv(keys ...string) string { return "" } +func appendUnique(existing, values []string) []string { + seen := make(map[string]struct{}, len(existing)+len(values)) + out := make([]string, 0, len(existing)+len(values)) + for _, value := range append(existing, values...) { + value = strings.TrimSpace(value) + if value == "" { + continue + } + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + out = append(out, value) + } + return out +} + +func splitCSV(v string) []string { + parts := strings.Split(v, ",") + out := make([]string, 0, len(parts)) + for _, part := range parts { + if part = strings.TrimSpace(part); part != "" { + out = append(out, strings.TrimRight(part, "/")) + } + } + return out +} + func parsePort(v string) int { var p int for _, ch := range v { diff --git a/apps/backend/internal/handler/routes.go b/apps/backend/internal/handler/routes.go index 9cc482b..650d6ec 100644 --- a/apps/backend/internal/handler/routes.go +++ b/apps/backend/internal/handler/routes.go @@ -401,12 +401,11 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { []rest.Route{ { Method: http.MethodPost, - Path: "/generate-image", - Handler: media.GenerateImageHandler(serverCtx), + Path: "/upload", + Handler: media.UploadHandler(serverCtx), }, }..., ), - rest.WithJwt(serverCtx.Config.Auth.AccessSecret), rest.WithPrefix("/api/v1/media"), ) @@ -416,11 +415,12 @@ func RegisterHandlers(server *rest.Server, serverCtx *svc.ServiceContext) { []rest.Route{ { Method: http.MethodPost, - Path: "/upload", - Handler: media.UploadHandler(serverCtx), + Path: "/generate-image", + Handler: media.GenerateImageHandler(serverCtx), }, }..., ), + rest.WithJwt(serverCtx.Config.Auth.AccessSecret), rest.WithPrefix("/api/v1/media"), ) diff --git a/apps/backend/internal/lib/redislock/redislock.go b/apps/backend/internal/lib/redislock/redislock.go new file mode 100644 index 0000000..125522f --- /dev/null +++ b/apps/backend/internal/lib/redislock/redislock.go @@ -0,0 +1,47 @@ +package redislock + +import ( + "context" + "fmt" + "time" + + "github.com/zeromicro/go-zero/core/stores/redis" +) + +// Lock is a small Redis lease. Value must uniquely identify the worker that +// owns it; Release uses a Lua compare-and-delete to avoid deleting a renewed +// lease owned by another worker. +type Lock struct { + client *redis.Redis + key string + owner string + ttl time.Duration +} + +func New(conf redis.RedisConf, key, owner string, ttl time.Duration) *Lock { + return &Lock{client: redis.MustNewRedis(conf), key: key, owner: owner, ttl: ttl} +} + +func (l *Lock) Acquire(ctx context.Context) (bool, error) { + if l == nil || l.client == nil || l.key == "" || l.owner == "" { + return false, fmt.Errorf("invalid redis lock") + } + seconds := int(l.ttl.Seconds()) + if seconds < 1 { + seconds = 1 + } + return l.client.SetnxExCtx(ctx, l.key, l.owner, seconds) +} + +func (l *Lock) Release(ctx context.Context) error { + if l == nil || l.client == nil { + return fmt.Errorf("invalid redis lock") + } + _, err := l.client.EvalCtx(ctx, ` +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +`, []string{l.key}, l.owner) + return err +} diff --git a/apps/backend/internal/logic/auth/o_auth_callback_logic.go b/apps/backend/internal/logic/auth/o_auth_callback_logic.go index d51fb62..fcb8cdb 100644 --- a/apps/backend/internal/logic/auth/o_auth_callback_logic.go +++ b/apps/backend/internal/logic/auth/o_auth_callback_logic.go @@ -34,27 +34,12 @@ func (l *OAuthCallbackLogic) OAuthCallback(req *types.AuthOAuthCallbackReq) (res if token == "" { token = req.IdToken } - if token == "" { - token = req.Code - } - // allow mock: code or token "mock:sub:email" if token == "" { return nil, domain.ErrOAuthFailed } - if !strings.HasPrefix(token, "mock:") && req.Code != "" && strings.HasPrefix(req.Code, "mock:") { - token = req.Code - } subject, email, name, err = l.svcCtx.Auth.VerifyGoogleIDToken(l.ctx, token) if err != nil { - // dev fallback when Google not configured: treat code as mock subject - if req.Code != "" { - subject = provider + "_" + req.Code - email = subject + "@oauth.local" - name = provider + " user" - err = nil - } else { - return nil, err - } + return nil, err } case domain.PlatformLine: code := req.Code @@ -63,23 +48,10 @@ func (l *OAuthCallbackLogic) OAuthCallback(req *types.AuthOAuthCallbackReq) (res } subject, name, email, err = l.svcCtx.Auth.LineCodeToProfile(l.ctx, code, req.RedirectUri) if err != nil { - if code != "" { - subject = provider + "_" + code - name = provider + " user" - err = nil - } else { - return nil, err - } + return nil, err } default: - // generic mock oauth - code := req.Code - if code == "" { - code = "anon" - } - subject = provider + "_" + code - email = subject + "@oauth.local" - name = provider + " user" + return nil, domain.ErrOAuthFailed } m, pair, err := l.svcCtx.Auth.LoginOAuth(l.ctx, provider, subject, email, name) diff --git a/apps/backend/internal/logic/auth/o_auth_start_logic.go b/apps/backend/internal/logic/auth/o_auth_start_logic.go index 793dbef..fc29607 100644 --- a/apps/backend/internal/logic/auth/o_auth_start_logic.go +++ b/apps/backend/internal/logic/auth/o_auth_start_logic.go @@ -2,6 +2,8 @@ package auth import ( "context" + "fmt" + "strings" "apps/backend/internal/svc" "apps/backend/internal/types" @@ -20,13 +22,24 @@ func NewOAuthStartLogic(ctx context.Context, svcCtx *svc.ServiceContext) *OAuthS } func (l *OAuthStartLogic) OAuthStart(req *types.AuthOAuthProviderPath) (resp *types.AuthOAuthStartData, err error) { - p := req.Provider + p := strings.ToLower(strings.TrimSpace(req.Provider)) if p == "" { p = "google" } - return &types.AuthOAuthStartData{ - AuthorizeUrl: "https://example.com/oauth/" + p + "/authorize?state=dev", - Provider: p, - State: "dev", - }, nil + // The current API accepts a verified provider credential in callback; it + // does not yet expose a state-consuming browser callback, so returning a + // fabricated authorization URL would be an authentication bypass. + switch p { + case "google": + if strings.TrimSpace(l.svcCtx.Config.OAuth.GoogleClientID) == "" { + return nil, fmt.Errorf("google OAuth is not configured") + } + case "line": + if strings.TrimSpace(l.svcCtx.Config.OAuth.LineClientID) == "" || strings.TrimSpace(l.svcCtx.Config.OAuth.LineClientSecret) == "" { + return nil, fmt.Errorf("LINE OAuth is not configured") + } + default: + return nil, fmt.Errorf("unsupported OAuth provider: %s", p) + } + return nil, fmt.Errorf("interactive OAuth start is unavailable until Redis-backed state and PKCE callback are configured") } diff --git a/apps/backend/internal/logic/scout/run_scan_logic.go b/apps/backend/internal/logic/scout/run_scan_logic.go index 62f67e1..e5b007f 100644 --- a/apps/backend/internal/logic/scout/run_scan_logic.go +++ b/apps/backend/internal/logic/scout/run_scan_logic.go @@ -2,6 +2,7 @@ package scout import ( "context" + "encoding/json" "apps/backend/internal/middleware" "apps/backend/internal/response" @@ -21,23 +22,24 @@ func NewRunScanLogic(ctx context.Context, svcCtx *svc.ServiceContext) *RunScanLo return &RunScanLogic{Logger: logx.WithContext(ctx), ctx: ctx, svcCtx: svcCtx} } -func (l *RunScanLogic) RunScan(req *types.ScoutScanReq) (*types.ScoutPostListData, error) { - if l.svcCtx.Scout == nil { - return nil, response.Biz(503, 503001, "scout not configured") +func (l *RunScanLogic) RunScan(req *types.ScoutScanReq) (*types.ScoutScanJobData, error) { + if l.svcCtx.Jobs == nil { + return nil, response.Biz(503, 503001, "jobs module not configured") } uid, ok := middleware.UIDFrom(l.ctx) if !ok { return nil, response.Biz(401, 401001, "missing authorization") } - list, err := l.svcCtx.Scout.RunScanFromBrief(l.ctx, uid, types.BriefToDomain(&req.Brief)) + brief := types.BriefToDomain(&req.Brief) + payload, err := json.Marshal(brief) if err != nil { return nil, err } - out := make([]types.ScoutPostPublic, 0, len(list)) - for _, p := range list { - out = append(out, types.ScoutPostFromDomain(p)) + j, err := l.svcCtx.Jobs.ScheduleScoutScan(l.ctx, uid, brief.ThemeKey, string(payload)) + if err != nil { + return nil, err } - return &types.ScoutPostListData{List: out}, nil + return &types.ScoutScanJobData{Job: types.JobFromModel(j)}, nil } diff --git a/apps/backend/internal/middleware/cors.go b/apps/backend/internal/middleware/cors.go index ebdcd68..bd452ab 100644 --- a/apps/backend/internal/middleware/cors.go +++ b/apps/backend/internal/middleware/cors.go @@ -1,19 +1,35 @@ package middleware -import "net/http" +import ( + "encoding/json" + "net/http" + "strings" +) // CorsMiddleware — browser CORS for Harbor Desk FE. // Note: cannot combine Access-Control-Allow-Origin: * with Allow-Credentials: true // (browsers reject). FE uses Bearer token (no cookies), so no credentials header. -type CorsMiddleware struct{} +type CorsMiddleware struct { + allowedOrigins map[string]struct{} +} -func NewCorsMiddleware() *CorsMiddleware { - return &CorsMiddleware{} +func NewCorsMiddleware(origins []string) *CorsMiddleware { + allowed := make(map[string]struct{}, len(origins)) + for _, origin := range origins { + origin = strings.TrimRight(strings.TrimSpace(origin), "/") + if origin != "" { + allowed[origin] = struct{}{} + } + } + return &CorsMiddleware{allowedOrigins: allowed} } func (m *CorsMiddleware) Handle(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - setHeader(w, r) + if !m.setHeader(w, r) { + writeOriginDenied(w) + return + } if r.Method == http.MethodOptions { w.WriteHeader(http.StatusNoContent) return @@ -24,7 +40,10 @@ func (m *CorsMiddleware) Handle(next http.HandlerFunc) http.HandlerFunc { func (m *CorsMiddleware) Handler() http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - setHeader(w, r) + if !m.setHeader(w, r) { + writeOriginDenied(w) + return + } if r.Method == http.MethodOptions { w.WriteHeader(http.StatusNoContent) return @@ -33,16 +52,37 @@ func (m *CorsMiddleware) Handler() http.Handler { }) } -func setHeader(w http.ResponseWriter, r *http.Request) { - origin := r.Header.Get("Origin") - if origin == "" { - origin = "*" +func writeOriginDenied(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + _ = json.NewEncoder(w).Encode(map[string]any{ + "code": 403003, "message": "origin is not allowed", "data": nil, "error": nil, + }) +} + +func (m *CorsMiddleware) setHeader(w http.ResponseWriter, r *http.Request) bool { + origin := strings.TrimRight(strings.TrimSpace(r.Header.Get("Origin")), "/") + if origin != "" { + if _, ok := m.allowedOrigins[origin]; !ok { + return false + } + w.Header().Set("Access-Control-Allow-Origin", origin) + appendVary(w, "Origin") } - // Echo request origin (dev-friendly). Production can whitelist later. - w.Header().Set("Access-Control-Allow-Origin", origin) - w.Header().Set("Vary", "Origin") w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-Request-Id, device_id") w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS, PATCH") + w.Header().Set("Access-Control-Max-Age", "600") w.Header().Set("Access-Control-Expose-Headers", "Content-Length, Content-Type") - // Do NOT set Allow-Credentials with wildcard; FE uses Authorization header only. + return true +} + +func appendVary(w http.ResponseWriter, value string) { + for _, existing := range w.Header().Values("Vary") { + for _, part := range strings.Split(existing, ",") { + if strings.EqualFold(strings.TrimSpace(part), value) { + return + } + } + } + w.Header().Add("Vary", value) } diff --git a/apps/backend/internal/middleware/cors_test.go b/apps/backend/internal/middleware/cors_test.go new file mode 100644 index 0000000..daff973 --- /dev/null +++ b/apps/backend/internal/middleware/cors_test.go @@ -0,0 +1,42 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestCorsAllowsConfiguredOriginPreflight(t *testing.T) { + m := NewCorsMiddleware([]string{"http://10.0.0.9:5173"}) + r := httptest.NewRequest(http.MethodOptions, "/api/v1/settings/ai", nil) + r.Header.Set("Origin", "http://10.0.0.9:5173") + w := httptest.NewRecorder() + + m.Handle(func(http.ResponseWriter, *http.Request) { t.Fatal("OPTIONS must not reach next handler") })(w, r) + + if w.Code != http.StatusNoContent { + t.Fatalf("status = %d, want %d", w.Code, http.StatusNoContent) + } + if got := w.Header().Get("Access-Control-Allow-Origin"); got != "http://10.0.0.9:5173" { + t.Fatalf("allow origin = %q", got) + } + if got := w.Header().Get("Access-Control-Max-Age"); got != "600" { + t.Fatalf("max age = %q", got) + } +} + +func TestCorsRejectsUnknownOriginWithEnvelope(t *testing.T) { + m := NewCorsMiddleware([]string{"http://10.0.0.9:5173"}) + r := httptest.NewRequest(http.MethodPut, "/api/v1/settings/ai", nil) + r.Header.Set("Origin", "http://untrusted.example") + w := httptest.NewRecorder() + + m.Handle(func(http.ResponseWriter, *http.Request) { t.Fatal("rejected origin reached next handler") })(w, r) + + if w.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d", w.Code, http.StatusForbidden) + } + if got := w.Header().Get("Content-Type"); got != "application/json" { + t.Fatalf("content type = %q", got) + } +} diff --git a/apps/backend/internal/middleware/m1_t119_auth_test.go b/apps/backend/internal/middleware/m1_t119_auth_test.go index 90a1c4c..4ffe948 100644 --- a/apps/backend/internal/middleware/m1_t119_auth_test.go +++ b/apps/backend/internal/middleware/m1_t119_auth_test.go @@ -32,15 +32,23 @@ func (m *mwRepo) FindByUID(_ context.Context, uid int64) (*domain.Member, error) return &cp, nil } -func (m *mwRepo) NextUID(context.Context) (int64, error) { return 0, nil } -func (m *mwRepo) CreateMember(context.Context, *domain.Member) error { return nil } -func (m *mwRepo) FindByEmail(context.Context, string) (*domain.Member, error) { return nil, domain.ErrNotFound } -func (m *mwRepo) FindByPhone(context.Context, string) (*domain.Member, error) { return nil, domain.ErrNotFound } -func (m *mwRepo) UpdateMember(context.Context, *domain.Member) error { return nil } +func (m *mwRepo) NextUID(context.Context) (int64, error) { return 0, nil } +func (m *mwRepo) CreateMember(context.Context, *domain.Member) error { return nil } +func (m *mwRepo) FindByEmail(context.Context, string) (*domain.Member, error) { + return nil, domain.ErrNotFound +} +func (m *mwRepo) FindByPhone(context.Context, string) (*domain.Member, error) { + return nil, domain.ErrNotFound +} +func (m *mwRepo) FindByInviteCode(context.Context, string) (*domain.Member, error) { + return nil, domain.ErrNotFound +} +func (m *mwRepo) UpdateMember(context.Context, *domain.Member) error { return nil } func (m *mwRepo) ListPage(context.Context, int, int, string, string) ([]*domain.Member, int64, error) { return nil, 0, nil } -func (m *mwRepo) CountActiveAdmins(context.Context) (int64, error) { return 0, nil } +func (m *mwRepo) ListAllMembers(context.Context) ([]*domain.Member, error) { return nil, nil } +func (m *mwRepo) CountActiveAdmins(context.Context) (int64, error) { return 0, nil } func (m *mwRepo) CreateIdentity(context.Context, *domain.Identity) error { return nil } @@ -55,11 +63,11 @@ func (m *mwRepo) UpdateIdentityPassword(context.Context, string, string, string) return nil } func (m *mwRepo) SetCode(string, string, string, time.Duration) {} -func (m *mwRepo) CheckCode(string, string, string) bool { return false } -func (m *mwRepo) PeekCode(string, string, string) bool { return false } -func (m *mwRepo) SaveRefresh(string, int64, time.Time) {} -func (m *mwRepo) ConsumeRefresh(string) (int64, bool) { return 0, false } -func (m *mwRepo) RevokeRefresh(string) {} +func (m *mwRepo) CheckCode(string, string, string) bool { return false } +func (m *mwRepo) PeekCode(string, string, string) bool { return false } +func (m *mwRepo) SaveRefresh(string, int64, time.Time) {} +func (m *mwRepo) ConsumeRefresh(string) (int64, bool) { return 0, false } +func (m *mwRepo) RevokeRefresh(string) {} func (m *mwRepo) RevokeAllRefreshByUID(context.Context, int64) error { return nil } diff --git a/apps/backend/internal/module/job/domain/job.go b/apps/backend/internal/module/job/domain/job.go index c5aecaf..3d138c3 100644 --- a/apps/backend/internal/module/job/domain/job.go +++ b/apps/backend/internal/module/job/domain/job.go @@ -20,6 +20,7 @@ const ( TemplateComposeMimic = "compose_mimic" // TemplatePlayGenerateScript — 互回／串場：一次產完整劇本（背景 job，避免 HTTP 卡住） TemplatePlayGenerateScript = "play_generate_script" + TemplateScoutScan = "scout_scan" ) // Job 生命週期契約（所有模板必須遵守，worker / API 入列時）： @@ -53,6 +54,9 @@ type Job struct { CreatedAt int64 `bson:"created_at" json:"created_at"` UpdatedAt int64 `bson:"updated_at" json:"updated_at"` CompletedAt int64 `bson:"completed_at,omitempty" json:"completed_at,omitempty"` + // Version is incremented by every persisted mutation and makes lifecycle + // writes compare-and-swap safe across gateway and worker instances. + Version int64 `bson:"version" json:"version"` } // DefaultTokenRenewDelayNs — 連帳後約 30 天做第一次 renew（長效 token ~60 天） diff --git a/apps/backend/internal/module/job/repository/memory.go b/apps/backend/internal/module/job/repository/memory.go index b47bd7b..600545f 100644 --- a/apps/backend/internal/module/job/repository/memory.go +++ b/apps/backend/internal/module/job/repository/memory.go @@ -27,11 +27,17 @@ func (s *MemoryStore) Insert(_ context.Context, j *domain.Job) error { func (s *MemoryStore) Update(_ context.Context, j *domain.Job) error { s.mu.Lock() defer s.mu.Unlock() - if _, ok := s.byID[j.ID]; !ok { + stored, ok := s.byID[j.ID] + if !ok { return domain.ErrNotFound } + if stored.Version != j.Version { + return domain.ErrIllegalStatus + } cp := *j + cp.Version++ s.byID[j.ID] = &cp + j.Version = cp.Version return nil } @@ -131,6 +137,7 @@ func (s *MemoryStore) ClaimNext(_ context.Context, workerID string) (*domain.Job best.Status = to best.WorkerID = workerID best.UpdatedAt = now + best.Version++ if best.ProgressSummary == "" { best.ProgressSummary = "已由 worker 領取（" + workerID + "）" } @@ -154,6 +161,7 @@ func (s *MemoryStore) CancelPendingByRef(_ context.Context, ownerUID int64, temp j.ProgressSummary = "已由新的定期排程取代" j.CompletedAt = now j.UpdatedAt = now + j.Version++ } return nil } diff --git a/apps/backend/internal/module/job/repository/mongo.go b/apps/backend/internal/module/job/repository/mongo.go index 0f66c9b..14b2514 100644 --- a/apps/backend/internal/module/job/repository/mongo.go +++ b/apps/backend/internal/module/job/repository/mongo.go @@ -28,14 +28,21 @@ func (s *MonStore) Insert(ctx context.Context, j *domain.Job) error { } func (s *MonStore) Update(ctx context.Context, j *domain.Job) error { + if j == nil { + return domain.ErrNotFound + } + expectedVersion := j.Version j.UpdatedAt = domain.NowNano() - res, err := s.jobs.ReplaceOne(ctx, bson.M{"_id": j.ID}, j) + candidate := *j + candidate.Version = expectedVersion + 1 + res, err := s.jobs.ReplaceOne(ctx, bson.M{"_id": j.ID, "version": expectedVersion}, &candidate) if err != nil { return err } if res.MatchedCount == 0 { - return domain.ErrNotFound + return domain.ErrIllegalStatus } + j.Version = candidate.Version return nil } @@ -127,7 +134,7 @@ func (s *MonStore) ClaimNext(ctx context.Context, workerID string) (*domain.Job, "worker_id": workerID, "updated_at": now, "progress_summary": "已由 worker 領取（" + workerID + "）", - }} + }, "$inc": bson.M{"version": 1}} opts := options.FindOneAndUpdate(). SetSort(bson.D{{Key: "run_after", Value: 1}, {Key: "created_at", Value: 1}}). SetReturnDocument(options.After) diff --git a/apps/backend/internal/module/job/usecase/service.go b/apps/backend/internal/module/job/usecase/service.go index b28dbcb..ce4b273 100644 --- a/apps/backend/internal/module/job/usecase/service.go +++ b/apps/backend/internal/module/job/usecase/service.go @@ -48,6 +48,22 @@ func (s *Service) StartDemo(ctx context.Context, ownerUID int64) (*domain.Job, e return j, nil } +// ScheduleScoutScan enqueues an immutable Scout brief for the worker. +func (s *Service) ScheduleScoutScan(ctx context.Context, ownerUID int64, themeKey, payload string) (*domain.Job, error) { + if strings.TrimSpace(payload) == "" { + return nil, fmt.Errorf("scout scan payload required") + } + now := domain.NowNano() + j := &domain.Job{ID: uuid.NewString(), OwnerUID: ownerUID, TemplateType: domain.TemplateScoutScan, + Status: domain.StatusQueued, RefID: themeKey, Payload: payload, + ProgressSummary: "海巡已排程 · 等待 worker", CreatedAt: now, UpdatedAt: now} + if err := s.Repo.Insert(ctx, j); err != nil { + return nil, err + } + s.notify(ctx, j) + return j, nil +} + func (s *Service) List(ctx context.Context, ownerUID int64) ([]*domain.Job, error) { return s.Repo.ListByOwner(ctx, ownerUID) } diff --git a/apps/backend/internal/module/member/usecase/account.go b/apps/backend/internal/module/member/usecase/account.go index f435577..64182d4 100644 --- a/apps/backend/internal/module/member/usecase/account.go +++ b/apps/backend/internal/module/member/usecase/account.go @@ -575,19 +575,9 @@ func (s *AccountService) VerifyPlatformAuthResult(ctx context.Context, loginID, return CheckPasswordHash(password, ident.PasswordHash), nil } -// ---------- OAuth providers (real when configured; else ErrOAuthFailed) ---------- +// ---------- OAuth providers ---------- func (s *AccountService) VerifyGoogleIDToken(ctx context.Context, idToken string) (subject, email, name string, err error) { - // Dev/mock: accept "mock::" when no Google client configured - if strings.HasPrefix(idToken, "mock:") { - parts := strings.SplitN(idToken, ":", 3) - if len(parts) >= 3 { - return parts[1], parts[2], parts[1], nil - } - if len(parts) == 2 { - return parts[1], parts[1] + "@oauth.local", parts[1], nil - } - } info, e := googleUserInfo(ctx, idToken) if e != nil { return "", "", "", domain.ErrOAuthFailed @@ -596,10 +586,6 @@ func (s *AccountService) VerifyGoogleIDToken(ctx context.Context, idToken string } func (s *AccountService) LineCodeToProfile(ctx context.Context, code, redirectURI string) (subject, displayName, email string, err error) { - if strings.HasPrefix(code, "mock:") { - sub := strings.TrimPrefix(code, "mock:") - return sub, "line " + sub, "", nil - } if s.LineClientID == "" || s.LineClientSecret == "" { return "", "", "", domain.ErrOAuthFailed } diff --git a/apps/backend/internal/module/member/usecase/auth_test.go b/apps/backend/internal/module/member/usecase/auth_test.go index 3b39f01..d66ffe7 100644 --- a/apps/backend/internal/module/member/usecase/auth_test.go +++ b/apps/backend/internal/module/member/usecase/auth_test.go @@ -370,17 +370,14 @@ func TestBind_Unbind(t *testing.T) { require.Error(t, err) } -func TestLoginOAuth_Mock(t *testing.T) { +func TestLoginOAuth_TrustedProviderIdentity(t *testing.T) { store := newFake() svc := usecase.NewAuthService(store, tokenUC.NewJWTIssuer("a", "b", 3600, 86400), 10) - sub, email, name, err := svc.VerifyGoogleIDToken(context.Background(), "mock:g123:u@g.com") - require.NoError(t, err) - require.Equal(t, "g123", sub) - m, pair, err := svc.LoginOAuth(context.Background(), domain.PlatformGoogle, sub, email, name) + m, pair, err := svc.LoginOAuth(context.Background(), domain.PlatformGoogle, "g123", "u@g.com", "G123") require.NoError(t, err) require.NotEmpty(t, pair.AccessToken) require.True(t, m.UID >= domain.MinMemberUID) - m2, _, err := svc.LoginOAuth(context.Background(), domain.PlatformGoogle, sub, email, name) + m2, _, err := svc.LoginOAuth(context.Background(), domain.PlatformGoogle, "g123", "u@g.com", "G123") require.NoError(t, err) require.Equal(t, m.UID, m2.UID) } diff --git a/apps/backend/internal/module/member/usecase/m1_t119_test.go b/apps/backend/internal/module/member/usecase/m1_t119_test.go index 112d47f..d2872a1 100644 --- a/apps/backend/internal/module/member/usecase/m1_t119_test.go +++ b/apps/backend/internal/module/member/usecase/m1_t119_test.go @@ -319,23 +319,17 @@ func TestAP_08_AvatarURLSetAndClear(t *testing.T) { require.Empty(t, out2.AvatarURL) } -// ---------- AO Third-party OAuth (service / mock provider) ---------- +// ---------- AO Third-party OAuth (verified provider identity handed to service) ---------- -func TestAO_01_OAuthProviderStartMock(t *testing.T) { - // Provider mock path via VerifyGoogleIDToken mock: prefix (API-level start is logic stub) +func TestAO_01_OAuthProviderRejectsUnverifiedToken(t *testing.T) { _, svc := newM1Svc() - sub, email, name, err := svc.VerifyGoogleIDToken(context.Background(), "mock:ao01:ao01@g.com") - require.NoError(t, err) - require.Equal(t, "ao01", sub) - require.Equal(t, "ao01@g.com", email) - require.NotEmpty(t, name) + _, _, _, err := svc.VerifyGoogleIDToken(context.Background(), "mock:ao01:ao01@g.com") + require.ErrorIs(t, err, domain.ErrOAuthFailed) } func TestAO_02_OAuthFirstLoginCreatesMember(t *testing.T) { _, svc := newM1Svc() - sub, email, name, err := svc.VerifyGoogleIDToken(context.Background(), "mock:ao02sub:ao02@g.com") - require.NoError(t, err) - m, pair, err := svc.LoginOAuth(context.Background(), domain.PlatformGoogle, sub, email, name) + m, pair, err := svc.LoginOAuth(context.Background(), domain.PlatformGoogle, "ao02sub", "ao02@g.com", "AO2") require.NoError(t, err) require.True(t, m.UID >= domain.MinMemberUID) require.NotEmpty(t, pair.AccessToken) @@ -343,11 +337,9 @@ func TestAO_02_OAuthFirstLoginCreatesMember(t *testing.T) { func TestAO_03_OAuthSecondLoginSameUID(t *testing.T) { _, svc := newM1Svc() - sub, email, name, err := svc.VerifyGoogleIDToken(context.Background(), "mock:ao03sub:ao03@g.com") + m1, _, err := svc.LoginOAuth(context.Background(), domain.PlatformGoogle, "ao03sub", "ao03@g.com", "AO3") require.NoError(t, err) - m1, _, err := svc.LoginOAuth(context.Background(), domain.PlatformGoogle, sub, email, name) - require.NoError(t, err) - m2, _, err := svc.LoginOAuth(context.Background(), domain.PlatformGoogle, sub, email, name) + m2, _, err := svc.LoginOAuth(context.Background(), domain.PlatformGoogle, "ao03sub", "ao03@g.com", "AO3") require.NoError(t, err) require.Equal(t, m1.UID, m2.UID) } diff --git a/apps/backend/internal/module/scout/domain/domain.go b/apps/backend/internal/module/scout/domain/domain.go index 9a5b8d4..14db36e 100644 --- a/apps/backend/internal/module/scout/domain/domain.go +++ b/apps/backend/internal/module/scout/domain/domain.go @@ -6,12 +6,12 @@ import ( ) var ( - ErrNotFound = errors.New("scout not found") - ErrForbidden = errors.New("scout forbidden") - ErrValidation = errors.New("scout validation") - ErrNoCrawlerSession = errors.New("crawler session required when dev_mode enabled") - ErrTopicRemoved = errors.New("ScoutTopic CRUD removed") - ErrHasProducts = errors.New("brand has products; remove products first") + ErrNotFound = errors.New("scout not found") + ErrForbidden = errors.New("scout forbidden") + ErrValidation = errors.New("scout validation") + ErrNoCrawlerSession = errors.New("crawler session required when dev_mode enabled") + ErrTopicRemoved = errors.New("ScoutTopic CRUD removed") + ErrHasProducts = errors.New("brand has products; remove products first") ) func NowNano() int64 { return time.Now().UTC().UnixNano() } @@ -24,10 +24,19 @@ const ( OutreachDrafted = "drafted" OutreachPublished = "published" OutreachSkipped = "skipped" + OutreachQueued = "queued" ModeProduct = "product" ModeTheme = "theme" ModeActivity = "activity" + + ClassificationSeekingHelp = "seeking_help" + ClassificationSeekingRecommendation = "seeking_recommendation" + ClassificationProviderOffer = "provider_offer" + ClassificationDiscussion = "discussion" + ClassificationAsking = "asking" + ClassificationAnnouncement = "announcement" + ClassificationNoise = "noise" ) type Brand struct { @@ -61,24 +70,26 @@ type ActiveBrand struct { } type RunBrief struct { - Intent string `json:"intent"` - Mode string `json:"mode"` - BrandID string `json:"brand_id,omitempty"` - ProductID string `json:"product_id,omitempty"` - ProductLabel string `json:"product_label,omitempty"` - Pains []string `json:"pains"` - Tags []string `json:"tags"` - Periphery []string `json:"periphery"` - ScanTerms []string `json:"scan_terms"` - PlacementNote string `json:"placement_note,omitempty"` - ResponseStance string `json:"response_stance,omitempty"` - ThemeKey string `json:"theme_key,omitempty"` - ThemeLabel string `json:"theme_label,omitempty"` - ProductContext string `json:"product_context,omitempty"` + Intent string `json:"intent"` + Mode string `json:"mode"` + BrandID string `json:"brand_id,omitempty"` + ProductID string `json:"product_id,omitempty"` + ProductLabel string `json:"product_label,omitempty"` + Pains []string `json:"pains"` + Tags []string `json:"tags"` + Periphery []string `json:"periphery"` + ScanTerms []string `json:"scan_terms"` + PlacementNote string `json:"placement_note,omitempty"` + ResponseStance string `json:"response_stance,omitempty"` + ThemeKey string `json:"theme_key,omitempty"` + ThemeLabel string `json:"theme_label,omitempty"` + ProductContext string `json:"product_context,omitempty"` } type Post struct { ID string `bson:"_id" json:"id"` + ExternalID string `bson:"external_id,omitempty" json:"external_id,omitempty"` + Permalink string `bson:"permalink,omitempty" json:"permalink,omitempty"` OwnerUID int64 `bson:"owner_uid" json:"owner_uid"` BrandID string `bson:"brand_id,omitempty" json:"brand_id,omitempty"` Author string `bson:"author" json:"author"` @@ -87,7 +98,9 @@ type Post struct { Opportunity string `bson:"opportunity" json:"opportunity"` OutreachStatus string `bson:"outreach_status" json:"outreach_status"` DraftText string `bson:"draft_text,omitempty" json:"draft_text,omitempty"` + OutboxID string `bson:"outbox_id,omitempty" json:"outbox_id,omitempty"` Score int `bson:"score" json:"score"` + Classification string `bson:"classification,omitempty" json:"classification,omitempty"` MatchedProductID string `bson:"matched_product_id,omitempty" json:"matched_product_id,omitempty"` MatchedProductLabel string `bson:"matched_product_label,omitempty" json:"matched_product_label,omitempty"` MatchReason string `bson:"match_reason,omitempty" json:"match_reason,omitempty"` @@ -109,9 +122,10 @@ type Homework struct { } type CrawlerSession struct { - OwnerUID int64 `bson:"_id" json:"owner_uid"` - Token string `bson:"token" json:"token"` - UpdatedAt int64 `bson:"updated_at" json:"updated_at"` + OwnerUID int64 `bson:"_id" json:"owner_uid"` + StorageStateEnc string `bson:"storage_state_enc" json:"storage_state_enc"` + UpdatedAt int64 `bson:"updated_at" json:"updated_at"` + ExpiresAt int64 `bson:"expires_at" json:"expires_at"` } type ImportDraft struct { diff --git a/apps/backend/internal/module/scout/repository/memory.go b/apps/backend/internal/module/scout/repository/memory.go index 010b446..e1c6496 100644 --- a/apps/backend/internal/module/scout/repository/memory.go +++ b/apps/backend/internal/module/scout/repository/memory.go @@ -255,7 +255,7 @@ func (s *MemoryStore) GetCrawlerSession(_ context.Context, ownerUID int64) (*dom s.mu.Lock() defer s.mu.Unlock() c, ok := s.crawler[ownerUID] - if !ok || c.Token == "" { + if !ok || c.StorageStateEnc == "" { return nil, domain.ErrNotFound } cp := *c diff --git a/apps/backend/internal/module/scout/repository/mongo.go b/apps/backend/internal/module/scout/repository/mongo.go index 1a7e001..ae41c7b 100644 --- a/apps/backend/internal/module/scout/repository/mongo.go +++ b/apps/backend/internal/module/scout/repository/mongo.go @@ -23,12 +23,12 @@ type MonStore struct { func NewMonStore(uri, database string) *MonStore { uri = libmongo.MustMongoURI(uri) return &MonStore{ - brands: mon.MustNewModel(uri, database, "scout_brands"), + brands: mon.MustNewModel(uri, database, "scout_brands"), products: mon.MustNewModel(uri, database, "scout_products"), - active: mon.MustNewModel(uri, database, "scout_active_brand"), - posts: mon.MustNewModel(uri, database, "scout_posts"), - hw: mon.MustNewModel(uri, database, "scout_homework"), - crawler: mon.MustNewModel(uri, database, "scout_crawler_session"), + active: mon.MustNewModel(uri, database, "scout_active_brand"), + posts: mon.MustNewModel(uri, database, "scout_posts"), + hw: mon.MustNewModel(uri, database, "scout_homework"), + crawler: mon.MustNewModel(uri, database, "scout_crawler_session"), } } @@ -166,7 +166,7 @@ func (s *MonStore) SaveHomework(ctx context.Context, h *domain.Homework) error { } func (s *MonStore) GetHomework(ctx context.Context, ownerUID int64, themeKey string) (*domain.Homework, error) { type hwDoc struct { - ID string `bson:"_id"` + ID string `bson:"_id"` domain.Homework `bson:",inline"` } var doc hwDoc @@ -205,7 +205,7 @@ func (s *MonStore) GetCrawlerSession(ctx context.Context, ownerUID int64) (*doma } return nil, err } - if c.Token == "" { + if c.StorageStateEnc == "" { return nil, domain.ErrNotFound } return &c, nil diff --git a/apps/backend/internal/module/scout/usecase/chrome_crawler_provider.go b/apps/backend/internal/module/scout/usecase/chrome_crawler_provider.go new file mode 100644 index 0000000..cf9f309 --- /dev/null +++ b/apps/backend/internal/module/scout/usecase/chrome_crawler_provider.go @@ -0,0 +1,126 @@ +package usecase + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// ChromeCrawlerProvider is the private worker-to-browser boundary. The +// browser service receives decrypted state only for this request. +type ChromeCrawlerProvider interface { + SearchChrome(ctx context.Context, storageState string, terms []string, limit int) ([]ThreadSearchResult, error) + ResolveMediaID(ctx context.Context, storageState, permalink string) (string, error) +} + +func (p *HTTPCrawlerProvider) ResolveMediaID(ctx context.Context, storageState, permalink string) (string, error) { + if p == nil || p.Endpoint == "" || p.Token == "" { + return "", fmt.Errorf("Chrome crawler is not configured") + } + body, err := json.Marshal(map[string]string{"storage_state": storageState, "permalink": permalink}) + if err != nil { + return "", err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.Endpoint+"/v1/threads/resolve", bytes.NewReader(body)) + if err != nil { + return "", err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+p.Token) + client := p.HTTP + if client == nil { + client = &http.Client{Timeout: 95 * time.Second} + } + res, err := client.Do(req) + if err != nil { + return "", err + } + defer res.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(res.Body, 64<<10)) + if res.StatusCode < 200 || res.StatusCode >= 300 { + return "", fmt.Errorf("Chrome resolver status %d: %s", res.StatusCode, truncate(string(raw), 160)) + } + var out struct { + MediaID string `json:"media_id"` + } + if err := json.Unmarshal(raw, &out); err != nil { + return "", err + } + if !isNumericMediaID(out.MediaID) { + return "", fmt.Errorf("Chrome resolver returned no numeric media ID") + } + return out.MediaID, nil +} + +type HTTPCrawlerProvider struct { + Endpoint string + Token string + HTTP *http.Client +} + +func NewHTTPCrawlerProvider(endpoint, token string) *HTTPCrawlerProvider { + return &HTTPCrawlerProvider{Endpoint: strings.TrimRight(strings.TrimSpace(endpoint), "/"), Token: strings.TrimSpace(token)} +} + +func (p *HTTPCrawlerProvider) SearchChrome(ctx context.Context, storageState string, terms []string, limit int) ([]ThreadSearchResult, error) { + if p == nil || p.Endpoint == "" || p.Token == "" { + return nil, fmt.Errorf("Chrome crawler is not configured") + } + if strings.TrimSpace(storageState) == "" { + return nil, fmt.Errorf("crawler session required") + } + if limit < 1 { + limit = 10 + } + if limit > 30 { + limit = 30 + } + body, err := json.Marshal(map[string]any{"storage_state": storageState, "terms": nonEmptyTerms(terms), "limit": limit}) + if err != nil { + return nil, err + } + runCtx, cancel := context.WithTimeout(ctx, 90*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(runCtx, http.MethodPost, p.Endpoint+"/v1/threads/search", bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+p.Token) + client := p.HTTP + if client == nil { + client = &http.Client{Timeout: 95 * time.Second} + } + res, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("Chrome crawler request failed: %w", err) + } + defer res.Body.Close() + raw, _ := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + if res.StatusCode < 200 || res.StatusCode >= 300 { + return nil, fmt.Errorf("Chrome crawler status %d: %s", res.StatusCode, truncate(string(raw), 160)) + } + var out struct { + Posts []struct { + Permalink string `json:"permalink"` + Author string `json:"author"` + Text string `json:"text"` + } `json:"posts"` + } + if err := json.Unmarshal(raw, &out); err != nil { + return nil, fmt.Errorf("Chrome crawler response invalid: %w", err) + } + results := make([]ThreadSearchResult, 0, len(out.Posts)) + for _, post := range out.Posts { + if !isThreadsURL(post.Permalink) || strings.TrimSpace(post.Text) == "" { + continue + } + results = append(results, ThreadSearchResult{URL: post.Permalink, Title: post.Author, Snippet: post.Text}) + } + return results, nil +} diff --git a/apps/backend/internal/module/scout/usecase/exa_threads_provider.go b/apps/backend/internal/module/scout/usecase/exa_threads_provider.go new file mode 100644 index 0000000..6e5fd8b --- /dev/null +++ b/apps/backend/internal/module/scout/usecase/exa_threads_provider.go @@ -0,0 +1,163 @@ +package usecase + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +const exaSearchURL = "https://api.exa.ai/search" + +// ThreadSearchProvider finds public Threads posts for a Scout scan. +type ThreadSearchProvider interface { + SearchThreads(ctx context.Context, terms []string, limit int) ([]ThreadSearchResult, error) +} + +type ThreadSearchResult struct { + URL string + Title string + Snippet string +} + +// ExaThreadsProvider searches only Threads-owned domains through Exa. +type ExaThreadsProvider struct { + APIKey string + BaseURL string + HTTP *http.Client +} + +func NewExaThreadsProvider(apiKey string) *ExaThreadsProvider { + return &ExaThreadsProvider{ + APIKey: strings.TrimSpace(apiKey), + BaseURL: exaSearchURL, + HTTP: &http.Client{Timeout: 25 * time.Second}, + } +} + +func newDefaultExaThreadsProvider() *ExaThreadsProvider { + key := os.Getenv("EXA_API_KEY") + if strings.TrimSpace(key) == "" { + key = os.Getenv("EXA_KEY") + } + return NewExaThreadsProvider(key) +} + +func (p *ExaThreadsProvider) SearchThreads(ctx context.Context, terms []string, limit int) ([]ThreadSearchResult, error) { + if p == nil || strings.TrimSpace(p.APIKey) == "" { + return nil, fmt.Errorf("exa Threads search is not configured") + } + if limit <= 0 { + limit = 5 + } + if limit > 20 { + limit = 20 + } + query := strings.Join(nonEmptyTerms(terms), " ") + if query == "" { + return nil, fmt.Errorf("exa Threads search query required") + } + + payload, err := json.Marshal(map[string]any{ + "query": query, + "type": "auto", + "numResults": limit, + "includeDomains": []string{"threads.net", "threads.com"}, + "contents": map[string]any{ + "highlights": true, + "text": map[string]any{"maxCharacters": 400}, + }, + }) + if err != nil { + return nil, err + } + baseURL := p.BaseURL + if baseURL == "" { + baseURL = exaSearchURL + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, baseURL, bytes.NewReader(payload)) + if err != nil { + return nil, err + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-api-key", p.APIKey) + httpClient := p.HTTP + if httpClient == nil { + httpClient = &http.Client{Timeout: 25 * time.Second} + } + res, err := httpClient.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + raw, err := io.ReadAll(io.LimitReader(res.Body, 1<<20)) + if err != nil { + return nil, err + } + if res.StatusCode != http.StatusOK { + return nil, fmt.Errorf("exa status %d: %s", res.StatusCode, truncate(string(raw), 120)) + } + + var response struct { + Results []struct { + Title string `json:"title"` + URL string `json:"url"` + Highlights []string `json:"highlights"` + Text string `json:"text"` + } `json:"results"` + } + if err := json.Unmarshal(raw, &response); err != nil { + return nil, err + } + results := make([]ThreadSearchResult, 0, len(response.Results)) + for _, hit := range response.Results { + url := strings.TrimSpace(hit.URL) + if !isThreadsURL(url) { + continue + } + snippet := firstNonEmpty(hit.Highlights) + if snippet == "" { + snippet = strings.TrimSpace(hit.Text) + } + if snippet == "" { + snippet = strings.TrimSpace(hit.Title) + } + results = append(results, ThreadSearchResult{URL: url, Title: strings.TrimSpace(hit.Title), Snippet: truncate(snippet, 400)}) + } + return results, nil +} + +func isThreadsURL(raw string) bool { + u, err := url.Parse(raw) + if err != nil { + return false + } + host := strings.ToLower(u.Hostname()) + return host == "threads.net" || strings.HasSuffix(host, ".threads.net") || host == "threads.com" || strings.HasSuffix(host, ".threads.com") +} + +func nonEmptyTerms(terms []string) []string { + out := make([]string, 0, len(terms)) + for _, term := range terms { + if term = strings.TrimSpace(term); term != "" { + out = append(out, term) + } + } + return out +} + +func firstNonEmpty(values []string) string { + for _, value := range values { + if value = strings.TrimSpace(value); value != "" { + return value + } + } + return "" +} diff --git a/apps/backend/internal/module/scout/usecase/exa_threads_provider_test.go b/apps/backend/internal/module/scout/usecase/exa_threads_provider_test.go new file mode 100644 index 0000000..11aefaa --- /dev/null +++ b/apps/backend/internal/module/scout/usecase/exa_threads_provider_test.go @@ -0,0 +1,36 @@ +package usecase + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestExaThreadsProviderSearchesThreadsDomains(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, http.MethodPost, r.Method) + require.Equal(t, "test-key", r.Header.Get("x-api-key")) + var body map[string]any + require.NoError(t, json.NewDecoder(r.Body).Decode(&body)) + require.Equal(t, []any{"threads.net", "threads.com"}, body["includeDomains"]) + _, _ = w.Write([]byte(`{"results":[{"title":"Not a Threads post","url":"https://example.com/post/1","highlights":["Ignore me"]},{"title":"Example","url":"https://www.threads.net/@alice/post/1","highlights":["A matching post"]}]}`)) + })) + defer server.Close() + + provider := NewExaThreadsProvider("test-key") + provider.BaseURL = server.URL + results, err := provider.SearchThreads(context.Background(), []string{"sensitive skin"}, 1) + require.NoError(t, err) + require.Equal(t, []ThreadSearchResult{{ + URL: "https://www.threads.net/@alice/post/1", Title: "Example", Snippet: "A matching post", + }}, results) +} + +func TestExaThreadsProviderRejectsMissingKey(t *testing.T) { + _, err := NewExaThreadsProvider("").SearchThreads(context.Background(), []string{"query"}, 1) + require.Error(t, err) +} diff --git a/apps/backend/internal/module/scout/usecase/m5_scout_test.go b/apps/backend/internal/module/scout/usecase/m5_scout_test.go index fb5f251..3236aef 100644 --- a/apps/backend/internal/module/scout/usecase/m5_scout_test.go +++ b/apps/backend/internal/module/scout/usecase/m5_scout_test.go @@ -2,6 +2,8 @@ package usecase_test import ( "context" + "fmt" + "strings" "testing" "apps/backend/internal/module/scout/domain" @@ -16,6 +18,48 @@ type staticDevMode struct { on bool } +type fakeThreadSearchProvider struct{} + +type fakeChromeCrawler struct{ fakeThreadSearchProvider } + +type failingChromeCrawler struct{ fakeChromeCrawler } + +type fakeReplyQueue struct{} + +func (fakeThreadSearchProvider) SearchThreads(_ context.Context, terms []string, limit int) ([]usecase.ThreadSearchResult, error) { + results := make([]usecase.ThreadSearchResult, 0, limit) + for i, term := range terms { + if i == limit { + break + } + results = append(results, usecase.ThreadSearchResult{ + URL: "https://www.threads.net/@test_user/post/" + term, + Title: term, + Snippet: "realistic result for " + term, + }) + } + return results, nil +} + +func (f fakeChromeCrawler) SearchChrome(ctx context.Context, storageState string, terms []string, limit int) ([]usecase.ThreadSearchResult, error) { + if strings.TrimSpace(storageState) == "" { + return nil, domain.ErrNoCrawlerSession + } + return f.SearchThreads(ctx, terms, limit) +} + +func (fakeChromeCrawler) ResolveMediaID(_ context.Context, _ string, _ string) (string, error) { + return "123456789012345", nil +} + +func (failingChromeCrawler) ResolveMediaID(_ context.Context, _ string, _ string) (string, error) { + return "", fmt.Errorf("crawler session expired") +} + +func (fakeReplyQueue) QueueExternalReply(_ context.Context, _ int64, _, _, _, _ string) (string, error) { + return "outbox_test", nil +} + func (s *staticDevMode) DevModeEnabled(_ context.Context, _ int64) (bool, error) { return s.on, nil } @@ -24,8 +68,11 @@ func newScout() (*usecase.Service, *studioPublish.FakeTransport, *staticDevMode) tp := studioPublish.NewFake() dev := &staticDevMode{} svc := usecase.New(repository.NewMemory()) + svc.SessionSecret = "test-crawler-session-secret" svc.Transport = tp svc.Settings = dev + svc.Provider = fakeThreadSearchProvider{} + svc.Crawler = fakeChromeCrawler{} return svc, tp, dev } @@ -83,7 +130,7 @@ func TestSC_06_CrawlerPath(t *testing.T) { svc, _, dev := newScout() uid := int64(5_002_006) dev.on = true - require.NoError(t, svc.SetCrawlerSession(context.Background(), uid, "ext-session-1")) + require.NoError(t, svc.SetCrawlerSession(context.Background(), uid, validStorageState())) brief, _ := svc.PrepareBrief(context.Background(), uid, "成分黨", "", "", "value", false) posts, err := svc.RunScanFromBrief(context.Background(), uid, brief) require.NoError(t, err) @@ -99,6 +146,44 @@ func TestSC_07_NoSession(t *testing.T) { require.ErrorIs(t, err, domain.ErrNoCrawlerSession) } +func TestCrawlerSessionEncryptedAndValidated(t *testing.T) { + ctx := context.Background() + store := repository.NewMemory() + svc := usecase.New(store) + svc.SessionSecret = "test-crawler-session-secret" + state := validStorageState() + + require.NoError(t, svc.SetCrawlerSession(ctx, 1, state)) + persisted, err := store.GetCrawlerSession(ctx, 1) + require.NoError(t, err) + require.NotEqual(t, state, persisted.StorageStateEnc) + require.Positive(t, persisted.UpdatedAt) + require.Positive(t, persisted.ExpiresAt) + got, err := svc.GetCrawlerSessionToken(ctx, 1) + require.NoError(t, err) + require.Equal(t, state, got) + + for _, invalid := range []string{ + "not json", + "[]", + `{"cookies":[]}`, + `{"cookies":[{"domain":"evil.example","expires":4102444800}]}`, + `{"cookies":null}`, + strings.Repeat("x", 256*1024+1), + } { + require.ErrorIs(t, svc.SetCrawlerSession(ctx, 1, invalid), domain.ErrValidation) + } +} + +func TestCrawlerSessionRequiresSecret(t *testing.T) { + svc := usecase.New(repository.NewMemory()) + require.ErrorIs(t, svc.SetCrawlerSession(context.Background(), 1, validStorageState()), domain.ErrValidation) +} + +func validStorageState() string { + return `{"cookies":[{"domain":".threads.net","expires":4102444800}]}` +} + func TestSC_08_ListPostsAfterScan(t *testing.T) { svc, _, _ := newScout() uid := int64(5_002_008) @@ -129,34 +214,31 @@ func TestSC_10_SkipAndMark(t *testing.T) { p, err := svc.SkipOutreach(context.Background(), uid, posts[0].ID) require.NoError(t, err) require.Equal(t, domain.OutreachSkipped, p.OutreachStatus) - if len(posts) > 1 { - p2, err := svc.MarkPublished(context.Background(), uid, posts[1].ID) - require.NoError(t, err) - require.Equal(t, domain.OutreachPublished, p2.OutreachStatus) - } + _, err = svc.MarkPublished(context.Background(), uid, posts[0].ID) + require.Error(t, err) } func TestSC_11_SendOutreach(t *testing.T) { - svc, tp, _ := newScout() + svc, _, _ := newScout() uid := int64(5_002_011) brief, _ := svc.PrepareBrief(context.Background(), uid, "send", "", "", "value", false) posts, _ := svc.RunScanFromBrief(context.Background(), uid, brief) - p, err := svc.SendOutreach(context.Background(), uid, posts[0].ID, "你好呀", "acc1") - require.NoError(t, err) - require.Equal(t, domain.OutreachPublished, p.OutreachStatus) - require.Equal(t, 1, tp.CallCount()) - // fail path - tp.Fail = true - brief2, _ := svc.PrepareBrief(context.Background(), uid, "sendfail", "", "", "value", false) - posts2, _ := svc.RunScanFromBrief(context.Background(), uid, brief2) - _, err = svc.SendOutreach(context.Background(), uid, posts2[0].ID, "nope", "") + _, err := svc.SendOutreach(context.Background(), uid, posts[0].ID, "你好呀", "acc1") require.Error(t, err) - got, _ := svc.ListPosts(context.Background(), uid, "") - for _, x := range got { - if x.ID == posts2[0].ID { - require.NotEqual(t, domain.OutreachPublished, x.OutreachStatus) - } - } +} + +func TestSC_11_SendOutreachResolverFailureIsValidation(t *testing.T) { + svc, _, _ := newScout() + uid := int64(5_002_012) + svc.Crawler = failingChromeCrawler{} + svc.ReplyQueue = fakeReplyQueue{} + require.NoError(t, svc.SetCrawlerSession(context.Background(), uid, validStorageState())) + brief, _ := svc.PrepareBrief(context.Background(), uid, "send", "", "", "value", false) + posts, _ := svc.RunScanFromBrief(context.Background(), uid, brief) + + _, err := svc.SendOutreach(context.Background(), uid, posts[0].ID, "你好呀", "acc1") + require.ErrorIs(t, err, domain.ErrValidation) + require.ErrorContains(t, err, "unable to resolve the target Threads post") } func TestSC_12_RemovePostTheme(t *testing.T) { diff --git a/apps/backend/internal/module/scout/usecase/planner.go b/apps/backend/internal/module/scout/usecase/planner.go new file mode 100644 index 0000000..89682e0 --- /dev/null +++ b/apps/backend/internal/module/scout/usecase/planner.go @@ -0,0 +1,128 @@ +package usecase + +import ( + "crypto/sha256" + "encoding/hex" + "net/url" + "strconv" + "strings" + + "apps/backend/internal/module/scout/domain" +) + +type classifiedPost struct { + classification string + score int + reason string +} + +func planScanTerms(brief *domain.RunBrief) []string { + return dedupeTerms(brief.Pains, brief.Tags, brief.Periphery, []string{brief.Intent}) +} + +func dedupeTerms(groups ...[]string) []string { + seen := make(map[string]struct{}) + var out []string + for _, group := range groups { + for _, term := range group { + term = strings.Join(strings.Fields(strings.TrimSpace(term)), " ") + key := strings.ToLower(term) + if term == "" { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + out = append(out, term) + } + } + return out +} + +func classifyPost(mode, text string, terms []string) classifiedPost { + lower := strings.ToLower(text) + signals := matchedSignals(lower, terms) + if hasAny(lower, "giveaway", "抽獎", "follow for follow", "互追", "crypto", "賺錢") { + return classifiedPost{domain.ClassificationNoise, 0, "noise signal"} + } + if mode == domain.ModeActivity { + return classifyActivity(lower, signals) + } + if hasAny(lower, "dm me", "私訊我", "服務洽詢", "立即購買", "限時優惠", "團購", "業配") { + return scored(domain.ClassificationProviderOffer, 35, signals, "provider-offer signal") + } + if hasAny(lower, "推薦", "求推", "有沒有推薦", "any recommendation", "what do you recommend") { + return scored(domain.ClassificationSeekingRecommendation, 60, signals, "recommendation signal") + } + if strings.Contains(lower, "?") || strings.Contains(lower, "？") || hasAny(lower, "怎麼", "如何", "請問", "求助", "help") { + return scored(domain.ClassificationSeekingHelp, 55, signals, "help-seeking signal") + } + return scored(domain.ClassificationDiscussion, 40, signals, "discussion signal") +} + +func classifyActivity(text string, signals []string) classifiedPost { + if hasAny(text, "dm me", "私訊我", "服務洽詢", "立即購買", "限時優惠", "團購", "業配") { + return scored(domain.ClassificationProviderOffer, 35, signals, "provider-offer signal; recency unavailable (neutral)") + } + if strings.Contains(text, "?") || strings.Contains(text, "？") || hasAny(text, "請問", "怎麼", "如何", "有人知道", "求") { + return scored(domain.ClassificationAsking, 60, signals, "asking signal; recency unavailable (neutral)") + } + if hasAny(text, "活動", "event", "開幕", "launch", "發布", "報名", "登記", "公告") { + return scored(domain.ClassificationAnnouncement, 50, signals, "announcement signal; recency unavailable (neutral)") + } + return scored(domain.ClassificationDiscussion, 40, signals, "discussion signal; recency unavailable (neutral)") +} + +func scored(classification string, base int, signals []string, label string) classifiedPost { + score := base + len(signals)*15 + if score > 100 { + score = 100 + } + reason := label + if len(signals) > 0 { + reason += "; matched: " + strings.Join(signals, ", ") + } + return classifiedPost{classification, score, reason} +} + +func matchedSignals(text string, terms []string) []string { + var signals []string + for _, term := range dedupeTerms(terms) { + if strings.Contains(text, strings.ToLower(term)) { + signals = append(signals, term) + } + } + return signals +} + +func hasAny(text string, signals ...string) bool { + for _, signal := range signals { + if strings.Contains(text, signal) { + return true + } + } + return false +} + +func canonicalPermalink(raw string) string { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Host == "" { + return "" + } + u.Scheme = "https" + u.Host = strings.ToLower(u.Host) + u.RawQuery = "" + u.Fragment = "" + u.Path = strings.TrimRight(u.Path, "/") + return u.String() +} + +func permalinkID(ownerUID int64, permalink string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(permalink) + "|" + formatOwnerUID(ownerUID))) + return "sp_" + hex.EncodeToString(sum[:])[:20] +} + +func formatOwnerUID(ownerUID int64) string { + return strconv.FormatInt(ownerUID, 10) +} diff --git a/apps/backend/internal/module/scout/usecase/planner_test.go b/apps/backend/internal/module/scout/usecase/planner_test.go new file mode 100644 index 0000000..bcf2bcd --- /dev/null +++ b/apps/backend/internal/module/scout/usecase/planner_test.go @@ -0,0 +1,120 @@ +package usecase_test + +import ( + "context" + "testing" + + "apps/backend/internal/module/scout/domain" + "apps/backend/internal/module/scout/repository" + "apps/backend/internal/module/scout/usecase" + + "github.com/stretchr/testify/require" +) + +type capturedSearchProvider struct { + terms []string + hits []usecase.ThreadSearchResult +} + +func TestPrepareBriefPlansDeduplicatedTermsFromAllInputs(t *testing.T) { + svc := usecase.New(repository.NewMemory()) + brand, err := svc.CreateBrand(context.Background(), 1, "B", "") + require.NoError(t, err) + product, err := svc.SaveProduct(context.Background(), 1, &domain.Product{ + BrandID: brand.ID, Label: "P", PainPoints: []string{"敏感肌", "敏感肌"}, MatchTags: []string{"保養", "敏感肌"}, + }) + require.NoError(t, err) + brief, err := svc.PrepareBrief(context.Background(), 1, "敏感肌保養", brand.ID, product.ID, "value", false) + require.NoError(t, err) + require.Equal(t, []string{"敏感肌", "保養", "使用情境", "替代方案", "成分／規格", "敏感肌保養"}, brief.ScanTerms) +} + +func (p *capturedSearchProvider) SearchThreads(_ context.Context, terms []string, _ int) ([]usecase.ThreadSearchResult, error) { + p.terms = append([]string(nil), terms...) + return p.hits, nil +} + +func TestPlannerProductSignalsAndDeterministicPersistence(t *testing.T) { + provider := &capturedSearchProvider{hits: []usecase.ThreadSearchResult{ + {URL: "https://www.threads.net/@a/post/1?utm_source=test", Snippet: "請問敏感肌怎麼舒緩？"}, + {URL: "https://www.threads.net/@b/post/2", Snippet: "敏感肌保養有沒有推薦？"}, + {URL: "https://www.threads.net/@c/post/3", Snippet: "敏感肌服務洽詢，現在限時優惠"}, + {URL: "https://www.threads.net/@d/post/4", Snippet: "分享敏感肌保養的使用心得"}, + {URL: "https://www.threads.net/@e/post/5", Snippet: "敏感肌抽獎，互追拿好禮"}, + }} + store := repository.NewMemory() + svc := usecase.New(store) + svc.Provider = provider + brief := &domain.RunBrief{ + Mode: domain.ModeProduct, Intent: "敏感肌保養", Pains: []string{"敏感肌", "敏感肌"}, + Tags: []string{"保養", "敏感肌"}, Periphery: []string{"使用情境", "保養"}, + } + brief.ScanTerms = []string{"敏感肌", "保養", "使用情境", "敏感肌保養"} + + posts, err := svc.RunScanFromBrief(context.Background(), 42, brief) + require.NoError(t, err) + require.Equal(t, []string{"敏感肌", "保養", "使用情境", "敏感肌保養"}, provider.terms) + require.Len(t, posts, 4) + byClass := map[string]*domain.Post{} + for _, post := range posts { + byClass[post.Classification] = post + require.NotEmpty(t, post.Permalink) + require.Equal(t, post.Permalink, post.ExternalID) + require.InDelta(t, 0, post.Score, 100) + require.NotEmpty(t, post.MatchReason) + } + require.Contains(t, byClass, domain.ClassificationSeekingHelp) + require.Contains(t, byClass, domain.ClassificationSeekingRecommendation) + require.Contains(t, byClass, domain.ClassificationProviderOffer) + require.Contains(t, byClass, domain.ClassificationDiscussion) + require.Contains(t, byClass[domain.ClassificationSeekingHelp].MatchReason, "敏感肌") + require.Equal(t, "https://www.threads.net/@a/post/1", byClass[domain.ClassificationSeekingHelp].Permalink) + + again, err := svc.RunScanFromBrief(context.Background(), 42, brief) + require.NoError(t, err) + require.Equal(t, byClass[domain.ClassificationSeekingHelp].ID, again[0].ID) + persisted, err := svc.ListPosts(context.Background(), 42, "") + require.NoError(t, err) + require.Len(t, persisted, 4) +} + +func TestPlannerActivityClassifiesWithNeutralRecency(t *testing.T) { + provider := &capturedSearchProvider{hits: []usecase.ThreadSearchResult{ + {URL: "https://www.threads.net/@a/post/1", Snippet: "請問這週市集怎麼報名？"}, + {URL: "https://www.threads.net/@b/post/2", Snippet: "市集活動公告，週六開幕"}, + {URL: "https://www.threads.net/@c/post/3", Snippet: "市集攤位服務洽詢"}, + {URL: "https://www.threads.net/@d/post/4", Snippet: "分享市集逛街心得"}, + {URL: "https://www.threads.net/@e/post/5", Snippet: "市集 crypto 賺錢群組"}, + }} + svc := usecase.New(repository.NewMemory()) + svc.Provider = provider + brief := &domain.RunBrief{Mode: domain.ModeActivity, Intent: "市集", ScanTerms: []string{"市集"}} + + posts, err := svc.RunScanFromBrief(context.Background(), 7, brief) + require.NoError(t, err) + require.Len(t, posts, 4) + classes := map[string]bool{} + for _, post := range posts { + classes[post.Classification] = true + require.Contains(t, post.MatchReason, "recency unavailable (neutral)") + require.Greater(t, post.Score, 0) + } + require.True(t, classes[domain.ClassificationAsking]) + require.True(t, classes[domain.ClassificationAnnouncement]) + require.True(t, classes[domain.ClassificationProviderOffer]) + require.True(t, classes[domain.ClassificationDiscussion]) +} + +func TestDraftOutreachIsDeterministicAndDoesNotNeedAI(t *testing.T) { + svc := usecase.New(repository.NewMemory()) + svc.Provider = &capturedSearchProvider{hits: []usecase.ThreadSearchResult{{ + URL: "https://www.threads.net/@author/post/1", Snippet: "請問怎麼處理頭皮困擾？", + }}} + posts, err := svc.RunScanFromBrief(context.Background(), 9, &domain.RunBrief{ + Mode: domain.ModeTheme, Intent: "頭皮", ScanTerms: []string{"頭皮"}, + }) + require.NoError(t, err) + draft, err := svc.DraftOutreach(context.Background(), 9, posts[0].ID, "") + require.NoError(t, err) + require.Equal(t, "嗨 @author，看到你提到「頭皮」，我也遇過類似情況。若你願意，想聽聽你後來怎麼處理。", draft.DraftText) +} diff --git a/apps/backend/internal/module/scout/usecase/service.go b/apps/backend/internal/module/scout/usecase/service.go index 90f85eb..c6f2aa7 100644 --- a/apps/backend/internal/module/scout/usecase/service.go +++ b/apps/backend/internal/module/scout/usecase/service.go @@ -2,13 +2,14 @@ package usecase import ( "context" + "encoding/json" "fmt" "net/url" "strings" + "time" "apps/backend/internal/module/ai" "apps/backend/internal/module/scout/domain" - studioDomain "apps/backend/internal/module/studio/domain" studioPublish "apps/backend/internal/module/studio/publish" threadsDomain "apps/backend/internal/module/threads/domain" @@ -20,22 +21,24 @@ type SettingsReader interface { DevModeEnabled(ctx context.Context, uid int64) (bool, error) } -// AccountLookup for send outreach -type AccountLookup interface { - Get(ctx context.Context, id string) (*threadsDomain.Account, error) - AccessToken(ctx context.Context, a *threadsDomain.Account) (string, error) +type ReplyQueue interface { + QueueExternalReply(ctx context.Context, ownerUID int64, accountID, replyToMediaID, text, title string) (outboxID string, err error) } type Service struct { - Repo domain.Repository - Settings SettingsReader - Transport studioPublish.Transport - Accounts AccountLookup - AI ai.Client + Repo domain.Repository + Settings SettingsReader + // Transport is retained only for test construction compatibility. Scout never publishes directly. + Transport studioPublish.Transport + AI ai.Client // Retained for service wiring; Scout drafts never call AI. + ReplyQueue ReplyQueue + Provider ThreadSearchProvider + Crawler ChromeCrawlerProvider + SessionSecret string } func New(repo domain.Repository) *Service { - return &Service{Repo: repo} + return &Service{Repo: repo, Provider: newDefaultExaThreadsProvider()} } func (s *Service) ListBrands(ctx context.Context, ownerUID int64) ([]*domain.Brand, error) { @@ -202,7 +205,7 @@ func (s *Service) ImportProductFromURL(_ context.Context, raw string) (*domain.I label = host + " 商品" } return &domain.ImportDraft{ - Label: label, + Label: label, ProductContext: "從 " + raw + " 推估的產品情境（live 可接真爬頁）", PainPoints: []string{"找不到適合的", "價格猶豫", "不確定是否適合自己"}, MatchTags: []string{"好物", "推薦", "踩雷"}, @@ -248,15 +251,7 @@ func (s *Service) PrepareBrief(ctx context.Context, ownerUID int64, intent, bran brief.Tags = tokenize(intent) } brief.Periphery = []string{"使用情境", "替代方案", "成分／規格"} - seen := map[string]bool{} - for _, t := range append(brief.Pains, brief.Tags...) { - t = strings.TrimSpace(t) - if t == "" || seen[t] { - continue - } - seen[t] = true - brief.ScanTerms = append(brief.ScanTerms, t) - } + brief.ScanTerms = planScanTerms(brief) brief.ThemeLabel = truncate(intent, 36) brief.ThemeKey = mode + "|" + productID + "|" + truncate(intent, 48) brief.ResponseStance = "先共鳴再給建議" @@ -278,27 +273,53 @@ func (s *Service) RunScanFromBrief(ctx context.Context, ownerUID int64, brief *d } } if devMode { - sess, err := s.Repo.GetCrawlerSession(ctx, ownerUID) - if err != nil || sess == nil || sess.Token == "" { + storageState, err := s.GetCrawlerSessionToken(ctx, ownerUID) + if err != nil { return nil, domain.ErrNoCrawlerSession } path = domain.PathCrawler + if s.Crawler == nil { + return nil, fmt.Errorf("Chrome crawler is not configured") + } + hits, err := s.Crawler.SearchChrome(ctx, storageState, brief.ScanTerms, 20) + if err != nil { + return nil, err + } + return s.persistSearchHits(ctx, ownerUID, brief, path, hits) } + if s.Provider == nil { + return nil, fmt.Errorf("scout search provider is not configured") + } + hits, err := s.Provider.SearchThreads(ctx, brief.ScanTerms, 5) + if err != nil { + return nil, err + } + return s.persistSearchHits(ctx, ownerUID, brief, path, hits) +} + +func (s *Service) persistSearchHits(ctx context.Context, ownerUID int64, brief *domain.RunBrief, path string, hits []ThreadSearchResult) ([]*domain.Post, error) { now := domain.NowNano() - var out []*domain.Post - authors := []string{"itchy_days", "new_mom_tw", "remote_worker", "curious_one"} - limit := len(brief.ScanTerms) - if limit > 5 { - limit = 5 - } - for i := 0; i < limit; i++ { - term := brief.ScanTerms[i] + out := make([]*domain.Post, 0, len(hits)) + for i, hit := range hits { + text := strings.TrimSpace(hit.Snippet) + if text == "" { + continue + } + permalink := canonicalPermalink(hit.URL) + if permalink == "" { + continue + } + term := matchingTerm(text+" "+hit.Title, brief.ScanTerms) + classified := classifyPost(brief.Mode, text+" "+hit.Title, brief.ScanTerms) + if classified.classification == domain.ClassificationNoise { + continue + } p := &domain.Post{ - ID: "sp_" + uuid.NewString()[:10], OwnerUID: ownerUID, BrandID: brief.BrandID, - Author: authors[i%len(authors)], Text: fmt.Sprintf("有人懂「%s」嗎？最近超煩…", term), - SearchTag: term, Opportunity: "可接話分享經驗", OutreachStatus: domain.OutreachNew, - Score: 55 + (i*11)%40, MatchedProductID: brief.ProductID, MatchedProductLabel: brief.ProductLabel, - MatchReason: "命中掃描詞 " + term, ScoutMode: brief.Mode, IntentSnippet: brief.Intent, + ID: permalinkID(ownerUID, permalink), ExternalID: permalink, Permalink: permalink, + OwnerUID: ownerUID, BrandID: brief.BrandID, Author: authorFromThreadsURL(permalink), Text: text, + SearchTag: term, Opportunity: "", OutreachStatus: domain.OutreachNew, + Score: classified.score, Classification: classified.classification, MatchedProductID: brief.ProductID, MatchedProductLabel: brief.ProductLabel, + MatchReason: classified.reason, ScoutMode: brief.Mode, IntentSnippet: brief.Intent, ThemeKey: brief.ThemeKey, ThemeLabel: brief.ThemeLabel, ScanPath: path, CreatedAt: now - int64(i)*1000, } @@ -310,6 +331,29 @@ func (s *Service) RunScanFromBrief(ctx context.Context, ownerUID int64, brief *d return out, nil } +func matchingTerm(text string, terms []string) string { + for _, term := range terms { + if term = strings.TrimSpace(term); term != "" && strings.Contains(strings.ToLower(text), strings.ToLower(term)) { + return term + } + } + return "" +} + +func authorFromThreadsURL(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "" + } + for _, segment := range strings.Split(u.Path, "/") { + segment = strings.TrimPrefix(strings.TrimSpace(segment), "@") + if segment != "" { + return segment + } + } + return "" +} + func (s *Service) ListPosts(ctx context.Context, ownerUID int64, brandID string) ([]*domain.Post, error) { return s.Repo.ListPosts(ctx, ownerUID, brandID) } @@ -320,12 +364,7 @@ func (s *Service) DraftOutreach(ctx context.Context, ownerUID int64, postID, per if err != nil { return nil, err } - draft := "嗨 @" + p.Author + "，看到你提到「" + p.SearchTag + "」，我也遇過類似情況…" - if s.AI != nil { - if out, aerr := s.AI.Complete(ctx, "fake", "grok-3", "outreach: "+p.Text); aerr == nil && out != "" { - draft = out - } - } + draft := "嗨 @" + p.Author + "，看到你提到「" + p.SearchTag + "」，我也遇過類似情況。若你願意，想聽聽你後來怎麼處理。" p.DraftText = draft p.OutreachStatus = domain.OutreachDrafted if err := s.Repo.SavePost(ctx, p); err != nil { @@ -347,15 +386,11 @@ func (s *Service) SkipOutreach(ctx context.Context, ownerUID int64, postID strin } func (s *Service) MarkPublished(ctx context.Context, ownerUID int64, postID string) (*domain.Post, error) { - p, err := s.getPostOwned(ctx, ownerUID, postID) + _, err := s.getPostOwned(ctx, ownerUID, postID) if err != nil { return nil, err } - p.OutreachStatus = domain.OutreachPublished - if err := s.Repo.SavePost(ctx, p); err != nil { - return nil, err - } - return p, nil + return nil, fmt.Errorf("manual published status is removed; send through Outbox") } func (s *Service) SendOutreach(ctx context.Context, ownerUID int64, postID, text, accountID string) (*domain.Post, error) { @@ -370,29 +405,61 @@ func (s *Service) SendOutreach(ctx context.Context, ownerUID int64, postID, text if text == "" { return nil, fmt.Errorf("%w: empty outreach text", domain.ErrValidation) } - if s.Transport != nil { - token := "fake" - if s.Accounts != nil && accountID != "" { - if acc, aerr := s.Accounts.Get(ctx, accountID); aerr == nil && acc != nil && acc.OwnerUID == ownerUID { - if t, terr := s.Accounts.AccessToken(ctx, acc); terr == nil && t != "" { - token = t - } - } + if s.ReplyQueue == nil { + return nil, fmt.Errorf("Scout reply Outbox is not configured") + } + mediaID := p.ExternalID + // A permalink can be resolved to a real Threads media ID before the official + // API sends. This also refreshes any legacy IDs that were not Graph media IDs. + if strings.TrimSpace(p.Permalink) != "" && s.Crawler != nil { + state, err := s.GetCrawlerSessionToken(ctx, ownerUID) + if err != nil { + return nil, domain.ErrNoCrawlerSession } - if _, perr := s.Transport.Publish(ctx, studioDomain.PublishRequest{ - AccessToken: token, AccountID: accountID, Text: text, - }); perr != nil { - return nil, perr + mediaID, err = s.Crawler.ResolveMediaID(ctx, state, p.Permalink) + if err != nil { + return nil, fmt.Errorf("%w: unable to resolve the target Threads post: %v", domain.ErrValidation, err) } } + if !isNumericMediaID(mediaID) { + if s.Crawler == nil { + return nil, fmt.Errorf("%w: target Threads media ID is not resolved; configure Chrome crawler", domain.ErrValidation) + } + state, err := s.GetCrawlerSessionToken(ctx, ownerUID) + if err != nil { + return nil, domain.ErrNoCrawlerSession + } + mediaID, err = s.Crawler.ResolveMediaID(ctx, state, p.Permalink) + if err != nil { + return nil, fmt.Errorf("%w: unable to resolve the target Threads post: %v", domain.ErrValidation, err) + } + } + p.ExternalID = mediaID + outboxID, err := s.ReplyQueue.QueueExternalReply(ctx, ownerUID, accountID, mediaID, text, "Scout 回覆 · "+truncate(p.Author, 30)) + if err != nil { + return nil, err + } p.DraftText = text - p.OutreachStatus = domain.OutreachPublished + p.OutboxID = outboxID + p.OutreachStatus = domain.OutreachQueued if err := s.Repo.SavePost(ctx, p); err != nil { return nil, err } return p, nil } +func isNumericMediaID(value string) bool { + if value == "" { + return false + } + for _, ch := range value { + if ch < '0' || ch > '9' { + return false + } + } + return true +} + func (s *Service) RemovePost(ctx context.Context, ownerUID int64, postID string) error { if _, err := s.getPostOwned(ctx, ownerUID, postID); err != nil { return err @@ -401,7 +468,14 @@ func (s *Service) RemovePost(ctx context.Context, ownerUID int64, postID string) } func (s *Service) RemoveTheme(ctx context.Context, ownerUID int64, themeKey string) error { - return s.Repo.DeletePostsByTheme(ctx, ownerUID, themeKey) + if err := s.Repo.DeletePostsByTheme(ctx, ownerUID, themeKey); err != nil { + return err + } + // A patrol batch owns both its hits and its research snapshot. + if err := s.Repo.DeleteHomework(ctx, ownerUID, themeKey); err != nil && err != domain.ErrNotFound { + return err + } + return nil } func (s *Service) ListHomework(ctx context.Context, ownerUID int64) ([]*domain.Homework, error) { @@ -430,22 +504,87 @@ func (s *Service) RemoveHomework(ctx context.Context, ownerUID int64, themeKey s return s.Repo.DeleteHomework(ctx, ownerUID, themeKey) } -func (s *Service) SetCrawlerSession(ctx context.Context, ownerUID int64, token string) error { +func (s *Service) SetCrawlerSession(ctx context.Context, ownerUID int64, storageState string) error { + expiresAt, err := validateStorageState(storageState) + if err != nil { + return err + } + if s.SessionSecret == "" { + return fmt.Errorf("%w: crawler session secret is required", domain.ErrValidation) + } + storageStateEnc, err := threadsDomain.Seal(s.SessionSecret, storageState) + if err != nil { + return err + } + now := domain.NowNano() return s.Repo.SetCrawlerSession(ctx, &domain.CrawlerSession{ - OwnerUID: ownerUID, Token: token, UpdatedAt: domain.NowNano(), + OwnerUID: ownerUID, StorageStateEnc: storageStateEnc, UpdatedAt: now, ExpiresAt: expiresAt, }) } -// GetCrawlerSessionToken returns Playwright storageState JSON for browser crawl (may be empty). +// GetCrawlerSessionToken returns decrypted Playwright storageState JSON for browser crawl. func (s *Service) GetCrawlerSessionToken(ctx context.Context, ownerUID int64) (string, error) { sess, err := s.Repo.GetCrawlerSession(ctx, ownerUID) if err != nil { return "", err } - if sess == nil { - return "", nil + if sess == nil || sess.StorageStateEnc == "" || (sess.ExpiresAt > 0 && sess.ExpiresAt <= domain.NowNano()) { + return "", domain.ErrNoCrawlerSession } - return sess.Token, nil + if s.SessionSecret == "" { + return "", fmt.Errorf("%w: crawler session secret is required", domain.ErrValidation) + } + storageState, err := threadsDomain.Open(s.SessionSecret, sess.StorageStateEnc) + if err != nil { + return "", err + } + return storageState, nil +} + +type storageStateCookie struct { + Domain string `json:"domain"` + Expires float64 `json:"expires"` +} + +func validateStorageState(storageState string) (int64, error) { + if len(storageState) > 256*1024 { + return 0, fmt.Errorf("%w: storage state exceeds 256KB", domain.ErrValidation) + } + var state map[string]json.RawMessage + if err := json.Unmarshal([]byte(storageState), &state); err != nil || state == nil { + return 0, fmt.Errorf("%w: storage state must be a JSON object", domain.ErrValidation) + } + rawCookies, ok := state["cookies"] + if !ok { + return 0, fmt.Errorf("%w: cookies required", domain.ErrValidation) + } + var cookies []storageStateCookie + if err := json.Unmarshal(rawCookies, &cookies); err != nil || len(cookies) == 0 { + return 0, fmt.Errorf("%w: cookies must be a nonempty array", domain.ErrValidation) + } + var expiresAt int64 + for _, cookie := range cookies { + domainName := strings.TrimPrefix(strings.ToLower(strings.TrimSpace(cookie.Domain)), ".") + if !allowedCookieDomain(domainName) { + return 0, fmt.Errorf("%w: cookie domain %q is not allowed", domain.ErrValidation, cookie.Domain) + } + if cookie.Expires > 0 { + expires := int64(cookie.Expires * float64(time.Second)) + if expiresAt == 0 || expires < expiresAt { + expiresAt = expires + } + } + } + return expiresAt, nil +} + +func allowedCookieDomain(domainName string) bool { + for _, allowed := range []string{"threads.net", "threads.com", "instagram.com", "facebook.com"} { + if domainName == allowed || strings.HasSuffix(domainName, "."+allowed) { + return true + } + } + return false } func (s *Service) ClearCrawlerSession(ctx context.Context, ownerUID int64) error { diff --git a/apps/backend/internal/module/studio/publish/unavailable.go b/apps/backend/internal/module/studio/publish/unavailable.go new file mode 100644 index 0000000..495e575 --- /dev/null +++ b/apps/backend/internal/module/studio/publish/unavailable.go @@ -0,0 +1,24 @@ +package publish + +import ( + "context" + "fmt" + + "apps/backend/internal/module/studio/domain" +) + +// UnavailableTransport is a deliberate non-success path for deployments that +// have not configured Threads. It replaces the old successful fake transport. +type UnavailableTransport struct { + reason string +} + +func NewUnavailable(reason string) *UnavailableTransport { + return &UnavailableTransport{reason: reason} +} + +func (t *UnavailableTransport) Publish(context.Context, domain.PublishRequest) (*domain.PublishResult, error) { + return nil, fmt.Errorf("Threads publishing is not configured: %s", t.reason) +} + +var _ Transport = (*UnavailableTransport)(nil) diff --git a/apps/backend/internal/module/studio/usecase/m4_test.go b/apps/backend/internal/module/studio/usecase/m4_test.go index fe81df8..495d6cc 100644 --- a/apps/backend/internal/module/studio/usecase/m4_test.go +++ b/apps/backend/internal/module/studio/usecase/m4_test.go @@ -356,6 +356,43 @@ func TestPL_08_NoUsableAccount(t *testing.T) { require.ErrorIs(t, err, domain.ErrNoAccount) } +func TestQueueExternalReply(t *testing.T) { + svc, _, acc := newStudio() + uid := int64(4_001_081) + addAccount(acc, uid, "acc1", "replying") + before := domain.NowNano() + bundle, err := svc.QueueExternalReply(context.Background(), uid, "acc1", "123456789", "Thanks for sharing", "External reply") + require.NoError(t, err) + require.Equal(t, uid, bundle.OwnerUID) + require.Equal(t, "External reply", bundle.Title) + require.Equal(t, domain.OBScheduling, bundle.Status) + require.Equal(t, "123456789", bundle.ReplyToMediaID) + require.Len(t, bundle.Steps, 1) + require.Equal(t, domain.StepReply, bundle.Steps[0].Kind) + require.Equal(t, "acc1", bundle.Steps[0].AccountID) + require.Equal(t, "123456789", bundle.Steps[0].ReplyTo) + require.Equal(t, domain.StepScheduled, bundle.Steps[0].Status) + require.InDelta(t, float64(before), float64(bundle.Steps[0].ScheduledAt), float64(2*time.Second)) + + persisted, err := svc.GetOutbox(context.Background(), uid, bundle.ID) + require.NoError(t, err) + require.Equal(t, bundle, persisted) +} + +func TestQueueExternalReplyValidation(t *testing.T) { + svc, _, acc := newStudio() + uid := int64(4_001_082) + addAccount(acc, uid, "acc1", "replying") + + _, err := svc.QueueExternalReply(context.Background(), uid, "acc1", "media_123", "reply", "") + require.ErrorIs(t, err, domain.ErrValidation) + _, err = svc.QueueExternalReply(context.Background(), uid, "missing", "123", "reply", "") + require.ErrorIs(t, err, domain.ErrNoAccount) + acc.byID["acc1"].IsUsable = false + _, err = svc.QueueExternalReply(context.Background(), uid, "acc1", "123", "reply", "") + require.ErrorIs(t, err, domain.ErrNoAccount) +} + func TestPL_09_RemovePlayKeepsOutbox(t *testing.T) { svc, _, acc := newStudio() uid := int64(4_001_009) diff --git a/apps/backend/internal/module/studio/usecase/service.go b/apps/backend/internal/module/studio/usecase/service.go index 05f70b4..11a6984 100644 --- a/apps/backend/internal/module/studio/usecase/service.go +++ b/apps/backend/internal/module/studio/usecase/service.go @@ -1666,6 +1666,43 @@ func (s *Service) PublishSingle(ctx context.Context, ownerUID int64, accountID, return bundle, nil } +// QueueExternalReply persists an immediate reply to an external Threads media ID. +func (s *Service) QueueExternalReply(ctx context.Context, ownerUID int64, accountID, replyToMediaID, text, title string) (*domain.OutboxBundle, error) { + replyToMediaID = strings.TrimSpace(replyToMediaID) + if replyToMediaID == "" { + return nil, fmt.Errorf("%w: reply_to_media_id must be numeric", domain.ErrValidation) + } + for i := 0; i < len(replyToMediaID); i++ { + if replyToMediaID[i] < '0' || replyToMediaID[i] > '9' { + return nil, fmt.Errorf("%w: reply_to_media_id must be numeric", domain.ErrValidation) + } + } + if accountID == "" { + return nil, domain.ErrNoAccount + } + if s.Accounts != nil { + acc, err := s.Accounts.Get(ctx, accountID) + if err != nil || acc == nil || acc.OwnerUID != ownerUID || !acc.IsUsable { + return nil, domain.ErrNoAccount + } + } + + now := domain.NowNano() + bundle := &domain.OutboxBundle{ + ID: "outbox_" + uuid.NewString()[:12], OwnerUID: ownerUID, Title: title, + Status: domain.OBScheduling, ReplyToMediaID: replyToMediaID, CreatedAt: now, UpdatedAt: now, + Steps: []domain.OutboxStep{{ + ID: "obstep_" + uuid.NewString()[:10], SortOrder: 0, Kind: domain.StepReply, + AccountID: accountID, Text: text, Status: domain.StepScheduled, + ScheduledAt: now, ReplyTo: replyToMediaID, + }}, + } + if err := s.Repo.SaveOutbox(ctx, bundle); err != nil { + return nil, err + } + return bundle, nil +} + func normalizePublishTopicTag(raw string) string { s := strings.TrimSpace(raw) s = strings.TrimPrefix(s, "#") diff --git a/apps/backend/internal/module/threads/provider/unavailable.go b/apps/backend/internal/module/threads/provider/unavailable.go new file mode 100644 index 0000000..a8f2f79 --- /dev/null +++ b/apps/backend/internal/module/threads/provider/unavailable.go @@ -0,0 +1,33 @@ +package provider + +import ( + "context" + "fmt" + + "apps/backend/internal/module/threads/domain" +) + +// UnavailableProvider keeps the service operational when a deployment has not +// configured Threads yet. Every provider operation fails explicitly; it never +// creates a synthetic OAuth account or token. +type UnavailableProvider struct { + reason string +} + +func NewUnavailable(reason string) *UnavailableProvider { + return &UnavailableProvider{reason: reason} +} + +func (p *UnavailableProvider) Name() string { return "unavailable" } + +func (p *UnavailableProvider) AuthorizeURL(string, string) string { return "" } + +func (p *UnavailableProvider) ExchangeCode(context.Context, string, string) (*domain.TokenPair, error) { + return nil, fmt.Errorf("Threads OAuth is not configured: %s", p.reason) +} + +func (p *UnavailableProvider) Refresh(context.Context, string) (*domain.TokenPair, error) { + return nil, fmt.Errorf("Threads OAuth is not configured: %s", p.reason) +} + +var _ domain.Provider = (*UnavailableProvider)(nil) diff --git a/apps/backend/internal/module/threads/usecase/service.go b/apps/backend/internal/module/threads/usecase/service.go index d268e8a..0e37ea2 100644 --- a/apps/backend/internal/module/threads/usecase/service.go +++ b/apps/backend/internal/module/threads/usecase/service.go @@ -71,6 +71,9 @@ func (s *Service) OAuthStart(ctx context.Context, ownerUID int64) (authorizeURL, if ownerUID <= 0 { return "", "", domain.ErrForbidden } + if s.Provider == nil || s.Provider.Name() == "unavailable" { + return "", "", fmt.Errorf("Threads OAuth is not configured") + } state = randomState() now := domain.NowNano() st := &domain.OAuthState{ diff --git a/apps/backend/internal/module/usage/usecase/platform_gate.go b/apps/backend/internal/module/usage/usecase/platform_gate.go index fa107aa..857b4cb 100644 --- a/apps/backend/internal/module/usage/usecase/platform_gate.go +++ b/apps/backend/internal/module/usage/usecase/platform_gate.go @@ -1,10 +1,13 @@ package usecase import ( + "fmt" "sync" "time" "apps/backend/internal/module/usage/domain" + + "github.com/zeromicro/go-zero/core/stores/redis" ) // PlatformGate decides whether platform-paid keys still have capacity. @@ -94,3 +97,67 @@ type AlwaysAllowPlatform struct{} func (AlwaysAllowPlatform) AllowPlatform(string) bool { return true } func (AlwaysAllowPlatform) MarkLimited(time.Time) {} + +// RedisPlatformGate coordinates rate limits and provider cooldowns across all +// gateway and worker instances. Redis failures deny platform-paid calls rather +// than bypassing a shared safety limit. +type RedisPlatformGate struct { + client *redis.Redis + aiPerMin int + searchPerMin int +} + +func NewRedisPlatformGate(conf redis.RedisConf, aiPerMin, searchPerMin int) *RedisPlatformGate { + return &RedisPlatformGate{ + client: redis.MustNewRedis(conf), aiPerMin: aiPerMin, searchPerMin: searchPerMin, + } +} + +func (g *RedisPlatformGate) MarkLimited(until time.Time) { + if g == nil || g.client == nil { + return + } + key := "haixun:usage:platform:limited" + if until.IsZero() || !until.After(time.Now()) { + _, _ = g.client.Del(key) + return + } + seconds := int(time.Until(until).Seconds()) + if seconds < 1 { + seconds = 1 + } + _ = g.client.Setex(key, "1", seconds) +} + +func (g *RedisPlatformGate) AllowPlatform(meter string) bool { + if g == nil || g.client == nil { + return false + } + limit, bucket := g.aiPerMin, "ai" + if meter == domain.MeterWebSearch { + limit, bucket = g.searchPerMin, "search" + } + if limit <= 0 { + limited, err := g.client.Exists("haixun:usage:platform:limited") + return err == nil && !limited + } + window := time.Now().UTC().Format("200601021504") + result, err := g.client.Eval(` +if redis.call('EXISTS', KEYS[1]) == 1 then return 0 end +local n = redis.call('INCR', KEYS[2]) +if n == 1 then redis.call('EXPIRE', KEYS[2], 120) end +if n > tonumber(ARGV[1]) then return 0 end +return 1 +`, []string{"haixun:usage:platform:limited", fmt.Sprintf("haixun:usage:platform:%s:%s", bucket, window)}, limit) + if err != nil { + return false + } + switch v := result.(type) { + case int64: + return v == 1 + case string: + return v == "1" + default: + return false + } +} diff --git a/apps/backend/internal/svc/service_context.go b/apps/backend/internal/svc/service_context.go index 5a13f4d..46c6a0d 100644 --- a/apps/backend/internal/svc/service_context.go +++ b/apps/backend/internal/svc/service_context.go @@ -16,6 +16,8 @@ import ( fsDomain "apps/backend/internal/module/filestorage/domain" "apps/backend/internal/module/filestorage/noop" "apps/backend/internal/module/filestorage/s3store" + inspireRepo "apps/backend/internal/module/inspire/repository" + inspireUC "apps/backend/internal/module/inspire/usecase" jobRepo "apps/backend/internal/module/job/repository" jobUC "apps/backend/internal/module/job/usecase" memberDomain "apps/backend/internal/module/member/domain" @@ -23,11 +25,9 @@ import ( memberUC "apps/backend/internal/module/member/usecase" notifDomain "apps/backend/internal/module/notification/domain" notifUC "apps/backend/internal/module/notification/usecase" - "apps/backend/internal/module/search" - inspireRepo "apps/backend/internal/module/inspire/repository" - inspireUC "apps/backend/internal/module/inspire/usecase" scoutRepo "apps/backend/internal/module/scout/repository" scoutUC "apps/backend/internal/module/scout/usecase" + "apps/backend/internal/module/search" studioDomain "apps/backend/internal/module/studio/domain" studioPublish "apps/backend/internal/module/studio/publish" studioRepo "apps/backend/internal/module/studio/repository" @@ -111,18 +111,8 @@ func NewServiceContext(c config.Config) *ServiceContext { PlatformOpenCode: c.Platform.OpenCodeKey, PlatformExa: c.Platform.ExaKey, } - // Dev: empty platform keys still allow PrepareCall for quota path; real LLM needs real key - if keyRes.PlatformAI == "" && keyRes.PlatformXAI == "" && keyRes.PlatformOpenCode == "" { - keyRes.PlatformAI = "fake-platform-ai" - logx.Info("usage: no platform AI keys — using fake-platform-ai for quota only (persona LLM needs real key)") - } - if keyRes.PlatformExa == "" { - keyRes.PlatformExa = "fake-platform-exa" - logx.Info("usage: Platform.ExaKey empty — using fake-platform-exa for demo/proxy") - } usageSvc := usageUC.New(usageRepo.NewMonStore(c.Mongo.URI, c.Mongo.Database), keyRes) - // 平台 key 容量：預設每分鐘 AI 120／搜尋 60；超限或 MarkPlatformLimited 後僅 BYOK 可用 - usageSvc.Gate = usageUC.NewMemoryPlatformGate(120, 60) + usageSvc.Gate = usageUC.NewRedisPlatformGate(c.CacheRedis[0].RedisConf, 120, 60) // M4 studio: mon store + publish transport（有 Meta 憑證則真發文／同步） var pub studioPublish.Transport @@ -132,19 +122,12 @@ func NewServiceContext(c config.Config) *ServiceContext { mediaSource = &metaMediaBridge{Meta: threadsProv.NewMeta(c.Platform.ThreadsAppId, c.Platform.ThreadsAppSecret)} logx.Info("studio: meta publish + own-posts sync enabled") } else { - pub = studioPublish.NewFake() - logx.Info("studio: fake publish transport (no Threads app credentials)") + pub = studioPublish.NewUnavailable("set THREADS_APP_ID and THREADS_APP_SECRET") + logx.Info("studio: Threads publishing disabled until credentials are configured") } aiRegistry := ai.NewRegistry() - aiClient := &ai.FakeClient{} // tests / synthetic keys only - var searchClient search.Client - if strings.TrimSpace(c.Platform.ExaKey) != "" { - searchClient = search.NewExa() - logx.Info("search: real Exa client") - } else { - searchClient = &search.FakeClient{} - logx.Info("search: FakeClient (no Platform.ExaKey)") - } + aiClient, _ := aiRegistry.Client(ai.ProviderXAI) + searchClient := search.NewExa() studioSvc := studioUC.New(studioRepo.NewMonStore(c.Mongo.URI, c.Mongo.Database), pub) studioSvc.Accounts = threadsSvc // *threadsUC.Service implements AccountLookup studioSvc.Usage = usageSvc @@ -171,9 +154,11 @@ func NewServiceContext(c config.Config) *ServiceContext { } scoutSvc := scoutUC.New(scoutRepo.NewMonStore(c.Mongo.URI, c.Mongo.Database)) scoutSvc.Settings = &devModeFromMembers{Members: repo} - scoutSvc.Transport = pub - scoutSvc.Accounts = threadsSvc scoutSvc.AI = aiClient + scoutSvc.ReplyQueue = &scoutReplyQueue{Studio: studioSvc} + scoutSvc.Provider = scoutUC.NewExaThreadsProvider(c.Platform.ExaKey) + scoutSvc.SessionSecret = c.Scout.SessionSecret + scoutSvc.Crawler = scoutUC.NewHTTPCrawlerProvider(c.Scout.CrawlerEndpoint, c.Scout.CrawlerToken) // 人設對標帳號分析：可用 Chrome 同步的 crawler session 讀公開頁 studioSvc.Crawler = scoutSvc // 靈感 chat 注入真人人設 + 品牌產品目錄 @@ -212,6 +197,16 @@ type devModeFromMembers struct { Members memberDomain.Repository } +type scoutReplyQueue struct{ Studio *studioUC.Service } + +func (q *scoutReplyQueue) QueueExternalReply(ctx context.Context, ownerUID int64, accountID, replyToMediaID, text, title string) (string, error) { + bundle, err := q.Studio.QueueExternalReply(ctx, ownerUID, accountID, replyToMediaID, text, title) + if err != nil { + return "", err + } + return bundle.ID, nil +} + func (d *devModeFromMembers) DevModeEnabled(ctx context.Context, uid int64) (bool, error) { if d == nil || d.Members == nil { return false, nil @@ -244,9 +239,6 @@ func findExtensionZip() string { func newThreads(c config.Config) *threadsUC.Service { repo := threadsRepo.NewMonStore(c.Mongo.URI, c.Mongo.Database) secret := c.Auth.AccessSecret - if secret == "" { - secret = "dev-token-secret" - } web := strings.TrimRight(c.PublicWebBase, "/") if web == "" { web = "http://127.0.0.1:5173" @@ -273,9 +265,8 @@ func newThreads(c config.Config) *threadsUC.Service { logx.Error("threads oauth: callback is HTTP — Meta will block (use https PublicAPIBase / PublicWebBase)") } } else { - cb := apiBase + "/api/v1/threads-accounts/oauth/callback" - provider = &threadsProv.FakeProvider{CallbackBase: cb} - logx.Info("threads oauth: fake provider (set Platform.ThreadsAppId/Secret for Meta)") + provider = threadsProv.NewUnavailable("set THREADS_APP_ID and THREADS_APP_SECRET") + logx.Info("threads oauth disabled until credentials are configured") } svc := threadsUC.New(repo, provider, secret) svc.APIPublicBase = apiBase @@ -602,7 +593,7 @@ func (b *inspireBrandBridge) ListCatalog(ctx context.Context, ownerUID int64) ([ } snap.Products = append(snap.Products, inspireUC.ProductSnapshot{ ID: pr.ID, Label: pr.Label, Context: pr.ProductContext, - Tags: append([]string{}, pr.MatchTags...), + Tags: append([]string{}, pr.MatchTags...), Pains: append([]string{}, pr.PainPoints...), }) } @@ -610,4 +601,3 @@ func (b *inspireBrandBridge) ListCatalog(ctx context.Context, ownerUID int64) ([ } return out, nil } - diff --git a/apps/backend/internal/types/m5_convert.go b/apps/backend/internal/types/m5_convert.go index 2207f76..c7cc695 100644 --- a/apps/backend/internal/types/m5_convert.go +++ b/apps/backend/internal/types/m5_convert.go @@ -114,7 +114,8 @@ func ScoutPostFromDomain(p *scoutDomain.Post) ScoutPostPublic { Opportunity: p.Opportunity, OutreachStatus: p.OutreachStatus, DraftText: p.DraftText, Score: p.Score, MatchedProductId: p.MatchedProductID, MatchedProductLabel: p.MatchedProductLabel, MatchReason: p.MatchReason, ScoutMode: p.ScoutMode, IntentSnippet: p.IntentSnippet, - ThemeKey: p.ThemeKey, ThemeLabel: p.ThemeLabel, ScanPath: p.ScanPath, CreatedAt: p.CreatedAt, + ThemeKey: p.ThemeKey, ThemeLabel: p.ThemeLabel, ScanPath: p.ScanPath, + Classification: p.Classification, Permalink: p.Permalink, CreatedAt: p.CreatedAt, } } diff --git a/apps/backend/internal/types/types.go b/apps/backend/internal/types/types.go index b3aae1a..77ec254 100644 --- a/apps/backend/internal/types/types.go +++ b/apps/backend/internal/types/types.go @@ -1127,9 +1127,15 @@ type ScoutPostPublic struct { ThemeKey string `json:"theme_key,optional"` ThemeLabel string `json:"theme_label,optional"` ScanPath string `json:"scan_path,optional"` + Classification string `json:"classification,optional"` + Permalink string `json:"permalink,optional"` CreatedAt int64 `json:"created_at"` } +type ScoutScanJobData struct { + Job JobPublic `json:"job"` +} + type ScoutScanReq struct { Brief ScoutBriefPublic `json:"brief"` } diff --git a/apps/extension/haixun-threads-sync/service-worker.js b/apps/extension/haixun-threads-sync/service-worker.js index 54a0c7f..5b75348 100644 --- a/apps/extension/haixun-threads-sync/service-worker.js +++ b/apps/extension/haixun-threads-sync/service-worker.js @@ -2,9 +2,6 @@ import { buildStorageState } from "./storage-state.js"; const CONTENT_SCRIPT_ID = "haixun-bridge"; const GO_API_SUCCESS_CODE = 102000; -/** 前端 dev ports → gateway 預設 8888（舊 8890 仍相容） */ -const DEV_WEB_PORTS = new Set(["3000", "4173", "5173"]); -const DEV_API_PORTS = ["8888", "8890"]; function unwrapGoApiResponse(raw) { if (raw && typeof raw === "object" && "code" in raw) { @@ -36,16 +33,9 @@ function equivalentOrigins(origin) { return [...origins]; } -/** - * 開發前端 :5173 等 → 後端 gateway :8888；正式部署 same-origin。 - */ +/** The extension uses the page's /api proxy in every environment. */ function resolveApiBases(serverUrl) { const url = new URL(serverUrl); - if (DEV_WEB_PORTS.has(url.port)) { - const host = url.hostname === "localhost" ? "127.0.0.1" : url.hostname; - return DEV_API_PORTS.map((p) => `${url.protocol}//${host}:${p}`); - } - // 遠端 / reverse-proxy：同源即可 return [url.origin]; } @@ -393,4 +383,3 @@ chrome.runtime.onStartup.addListener(() => { // service worker 冷啟動也試一次 void bootstrapBridgeFromStorage(); - diff --git a/apps/web/.env.example b/apps/web/.env.example index 6005dfb..5b2f882 100644 --- a/apps/web/.env.example +++ b/apps/web/.env.example @@ -4,7 +4,8 @@ # 遠端開 FE（threads-tool-dev.30cm.net）時必須這樣， # 不要寫死 http://127.0.0.1:8888（那會打到使用者自己的電腦）。 # -# 本機若要直連 gateway（跳過 proxy）： +# 本機若要直連 gateway（跳過 proxy）：須將這個前端 origin 加入 +# backend 的 CORS_ALLOWED_ORIGINS；HTTPS 頁面不可直連 HTTP gateway。 # VITE_API_BASE=http://127.0.0.1:8888 # # 走 proxy（推薦）： diff --git a/apps/web/src/data/live/m5Repos.ts b/apps/web/src/data/live/m5Repos.ts index 0a4c313..ba2d89e 100644 --- a/apps/web/src/data/live/m5Repos.ts +++ b/apps/web/src/data/live/m5Repos.ts @@ -5,6 +5,7 @@ import type { Brand, BrandProduct, InspirationIdea, + Job, InspireAngle, InspireChatMessage, InspireElement, @@ -180,6 +181,27 @@ function mapScoutPost(raw: Record): ScoutPost { intent_snippet: raw.intent_snippet != null ? String(raw.intent_snippet) : undefined, theme_key: raw.theme_key != null ? String(raw.theme_key) : undefined, theme_label: raw.theme_label != null ? String(raw.theme_label) : undefined, + scan_path: raw.scan_path != null ? String(raw.scan_path) : undefined, + classification: raw.classification != null ? String(raw.classification) : undefined, + permalink: raw.permalink != null ? String(raw.permalink) : undefined, + created_at: raw.created_at != null ? Number(raw.created_at) : undefined, + }; +} + +function mapScanJob(raw: Record): Job { + return { + id: String(raw.id ?? ""), + template_type: String(raw.template_type ?? "scout_scan"), + status: (raw.status as Job["status"]) || "pending", + progress_summary: String(raw.progress_summary ?? ""), + progress_percent: Number(raw.progress_percent ?? 0), + error: raw.error != null ? String(raw.error) : undefined, + ref_id: raw.ref_id != null ? String(raw.ref_id) : undefined, + payload: raw.payload != null ? String(raw.payload) : undefined, + run_after: raw.run_after != null ? Number(raw.run_after) : undefined, + created_at: Number(raw.created_at ?? 0), + updated_at: Number(raw.updated_at ?? raw.created_at ?? 0), + completed_at: raw.completed_at != null ? Number(raw.completed_at) : undefined, }; } @@ -633,11 +655,13 @@ export function createLiveScout(): ScoutRepo { return mapBrief(raw); }, async runScanFromBrief(brief) { - const data = await apiRequest<{ list: Record[] }>("/api/v1/scout/scan", { + const data = await apiRequest<{ job?: Record }>("/api/v1/scout/scan", { method: "POST", body: { brief: briefToBody(brief) }, }); - return (data.list ?? []).map(mapScoutPost); + const job = mapScanJob(data.job ?? {}); + if (!job.id) throw new Error("海巡任務未建立"); + return { job }; }, async getScanContext(brandId, productId) { const brief = await this.prepareBrief({ diff --git a/apps/web/src/data/repos.ts b/apps/web/src/data/repos.ts index 10eb7bf..78af976 100644 --- a/apps/web/src/data/repos.ts +++ b/apps/web/src/data/repos.ts @@ -537,8 +537,8 @@ export type ScoutRepo = { */ deep?: boolean; }): Promise; - /** 依 brief（含使用者勾選後的 scan_terms）海巡 */ - runScanFromBrief(brief: ScoutRunBrief): Promise; + /** 依 brief（含使用者勾選後的 scan_terms）建立背景海巡任務。 */ + runScanFromBrief(brief: ScoutRunBrief): Promise<{ job: Job }>; /** @deprecated 用 prepareBrief */ getScanContext(brandId: string, productId?: string | null): Promise; /** 列出命中；brandId 空 = 全部（含主題模式） */ @@ -548,7 +548,7 @@ export type ScoutRepo = { brandId: string, productId?: string | null, extraTerms?: string[], - ): Promise; + ): Promise<{ job: Job }>; draftOutreach(postId: string, personaId?: string): Promise; skipOutreach(postId: string): Promise; markPublished(postId: string): Promise; diff --git a/apps/web/src/domain/types.ts b/apps/web/src/domain/types.ts index 3ac7d27..77b67dd 100644 --- a/apps/web/src/domain/types.ts +++ b/apps/web/src/domain/types.ts @@ -647,7 +647,7 @@ export type ScoutTopic = { preferred_product_id?: string | null; }; -export type ScoutOutreachStatus = "new" | "drafted" | "published" | "skipped"; +export type ScoutOutreachStatus = "new" | "drafted" | "queued" | "published" | "skipped"; /** * 探查模式： @@ -685,6 +685,14 @@ export type ScoutPost = { theme_key?: string; /** 分組顯示名：主題／產品／活躍關鍵字 */ theme_label?: string; + /** 掃描來源，例如 api 或 crawler */ + scan_path?: string; + /** 後端分類結果；舊資料或目前契約未提供時省略 */ + classification?: string; + /** 原始 Threads 貼文連結 */ + permalink?: string; + /** 掃描命中的建立時間（UTC unix nanoseconds） */ + created_at?: number; }; /** diff --git a/apps/web/src/lib/i18n/messages.ts b/apps/web/src/lib/i18n/messages.ts index 3c3c9f3..301351b 100644 --- a/apps/web/src/lib/i18n/messages.ts +++ b/apps/web/src/lib/i18n/messages.ts @@ -1031,6 +1031,7 @@ export const zhTW: MessageDict = { "scout.skip": "略過", "scout.regen": "再產", "scout.send": "發送", + "scout.resend": "重新送出", "scout.sending": "發送中…", "scout.needAccountBefore": "請先到", "scout.needAccountAfter": "連線可用帳號。", @@ -1049,7 +1050,10 @@ export const zhTW: MessageDict = { "scout.focus": "焦點", "scout.all": "全部", "scout.noWebSummary": "尚無網頁摘要", - "scout.queue": "佇列 · {n}", + "scout.queue": "命中紀錄 · {n}", + "scout.valueQueue": "痛點／產品接話 · {n}", + "scout.activityQueue": "活躍短回 · {n}", + "scout.noMatchesInQueue": "這個佇列暫無命中", "scout.collapseQueue": "收合佇列", "scout.expandQueue": "展開佇列", "scout.noOtherPending": "沒有其他待回", @@ -1069,13 +1073,32 @@ export const zhTW: MessageDict = { "scout.newRunValue": "新批次「{label}」· {n} 則 · 請處理「現在這一則」", "scout.knowledgeReady": "「{label}」周邊知識已備好 · {n} 則可學", "scout.patrolFail": "這輪海巡失敗", + "scout.loadFail": "海巡資料載入失敗，請重新整理後再試。", + "scout.scanQueued": "已建立海巡任務「{label}」，完成後會自動載入命中。", + "scout.workerWaiting": "海巡任務仍在等待 worker。請確認 apps/backend worker 正在執行。", + "scout.scanReady": "海巡完成，找到 {n} 則待回。", + "scout.queued": "已交由 @{who} 的 Outbox 發送佇列處理 · 今日 {done}/{goal}", + "scout.scanJob": "海巡任務", + "scout.scanInProgress": "正在掃描", + "scout.crawlerSessionRequired": "海巡爬蟲需要有效的 Chrome session。請在設定頁同步已登入 Threads 分頁的 session。", + "scout.openSettings": "開啟設定", + "scout.source": "來源：{source}", + "scout.classification": "分類：{classification}", + "scout.createdAt": "建立於 {time}", + "scout.openPermalink": "開啟原文", "scout.draftFail": "產草稿失敗", "scout.skipped": "已略過", + "scout.status.new": "待處理", + "scout.status.drafted": "已起草", + "scout.status.queued": "發送佇列中", + "scout.status.published": "已發送", + "scout.status.skipped": "已略過", "scout.noDraft": "沒有可發送的草稿", "scout.accountFallback": "帳號", "scout.sent": "已發送（@{who}）· 今日 {done}/{goal}", "scout.sendFail": "發送失敗", "scout.confirmDeletePost": "刪除這則命中？", + "scout.deletedPost": "已刪除這則命中", "scout.stanceActivity": "短回 · 養活躍", "scout.stanceProduct": "共感 · 可輕帶產品", "scout.stanceRelation": "接話 · 建關係", @@ -2685,6 +2708,7 @@ export const en: MessageDict = { "scout.skip": "Skip", "scout.regen": "Regen", "scout.send": "Send", + "scout.resend": "Resend", "scout.sending": "Sending…", "scout.needAccountBefore": "Connect a usable account under", "scout.needAccountAfter": "first.", @@ -2703,7 +2727,10 @@ export const en: MessageDict = { "scout.focus": "Focus", "scout.all": "All", "scout.noWebSummary": "No web summaries yet", - "scout.queue": "Queue · {n}", + "scout.queue": "Matches · {n}", + "scout.valueQueue": "Value replies · {n}", + "scout.activityQueue": "Activity short replies · {n}", + "scout.noMatchesInQueue": "No matches in this queue", "scout.collapseQueue": "Collapse queue", "scout.expandQueue": "Expand queue", "scout.noOtherPending": "No other pending", @@ -2723,13 +2750,32 @@ export const en: MessageDict = { "scout.newRunValue": "New batch “{label}” · {n} posts · handle “Now this post”", "scout.knowledgeReady": "“{label}” knowledge ready · {n} notes", "scout.patrolFail": "This patrol failed", + "scout.loadFail": "Scout data could not be loaded. Refresh and try again.", + "scout.scanQueued": "Patrol job “{label}” queued. Matches will load automatically when it completes.", + "scout.workerWaiting": "The patrol job is still waiting for a worker. Check that apps/backend worker is running.", + "scout.scanReady": "Patrol complete. Found {n} pending replies.", + "scout.queued": "Queued in @{who}'s Outbox for delivery · today {done}/{goal}", + "scout.scanJob": "Patrol job", + "scout.scanInProgress": "Scanning", + "scout.crawlerSessionRequired": "The patrol crawler needs a valid Chrome session. Sync the session from a signed-in Threads tab in Settings.", + "scout.openSettings": "Open Settings", + "scout.source": "Source: {source}", + "scout.classification": "Class: {classification}", + "scout.createdAt": "Created {time}", + "scout.openPermalink": "Open original", "scout.draftFail": "Draft failed", "scout.skipped": "Skipped", + "scout.status.new": "To handle", + "scout.status.drafted": "Drafted", + "scout.status.queued": "Queued", + "scout.status.published": "Sent", + "scout.status.skipped": "Skipped", "scout.noDraft": "No draft to send", "scout.accountFallback": "account", "scout.sent": "Sent (@{who}) · today {done}/{goal}", "scout.sendFail": "Send failed", "scout.confirmDeletePost": "Delete this hit?", + "scout.deletedPost": "Deleted this hit", "scout.stanceActivity": "Short reply · activity", "scout.stanceProduct": "Empathy · soft product", "scout.stanceRelation": "Engage · build rapport", diff --git a/apps/web/src/pages/ScoutPage.tsx b/apps/web/src/pages/ScoutPage.tsx index 9b1bf71..71eccba 100644 --- a/apps/web/src/pages/ScoutPage.tsx +++ b/apps/web/src/pages/ScoutPage.tsx @@ -1,4 +1,4 @@ -import { useEffect, useMemo, useState } from "react"; +import { useEffect, useEffectEvent, useMemo, useState } from "react"; import { Link } from "react-router-dom"; import { PageHeader } from "../components/layout/PageHeader"; import { Badge, Button, Card, EmptyState, Input, Pager, Select, Textarea } from "../components/ui"; @@ -7,7 +7,6 @@ import { useData, useRepos } from "../data/DataContext"; import type { Brand, BrandProduct, - Persona, ScoutHomeworkRecord, ScoutPost, ScoutPurpose, @@ -22,16 +21,78 @@ import { noteReplyHooks, formatResearchLink, } from "../lib/mockResearch"; -import { isPersonaReady, personaOptionLabel } from "../lib/personaPrompt"; import { newId } from "../lib/id"; import { bumpScoutTodayDone, loadScoutToday, saveScoutToday } from "../lib/scoutToday"; import { nowUnixNano } from "../lib/time"; import { useI18n } from "../i18n/I18nContext"; +import { useJobLive } from "../data/JobLiveContext"; +import { jobStatusLabel, jobStatusTone } from "../lib/jobLabels"; +import { formatLocalDateTime } from "../lib/time"; +import type { Job } from "../domain/types"; function isPending(p: ScoutPost): boolean { return p.outreach_status === "new" || p.outreach_status === "drafted"; } +function canSend(p: ScoutPost): boolean { + return isPending(p) || p.outreach_status === "queued"; +} + +function outreachStatusLabel( + p: ScoutPost, + t: (k: string, p?: Record) => string, +): string { + return t(`scout.status.${p.outreach_status}`); +} + +function MatchQueue({ + title, + posts, + page, + onPageChange, + currentId, + onSelect, + t, +}: { + title: string; + posts: ScoutPost[]; + page: number; + onPageChange: (page: number) => void; + currentId: string | null; + onSelect: (id: string) => void; + t: (k: string, p?: Record) => string; +}) { + const pageSize = 8; + return ( + + {posts.length === 0 ? ( + + ) : ( +
+ {pageSlice(posts, page, pageSize).map((p) => ( + + ))} + +
+ )} + + ); +} + function stanceOf(p: ScoutPost, t: (k: string, p?: Record) => string): string { if (p.scout_mode === "activity") return t("scout.stanceActivity"); if (p.scout_mode === "product" || p.matched_product_label) return t("scout.stanceProduct"); @@ -143,19 +204,18 @@ function KnowledgeNoteCard({ */ export function ScoutPage() { const repos = useRepos(); - const { refresh, tick } = useData(); + const { tick } = useData(); const { t } = useI18n(); + const { jobs: liveJobs, reload: reloadJobs } = useJobLive(); const [brands, setBrands] = useState([]); const [allProducts, setAllProducts] = useState([]); - const [personas, setPersonas] = useState([]); const [accounts, setAccounts] = useState([]); const [posts, setPosts] = useState([]); const [purpose, setPurpose] = useState("value"); const [intent, setIntent] = useState(""); const [productId, setProductId] = useState(""); - const [personaId, setPersonaId] = useState(""); const [accountId, setAccountId] = useState(""); const [goal, setGoal] = useState(8); @@ -166,18 +226,19 @@ export function ScoutPage() { const [report, setReport] = useState(null); const [reportOpen, setReportOpen] = useState(false); - const [queueOpen, setQueueOpen] = useState(false); - const [queuePage, setQueuePage] = useState(1); + const [valueQueuePage, setValueQueuePage] = useState(1); + const [activityQueuePage, setActivityQueuePage] = useState(1); const [researchTier, setResearchTier] = useState<"all" | ScoutResearchTier>("all"); - const QUEUE_PAGE = 8; const [homeworkList, setHomeworkList] = useState([]); - /** 背景補完整功課（不擋主流程）；綁定批次 key */ - const [homeworkLoadingKey, setHomeworkLoadingKey] = useState(null); /** 目前檢視的海巡批次（每次按開始 = 一筆） */ const [activeRunKey, setActiveRunKey] = useState(null); const [busy, setBusy] = useState(""); const [message, setMessage] = useState(""); + const [scanJob, setScanJob] = useState(null); + const [scanJobThemeKey, setScanJobThemeKey] = useState(null); + const [crawlerSessionRequired, setCrawlerSessionRequired] = useState(false); + const [loadError, setLoadError] = useState(""); useEffect(() => { const t = loadScoutToday(); @@ -187,16 +248,15 @@ export function ScoutPage() { useEffect(() => { void (async () => { - const [b, p, hw, prods, postList, acc] = await Promise.all([ - repos.scout.listBrands(), - repos.personas.list(), + try { + const [b, hw, prods, postList, acc] = await Promise.all([ + repos.scout.listBrands(), repos.scout.listHomework(), repos.scout.listAllProducts(), repos.scout.listPosts(), repos.accounts.list(), - ]); + ]); setBrands(b); - setPersonas(p); setHomeworkList(hw); setAllProducts(prods); setPosts(postList); @@ -204,24 +264,14 @@ export function ScoutPage() { setAccounts(usable); if (!accountId && usable[0]) setAccountId(usable[0].id); - const pending = postList.filter(isPending).sort((a, b) => (b.score || 0) - (a.score || 0)); - if (pending[0]) { - setCurrentId((cur) => cur || pending[0]!.id); - setActiveRunKey((cur) => cur || postRunKey(pending[0]!)); - } else if (postList[0]) { - setActiveRunKey((cur) => cur || postRunKey(postList[0]!)); - } - - const lastValue = hw.find((h) => h.purpose === "value"); - if (lastValue && !report) { - setReport(lastValue.brief); - setActiveRunKey((cur) => cur || lastValue.theme_key); - if (lastValue.brief.intent) setIntent((prev) => prev || lastValue.brief.intent); - const pid = lastValue.brief.product_id || ""; - if (pid && prods.some((x) => x.id === pid)) setProductId(pid); + const pending = postList.filter(isPending).sort((a, b) => (b.score || 0) - (a.score || 0)); + if (pending[0]) setActiveRunKey((cur) => cur || postRunKey(pending[0]!)); + setLoadError(""); + } catch (e) { + setLoadError(e instanceof Error ? e.message : t("scout.loadFail")); } })(); - }, [repos, tick]); // eslint-disable-line react-hooks/exhaustive-deps + }, [repos, t, tick]); const productOptions = useMemo(() => { return allProducts.map((p) => { @@ -283,28 +333,82 @@ export function ScoutPage() { [runGroups, activeRunKey], ); - const pendingQueue = useMemo(() => { + const runQueue = useMemo(() => { return posts - .filter(isPending) .filter((p) => !activeRunKey || postRunKey(p) === activeRunKey) .slice() - .sort((a, b) => (b.score || 0) - (a.score || 0)); + .sort((a, b) => Number(isPending(b)) - Number(isPending(a)) || (b.score || 0) - (a.score || 0)); }, [posts, activeRunKey]); + const pendingQueue = useMemo(() => runQueue.filter(isPending), [runQueue]); + const valueQueue = useMemo(() => runQueue.filter((p) => p.scout_mode !== "activity"), [runQueue]); + const activityQueue = useMemo(() => runQueue.filter((p) => p.scout_mode === "activity"), [runQueue]); + const current = useMemo( () => posts.find((p) => p.id === currentId) || null, [posts, currentId], ); - // 切換當前則時帶入草稿；若跳出目前批次則跟著切批次 + const visibleScanJob = useMemo( + () => (scanJob ? liveJobs.find((job) => job.id === scanJob.id) || scanJob : null), + [liveJobs, scanJob], + ); + + const pollScanJob = useEffectEvent(async (jobId: string, cancelled: () => boolean) => { + try { + await reloadJobs(); + const job = await repos.jobs.get(jobId); + if (cancelled() || !job) return; + setScanJob((previous) => + previous?.id === job.id && + previous.status === job.status && + previous.updated_at === job.updated_at + ? previous + : job, + ); + if (job.status === "succeeded") { + const list = await repos.scout.listPosts(); + if (cancelled()) return; + setPosts(list); + const completedThemeKey = scanJobThemeKey || job.ref_id || activeRunKey; + const pending = list + .filter((p) => postRunKey(p) === completedThemeKey && isPending(p)) + .sort((a, b) => (b.score || 0) - (a.score || 0)); + setMessage(t("scout.scanReady", { n: pending.length })); + } else if (job.status === "failed" || job.status === "cancelled") { + setMessage(job.error || t("scout.patrolFail")); + setCrawlerSessionRequired(/crawler.?session|chrome session/i.test(job.error || "")); + } else if ( + job.status === "queued" && + Number(job.created_at || 0) > 0 && + Date.now() - Number(job.created_at) / 1_000_000 > 12_000 + ) { + setMessage(t("scout.workerWaiting")); + } + } catch { + // The shared JobLiveContext keeps refreshing; leave the last known status visible. + } + }); + + useEffect(() => { + if (!scanJob || ["succeeded", "failed", "cancelled"].includes(scanJob.status)) return; + let cancelled = false; + const poll = () => pollScanJob(scanJob.id, () => cancelled); + void poll(); + const id = window.setInterval(() => void poll(), 1500); + return () => { + cancelled = true; + window.clearInterval(id); + }; + }, [scanJob?.id, scanJob?.status]); + + // 選取貼文只帶入草稿；批次由使用者明確選取，不能反向切換。 useEffect(() => { if (!current) { setDraftText(""); return; } setDraftText(current.draft_text || ""); - const rk = postRunKey(current); - if (rk && rk !== activeRunKey) setActiveRunKey(rk); }, [current?.id]); // eslint-disable-line react-hooks/exhaustive-deps function pickNext(afterId?: string | null, list?: ScoutPost[], runKey?: string | null) { @@ -327,32 +431,21 @@ export function ScoutPage() { function selectRun(key: string) { setActiveRunKey(key); - setQueuePage(1); + setValueQueuePage(1); + setActivityQueuePage(1); const hw = homeworkList.find((h) => h.theme_key === key); if (hw) { setReport(hw.brief); - setPurpose(hw.purpose); - if (hw.brief.intent) setIntent(hw.brief.intent); - if (hw.brief.product_id && allProducts.some((x) => x.id === hw.brief.product_id)) { - setProductId(hw.brief.product_id); - } else if (hw.purpose === "activity") { - setProductId(""); - } setResearchTier( hw.brief.research_notes?.some((n) => (n.tier || "core") === "core") ? "core" : "all", ); } else { - const sample = posts.find((p) => postRunKey(p) === key); - if (sample?.intent_snippet) setIntent((prev) => prev || sample.intent_snippet || ""); - if (sample?.matched_product_id) setProductId(sample.matched_product_id); - if (sample?.scout_mode === "activity") setPurpose("activity"); - // 無功課時不硬清 report（避免 thrash）；若 key 不同再清 - if (report?.theme_key && report.theme_key !== key) setReport(null); + setReport(null); } const pending = posts .filter((p) => postRunKey(p) === key && isPending(p)) .sort((a, b) => (b.score || 0) - (a.score || 0)); - setCurrentId(pending[0]?.id || posts.find((p) => postRunKey(p) === key)?.id || null); + setCurrentId(pending[0]?.id || null); } async function removeRun(key: string) { @@ -370,28 +463,13 @@ export function ScoutPage() { const hw = await repos.scout.listHomework(); setHomeworkList(hw); if (activeRunKey === key) { - const nextFromList = list.find((p) => postRunKey(p) !== key); - const chosen = - (nextFromList ? postRunKey(nextFromList) : null) || - hw.find((h) => h.theme_key !== key)?.theme_key || - null; - setActiveRunKey(chosen); - if (chosen) { - const nextHw = hw.find((h) => h.theme_key === chosen); - setReport(nextHw?.brief || null); - const pending = list - .filter((p) => postRunKey(p) === chosen && isPending(p)) - .sort((a, b) => (b.score || 0) - (a.score || 0)); - setCurrentId(pending[0]?.id || null); - } else { - setReport(null); - setCurrentId(null); - } + setActiveRunKey(null); + setReport(null); + setCurrentId(null); } else if (currentId && !list.some((p) => p.id === currentId)) { pickNext(currentId, list, activeRunKey); } setMessage(t("scout.deletedRun", { label })); - refresh(); } catch (e) { setMessage(e instanceof Error ? e.message : t("scout.deleteRunFail")); } finally { @@ -418,7 +496,7 @@ export function ScoutPage() { } setBusy("run"); setMessage(""); - setHomeworkLoadingKey(null); + setCrawlerSessionRequired(false); try { const freshProducts = await repos.scout.listAllProducts(); setAllProducts(freshProducts); @@ -441,6 +519,10 @@ export function ScoutPage() { const theme_label = `${baseLabel} · ${runTimeSuffix()}`; const briefSaved: ScoutRunBrief = { ...brief, theme_key, theme_label }; + const { job } = await repos.scout.runScanFromBrief({ + ...briefSaved, + scan_terms: briefSaved.scan_terms, + }); await repos.scout.saveHomework({ theme_key, theme_label, @@ -449,80 +531,18 @@ export function ScoutPage() { created_at: nowUnixNano(), }); setHomeworkList(await repos.scout.listHomework()); - setReport(briefSaved); - setReportOpen(false); - setActiveRunKey(theme_key); + setScanJob(job); + setScanJobThemeKey(theme_key); + void reloadJobs(); + setMessage(t("scout.scanQueued", { label: theme_label })); - await repos.scout.runScanFromBrief({ - ...briefSaved, - scan_terms: briefSaved.scan_terms, - }); - const list = await repos.scout.listPosts(); - setPosts(list); - const pending = list - .filter((p) => postRunKey(p) === theme_key && isPending(p)) - .sort((a, b) => (b.score || 0) - (a.score || 0)); - setCurrentId(pending[0]?.id || null); - setMessage( - purpose === "activity" - ? t("scout.newRunActivity", { label: theme_label, n: pending.length }) - : t("scout.newRunValue", { label: theme_label, n: pending.length }), - ); - refresh(); - setBusy(""); - - // ② 痛點模式：背景補周邊知識（綁同一批次 theme_key） - if (purpose === "value") { - setHomeworkLoadingKey(theme_key); - void (async () => { - try { - const deep = await repos.scout.prepareBrief({ - intent: text, - brandId: selected?.brand_id || null, - productId: selected?.id || null, - purpose: "value", - deep: true, - }); - const deepSaved: ScoutRunBrief = { - ...deep, - theme_key, - theme_label, - }; - await repos.scout.saveHomework({ - theme_key, - theme_label, - purpose: "value", - brief: deepSaved, - created_at: nowUnixNano(), - }); - setHomeworkList(await repos.scout.listHomework()); - // 僅在使用者仍看此批次時更新畫面 - setReport((prev) => - !prev || prev.theme_key === theme_key ? deepSaved : prev, - ); - const noteCount = deepSaved.research_notes?.length || 0; - if (noteCount > 0) { - setMessage(t("scout.knowledgeReady", { label: theme_label, n: noteCount })); - // 勿在 setState updater 內再 setState（Strict Mode 可能雙重觸發） - setActiveRunKey((cur) => { - if (cur === theme_key) { - queueMicrotask(() => { - setResearchTier("core"); - setReportOpen(true); - }); - } - return cur; - }); - } - } catch { - // 背景失敗不擋主流程 - } finally { - setHomeworkLoadingKey((k) => (k === theme_key ? null : k)); - } - })(); - } } catch (e) { setMessage(e instanceof Error ? e.message : t("scout.patrolFail")); + setCrawlerSessionRequired( + typeof e === "object" && e !== null && "code" in e && (e as { code?: number }).code === 400061, + ); + setBusy(""); + } finally { setBusy(""); } } @@ -537,10 +557,9 @@ export function ScoutPage() { if (!current) return; setBusy("draft"); try { - const next = await repos.scout.draftOutreach(current.id, personaId || undefined); + const next = await repos.scout.draftOutreach(current.id); await reloadPosts(); setDraftText(next.draft_text || ""); - refresh(); } catch (e) { setMessage(e instanceof Error ? e.message : t("scout.draftFail")); } finally { @@ -556,8 +575,10 @@ export function ScoutPage() { await repos.scout.skipOutreach(id); const list = await reloadPosts(); pickNext(id, list); + setTodayDone(bumpScoutTodayDone().done); setMessage(t("scout.skipped")); - refresh(); + } catch (e) { + setMessage(e instanceof Error ? e.message : t("scout.patrolFail")); } finally { setBusy(""); } @@ -570,7 +591,7 @@ export function ScoutPage() { try { let text = draftText.trim(); if (!text) { - const d = await repos.scout.draftOutreach(current.id, personaId || undefined); + const d = await repos.scout.draftOutreach(current.id); text = (d.draft_text || "").trim(); setDraftText(text); } @@ -580,15 +601,13 @@ export function ScoutPage() { postId: id, text, accountId: accountId || undefined, - personaId: personaId || undefined, }); - const today = bumpScoutTodayDone(1); - setTodayDone(today.done); const list = await reloadPosts(); pickNext(id, list); + const today = bumpScoutTodayDone(); + setTodayDone(today.done); const who = accounts.find((a) => a.id === accountId)?.username || t("scout.accountFallback"); - setMessage(t("scout.sent", { who, done: today.done, goal })); - refresh(); + setMessage(t("scout.queued", { who, done: today.done, goal })); } catch (e) { setMessage(e instanceof Error ? e.message : t("scout.sendFail")); } finally { @@ -605,25 +624,34 @@ export function ScoutPage() { await repos.scout.removePost(id); const list = await reloadPosts(); pickNext(id, list); - refresh(); + setMessage(t("scout.deletedPost")); + } catch (e) { + setMessage(e instanceof Error ? e.message : t("common.error")); } finally { setBusy(""); } } const progressPct = Math.min(100, Math.round((todayDone / Math.max(1, goal)) * 100)); - const queueRest = pendingQueue.filter((p) => p.id !== currentId); return ( <> + {loadError ? : null} + {message ? (

{message}

) : null} + {crawlerSessionRequired ? ( +

+ {t("scout.crawlerSessionRequired")} {t("scout.openSettings")} +

+ ) : null} + {/* ① 今日設定 */}
@@ -730,28 +758,44 @@ export function ScoutPage() {
- {/* ② 海巡批次：單一 spar 選單 + 刪除 */} + {visibleScanJob ? ( + +
+ + {jobStatusLabel(visibleScanJob.status, t)} + + + {visibleScanJob.progress_summary || t("scout.scanInProgress")} + {visibleScanJob.progress_percent ? ` · ${visibleScanJob.progress_percent}%` : ""} + +
+ + ) : null} + + {/* ② 海巡批次：只由使用者選取，不會因 Job 或貼文更新跳動。 */} {runGroups.length > 0 ? (
-
- + + + ))} +
+
{activeRunKey ? (

{stanceOf(current, t)} @@ -788,6 +836,17 @@ export function ScoutPage() {

{current.text}

+ {current.created_at || current.permalink ? ( +

+ {current.created_at ? t("scout.createdAt", { time: formatLocalDateTime(current.created_at) }) : ""} + {current.created_at && current.permalink ? " · " : ""} + {current.permalink ? ( + + {t("scout.openPermalink")} + + ) : null} +

+ ) : null} {current.match_reason ? (

{current.match_reason} @@ -803,11 +862,11 @@ export function ScoutPage() { label={t("scout.draft")} value={draftText} onChange={(e) => setDraftText(e.target.value)} - rows={purpose === "activity" ? 3 : 5} - placeholder={purpose === "activity" ? t("scout.draftPhActivity") : t("scout.draftPhValue")} + rows={current.scout_mode === "activity" ? 3 : 5} + placeholder={current.scout_mode === "activity" ? t("scout.draftPhActivity") : t("scout.draftPhValue")} /> -

+
setPersonaId(e.target.value)} - > - - {personas.map((p) => ( - - ))} -
- ) : homeworkLoadingKey === activeRunKey ? null : ( + ) : (

{t("scout.noWebSummary")}

)} @@ -1058,42 +1102,25 @@ export function ScoutPage() { ) : null} - {/* ⑤ 佇列（目前批次其餘待回） */} - -
- -
- {!queueOpen ? null : queueRest.length === 0 ? ( - - ) : ( -
- {pageSlice(queueRest, queuePage, QUEUE_PAGE).map((p) => ( - - ))} - -
- )} - + {/* ⑤ 不同回覆策略不能混排，避免將短回誤當成痛點接話。 */} + + ); } diff --git a/apps/web/src/pages/SettingsPage.tsx b/apps/web/src/pages/SettingsPage.tsx index fa95aea..db06b99 100644 --- a/apps/web/src/pages/SettingsPage.tsx +++ b/apps/web/src/pages/SettingsPage.tsx @@ -162,6 +162,8 @@ export function SettingsPage() { setExaKey(""); setMessage(t("settings.searchSaved")); refresh(); + } catch (e) { + setError(e instanceof Error ? e.message : t("common.error")); } finally { setBusy(""); } diff --git a/apps/web/src/styles/global.css b/apps/web/src/styles/global.css index d0cff07..af90326 100644 --- a/apps/web/src/styles/global.css +++ b/apps/web/src/styles/global.css @@ -2027,7 +2027,41 @@ svg { font-size: 0.85em; } -/* 海巡批次：單一 spar */ +/* 海巡批次 */ +.hb-scout-run-list { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(13rem, 1fr)); + gap: 0.5rem; +} + +.hb-scout-run-item { + display: flex; + min-width: 0; + flex-direction: column; + align-items: flex-start; + gap: 0.35rem; + padding: 0.7rem 0.8rem; + border: 1px solid var(--hb-line); + border-radius: var(--hb-radius); + background: var(--hb-surface-muted); + color: inherit; + font: inherit; + text-align: left; + cursor: pointer; +} + +.hb-scout-run-item:hover, +.hb-scout-run-item.is-active { + border-color: color-mix(in srgb, var(--hb-brand) 55%, var(--hb-line)); + background: color-mix(in srgb, var(--hb-brand) 9%, var(--hb-surface-muted)); +} + +.hb-scout-run-item__meta { + color: var(--hb-muted); + font-size: 0.75rem; + font-weight: 600; +} + .hb-scout-run-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; @@ -2153,6 +2187,11 @@ svg { border-color: color-mix(in srgb, var(--hb-brand) 40%, var(--hb-line)); } +.hb-scout-queue-item.is-active { + border-color: color-mix(in srgb, var(--hb-brand) 55%, var(--hb-line)); + background: color-mix(in srgb, var(--hb-brand) 9%, var(--hb-surface-muted)); +} + .hb-scout-queue-item__meta { font-size: 0.75rem; color: var(--hb-muted); diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 738a8f3..d2a8680 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -1,6 +1,25 @@ -import { defineConfig } from "vite"; +import { defineConfig, type ProxyOptions } from "vite"; import react from "@vitejs/plugin-react"; +const proxy = { + // Live API -> gateway + "/api": { + target: "http://127.0.0.1:8888", + changeOrigin: true, + configure: (gateway) => { + // Browser -> Vite is same-origin. Do not forward that Origin to the + // private gateway, where it would be mistaken for a direct CORS call. + gateway.on("proxyReq", (request) => request.removeHeader("origin")); + }, + }, + // 頭像等公開物件（MinIO path-style）。nginx 未轉 /haixun-assets 時， + // 流量會落到 Vite；由此代轉，避免回 SPA HTML。 + "/haixun-assets": { + target: "http://127.0.0.1:9000", + changeOrigin: true, + }, +} satisfies Record; + // https://vite.dev/config/ export default defineConfig({ plugins: [react()], @@ -9,23 +28,12 @@ export default defineConfig({ port: 5173, strictPort: false, allowedHosts: ["threads-tool-dev.30cm.net", ".30cm.net"], - proxy: { - // Live API → gateway - "/api": { - target: "http://127.0.0.1:8888", - changeOrigin: true, - }, - // 頭像等公開物件（MinIO path-style）。nginx 未轉 /haixun-assets 時， - // 流量會落到 Vite；由此代轉，避免回 SPA HTML。 - "/haixun-assets": { - target: "http://127.0.0.1:9000", - changeOrigin: true, - }, - }, + proxy, }, preview: { host: "0.0.0.0", port: 4173, allowedHosts: ["threads-tool-dev.30cm.net", ".30cm.net"], + proxy, }, }); diff --git a/deploy/nginx/haixun-dev.conf b/deploy/nginx/haixun-dev.conf index 30dd356..dce4dbc 100644 --- a/deploy/nginx/haixun-dev.conf +++ b/deploy/nginx/haixun-dev.conf @@ -59,6 +59,8 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + # Browser -> nginx is same-origin; gateway should not apply direct-API CORS. + proxy_set_header Origin ""; proxy_set_header Authorization $http_authorization; proxy_set_header X-Member-Authorization $http_x_member_authorization; proxy_set_header Connection "";