199 lines
11 KiB
Go
199 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"fmt"
|
|
"time"
|
|
|
|
"apps/backend/internal/config"
|
|
|
|
"github.com/zeromicro/go-zero/core/conf"
|
|
"go.mongodb.org/mongo-driver/bson"
|
|
"go.mongodb.org/mongo-driver/mongo"
|
|
"go.mongodb.org/mongo-driver/mongo/options"
|
|
)
|
|
|
|
// init creates idempotent operational indexes only. It intentionally never
|
|
// inserts members or other business records.
|
|
func main() {
|
|
configFile := flag.String("f", "etc/gateway.yaml", "config file")
|
|
flag.Parse()
|
|
|
|
var c config.Config
|
|
conf.MustLoad(*configFile, &c)
|
|
c.ApplyEnv()
|
|
if c.Mongo.URI == "" || c.Mongo.Database == "" {
|
|
panic("Mongo.URI and Mongo.Database are required")
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
defer cancel()
|
|
client, err := mongo.Connect(ctx, options.Client().ApplyURI(c.Mongo.URI))
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
defer client.Disconnect(context.Background())
|
|
|
|
db := client.Database(c.Mongo.Database)
|
|
for collection, models := range indexModels() {
|
|
if _, err := db.Collection(collection).Indexes().CreateMany(ctx, models); err != nil {
|
|
panic(fmt.Errorf("create %s indexes: %w", collection, err))
|
|
}
|
|
}
|
|
fmt.Println("database indexes initialized")
|
|
}
|
|
|
|
// ownerIndex covers the near-universal "list one member's rows, newest first" shape. The sort key
|
|
// belongs in the index too, otherwise Mongo fetches every matching row before sorting.
|
|
func ownerIndex(sortField, name string) mongo.IndexModel {
|
|
return mongo.IndexModel{
|
|
Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: sortField, Value: -1}},
|
|
Options: options.Index().SetName(name),
|
|
}
|
|
}
|
|
|
|
// indexModels lists every index the running code depends on. Owner-scoped collections were all
|
|
// unindexed, so each list endpoint was a full collection scan that grew with total rows across
|
|
// every member rather than with the member's own data.
|
|
//
|
|
// These are deliberately all non-unique. Adding a uniqueness constraint to existing data can make
|
|
// CreateMany fail and take a deploy down, so that belongs in its own migration with a duplicate
|
|
// pre-check rather than here.
|
|
//
|
|
// Never list an index here that a migration creates as unique: Mongo rejects a second createIndex
|
|
// with the same name and a different spec (IndexKeySpecsConflict, code 86), so the duplicate would
|
|
// make this whole command panic once the migration has run.
|
|
func indexModels() map[string][]mongo.IndexModel {
|
|
return map[string][]mongo.IndexModel{
|
|
// members.uid is read on every authenticated request via the auth middleware, and it is a
|
|
// field lookup rather than _id, so without this it scanned the whole collection whenever
|
|
// the Redis cache missed.
|
|
"members": {
|
|
{Keys: bson.D{{Key: "uid", Value: 1}}, Options: options.Index().SetName("member_uid")},
|
|
{Keys: bson.D{{Key: "email", Value: 1}}, Options: options.Index().SetName("member_email")},
|
|
{Keys: bson.D{{Key: "phone", Value: 1}}, Options: options.Index().SetName("member_phone")},
|
|
},
|
|
"identities": {
|
|
{Keys: bson.D{{Key: "login_id", Value: 1}, {Key: "platform", Value: 1}}, Options: options.Index().SetName("identity_login_platform")},
|
|
},
|
|
"member_settings": {
|
|
{Keys: bson.D{{Key: "uid", Value: 1}}, Options: options.Index().SetName("settings_uid")},
|
|
},
|
|
"jobs": {
|
|
{Keys: bson.D{{Key: "status", Value: 1}, {Key: "run_after", Value: 1}, {Key: "created_at", Value: 1}}, Options: options.Index().SetName("claim_due_jobs")},
|
|
{Keys: bson.D{{Key: "status", Value: 1}, {Key: "completed_at", Value: 1}}, Options: options.Index().SetName("purge_terminal_jobs")},
|
|
// The jobs list is polled continuously by the UI and runs a count plus a find per poll.
|
|
ownerIndex("updated_at", "owner_jobs_updated"),
|
|
},
|
|
// One notification is written per job state change and nothing purges them, so this
|
|
// collection grows without bound and its scan cost grows with it.
|
|
"notifications": {
|
|
ownerIndex("created_at", "owner_notifications_created"),
|
|
},
|
|
"threads_accounts": {
|
|
ownerIndex("created_at", "owner_accounts_created"),
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "threads_user_id", Value: 1}}, Options: options.Index().SetName("owner_threads_user")},
|
|
},
|
|
"studio_outbox": {
|
|
{Keys: bson.D{{Key: "status", Value: 1}, {Key: "updated_at", Value: 1}}, Options: options.Index().SetName("worker_outbox_status_updated")},
|
|
{Keys: bson.D{{Key: "steps.status", Value: 1}, {Key: "steps.scheduled_at", Value: 1}, {Key: "steps.lease_expires_at", Value: 1}}, Options: options.Index().SetName("claim_due_outbox_steps")},
|
|
ownerIndex("updated_at", "owner_outbox_updated"),
|
|
},
|
|
"studio_personas": {ownerIndex("updated_at", "owner_personas_updated")},
|
|
"studio_plays": {ownerIndex("updated_at", "owner_plays_updated")},
|
|
"studio_own_posts": {
|
|
ownerIndex("published_at", "owner_own_posts_published"),
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "account_id", Value: 1}, {Key: "published_at", Value: -1}}, Options: options.Index().SetName("owner_account_own_posts")},
|
|
},
|
|
"studio_mentions": {
|
|
ownerIndex("created_at", "owner_mentions_created"),
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "account_id", Value: 1}, {Key: "created_at", Value: -1}}, Options: options.Index().SetName("owner_account_mentions")},
|
|
},
|
|
"scout_brands": {{Keys: bson.D{{Key: "owner_uid", Value: 1}}, Options: options.Index().SetName("owner_brands")}},
|
|
"scout_products": {{Keys: bson.D{{Key: "owner_uid", Value: 1}}, Options: options.Index().SetName("owner_products")}},
|
|
"scout_posts": {
|
|
ownerIndex("created_at", "owner_posts_created"),
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "brand_id", Value: 1}, {Key: "created_at", Value: -1}}, Options: options.Index().SetName("owner_brand_posts")},
|
|
},
|
|
"scout_homework": {{Keys: bson.D{{Key: "owner_uid", Value: 1}}, Options: options.Index().SetName("owner_homework")}},
|
|
"inspire_elements": {ownerIndex("updated_at", "owner_elements_updated")},
|
|
"inspire_sessions": {ownerIndex("updated_at", "owner_sessions_updated")},
|
|
"usage_events": {
|
|
{Keys: bson.D{{Key: "uid", Value: 1}, {Key: "month_key", Value: 1}, {Key: "created_at", Value: -1}}, Options: options.Index().SetName("uid_month_events")},
|
|
{Keys: bson.D{{Key: "month_key", Value: 1}, {Key: "created_at", Value: -1}}, Options: options.Index().SetName("tenant_month_events")},
|
|
{Keys: bson.D{{Key: "created_at", Value: 1}}, Options: options.Index().SetName("events_created_range")},
|
|
},
|
|
"usage_prefs": {
|
|
{Keys: bson.D{{Key: "uid", Value: 1}}, Options: options.Index().SetName("prefs_uid")},
|
|
// Stripe webhooks arrive knowing only the customer, so this is the reverse lookup
|
|
// back to a member and it runs on every billing event.
|
|
{Keys: bson.D{{Key: "stripe_customer_id", Value: 1}}, Options: options.Index().SetName("prefs_stripe_customer")},
|
|
},
|
|
"usage_purchases": {
|
|
{Keys: bson.D{{Key: "uid", Value: 1}, {Key: "created_at", Value: -1}}, Options: options.Index().SetName("uid_purchases_created")},
|
|
{Keys: bson.D{{Key: "stripe_event_id", Value: 1}}, Options: options.Index().SetName("purchase_stripe_event")},
|
|
},
|
|
"growth_outcomes": {
|
|
// The observe worker scans by status across all tenants on every tick.
|
|
{Keys: bson.D{{Key: "status", Value: 1}}, Options: options.Index().SetName("observe_status")},
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "source_type", Value: 1}, {Key: "source_id", Value: 1}}, Options: options.Index().SetName("owner_outcome_source")},
|
|
ownerIndex("created_at", "owner_outcomes_created"),
|
|
},
|
|
"growth_checkups": {ownerIndex("created_at", "owner_checkups_created")},
|
|
"growth_account_health": {{Keys: bson.D{{Key: "owner_uid", Value: 1}}, Options: options.Index().SetName("owner_health")}},
|
|
"growth_workspaces": {{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "archived", Value: 1}}, Options: options.Index().SetName("owner_workspaces_archived")}},
|
|
"growth_draft_reviews": {
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "ref_type", Value: 1}, {Key: "ref_id", Value: 1}, {Key: "updated_at", Value: -1}}, Options: options.Index().SetName("owner_review_ref")},
|
|
},
|
|
"growth_invite_rewards": {
|
|
{Keys: bson.D{{Key: "inviter_uid", Value: 1}, {Key: "created_at", Value: -1}}, Options: options.Index().SetName("inviter_rewards_created")},
|
|
{Keys: bson.D{{Key: "invitee_uid", Value: 1}}, Options: options.Index().SetName("invitee_reward")},
|
|
},
|
|
"growth_utm_links": {
|
|
// Looked up by code on the unauthenticated public redirect.
|
|
{Keys: bson.D{{Key: "code", Value: 1}}, Options: options.Index().SetName("utm_code")},
|
|
ownerIndex("created_at", "owner_utm_created"),
|
|
},
|
|
"growth_ws_members": {
|
|
{Keys: bson.D{{Key: "workspace_id", Value: 1}, {Key: "uid", Value: 1}}, Options: options.Index().SetName("workspace_member")},
|
|
},
|
|
// demand-radar. Names and specs match migration 000014 exactly so both paths are
|
|
// idempotent; the two unique keys (radar_opportunities.owner_opportunity_external and
|
|
// crm_contacts.owner_contact_identity) are intentionally absent and owned by that migration.
|
|
"radar_watches": {
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "status", Value: 1}}, Options: options.Index().SetName("owner_watches_status")},
|
|
ownerIndex("created_at", "owner_watches_created"),
|
|
},
|
|
"radar_sweeps": {
|
|
ownerIndex("created_at", "owner_sweeps_created"),
|
|
{Keys: bson.D{{Key: "watch_id", Value: 1}, {Key: "created_at", Value: -1}}, Options: options.Index().SetName("watch_sweeps_created")},
|
|
{Keys: bson.D{{Key: "job_id", Value: 1}}, Options: options.Index().SetName("sweep_job")},
|
|
},
|
|
"radar_opportunities": {
|
|
ownerIndex("created_at", "owner_opportunities_created"),
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "intent_band", Value: 1}, {Key: "status", Value: 1}}, Options: options.Index().SetName("owner_opportunity_band_status")},
|
|
},
|
|
"radar_replies": {
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "opportunity_id", Value: 1}, {Key: "variant", Value: 1}}, Options: options.Index().SetName("owner_reply_variant")},
|
|
},
|
|
"crm_contacts": {
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "stage", Value: 1}, {Key: "last_touch_at", Value: -1}}, Options: options.Index().SetName("owner_contacts_stage_touch")},
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "needs_follow_up", Value: 1}}, Options: options.Index().SetName("owner_contacts_follow_up")},
|
|
},
|
|
"crm_touches": {
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "contact_id", Value: 1}, {Key: "created_at", Value: -1}}, Options: options.Index().SetName("owner_touches_created")},
|
|
},
|
|
// The follow-up scan is a cross-tenant due sweep, so its driving index is not owner-scoped.
|
|
"crm_followups": {
|
|
{Keys: bson.D{{Key: "status", Value: 1}, {Key: "due_at", Value: 1}}, Options: options.Index().SetName("followup_due_scan")},
|
|
{Keys: bson.D{{Key: "owner_uid", Value: 1}, {Key: "contact_id", Value: 1}}, Options: options.Index().SetName("owner_contact_followup")},
|
|
},
|
|
"billing_checkout_attempts": {
|
|
{Keys: bson.D{{Key: "uid", Value: 1}, {Key: "request_id", Value: 1}}, Options: options.Index().SetName("uid_request")},
|
|
{Keys: bson.D{{Key: "session_id", Value: 1}}, Options: options.Index().SetName("checkout_session")},
|
|
{Keys: bson.D{{Key: "customer_id", Value: 1}}, Options: options.Index().SetName("checkout_customer")},
|
|
},
|
|
}
|
|
}
|