finance-dashboard/server.js

// ═══════════════════════════════════════════════════════════
//  MacroScope 伺服器
//  - 對外提供 index.html（前端）
//  - GET /api/macro          整理好的總經資料（後端持金鑰呼叫 FRED）
//  - GET /api/series/:key     單一指標的歷史序列（給「走勢大圖」）
//  - GET /api/score-history   每日健康分數累積歷史
//  資料持久化於 SQLite（data.db）：重啟即時載入、每天累積分數快照
// ═══════════════════════════════════════════════════════════

import 'dotenv/config';
import express from 'express';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import fs from 'node:fs';

import { GROUPS, INDICATOR_MAP } from './lib/indicators.js';
import { getIndicatorCards, getYieldCurve, MissingKeyError } from './lib/fred.js';
import { computeScore } from './lib/score.js';
import { EVENTS, EPISODES } from './lib/events.js';
import {
  savePayload, loadPayload, saveSeries, getSeries,
  saveScoreSnapshot, getScoreHistory,
  listTrades, getTrade, insertTrade, updateTrade, deleteTrade, tradeStats,
  getCachedJSON, putCachedJSON, getCachedEntry,
} from './lib/db.js';
import { getKnowledge, getNote, knowledgeReady } from './lib/knowledge.js';
import { getFundamentals, getLatestFilingInfo, getQuote, getCompanyProfile } from './lib/fundamentals.js';
import { buildReport } from './lib/fincheck.js';
import { getHistory, getHistorySince, RANGES, INTERVALS } from './lib/marketdata.js';
import { runBacktest, STRATEGIES } from './lib/backtest.js';
import { getInvestMap } from './lib/investmap.js';
import { buildGraph } from './lib/graph.js';
import {
  getCalendarPayload, getCalendarWatchlist, saveCalendarWatchlist, warmCalendarCache,
} from './lib/calendar-cache.js';
import { getCompanyIntel } from './lib/companyintel.js';

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const ENV_PATH = path.join(__dirname, '.env');
const app = express();
app.use(express.json({ limit: '1mb' }));
const PORT = process.env.PORT || 3000;
const CACHE_TTL_MS = (Number(process.env.CACHE_TTL_SECONDS) || 3600) * 1000;
// 財報變動頻率低（季報），因此長期存資料庫、盡量沿用：
//  FUND_SOFT_MS 內直接用快取、完全不連網；超過才用輕量探針查 SEC 是否有新財報，
//  沒有新財報就續用快取（只更新檢查時間），有新財報或探針失敗且超過 FUND_HARD_MS 才重抓。
const FUND_SOFT_MS = (Number(process.env.FUND_SOFT_HOURS) || 12) * 3600 * 1000;
const FUND_HARD_MS = (Number(process.env.FUND_HARD_DAYS) || 3) * 24 * 3600 * 1000;
// 歷史股價快取：日線 6 小時內沿用、週/月線 1 天內沿用（節省 API）
const HIST_TTL_MS = (Number(process.env.HIST_SOFT_HOURS) || 6) * 3600 * 1000;
// 近即時報價：免費來源可能延遲，短快取避免切換畫面時密集連打。
const QUOTE_TTL_MS = (Number(process.env.QUOTE_TTL_SECONDS) || 60) * 1000;
const PROFILE_TTL_MS = (Number(process.env.PROFILE_TTL_HOURS) || 24) * 3600 * 1000;
const INTEL_TTL_MS = (Number(process.env.INTEL_TTL_HOURS) || 6) * 3600 * 1000;
const SYMBOL_RE = /^[A-Z0-9.\-]{1,12}$/;
const hasKey = process.env.FRED_API_KEY && process.env.FRED_API_KEY !== 'your_fred_api_key_here';

const SETTINGS_FIELDS = [
  { key: 'FRED_API_KEY', label: 'FRED API Key', type: 'secret', group: 'market', hint: '總經資料與部分日曆資料使用。' },
  { key: 'OPENCODE_GO_API_KEY', label: 'OpenCode Go API Key', type: 'secret', group: 'ai', hint: 'OpenCode Go provider。' },
  { key: 'OPENCODE_GO_MODEL', label: 'OpenCode Go Model', type: 'text', group: 'ai', hint: '從 OpenCode Go 的 /models 端點抓取後選擇。' },
  { key: 'GROK_API_KEY', label: 'Grok API Key', type: 'secret', group: 'ai', hint: 'xAI / Grok provider。' },
  { key: 'GROK_MODEL', label: 'Grok Model', type: 'text', group: 'ai', hint: '從 xAI /models 端點抓取後選擇。' },
  { key: 'AI_ACTIVE_PROVIDER', label: '預設 AI Provider', type: 'text', group: 'ai', hint: 'opencode-go 或 grok。' },
];

function parseEnvText(text) {
  const out = {};
  for (const line of String(text || '').split(/\r?\n/)) {
    const m = line.match(/^\s*([\w.-]+)\s*=\s*(.*)\s*$/);
    if (!m) continue;
    let v = m[2] || '';
    if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
    out[m[1]] = v;
  }
  return out;
}
function readEnvFile() {
  try { return parseEnvText(fs.readFileSync(ENV_PATH, 'utf8')); }
  catch (_) { return {}; }
}
function quoteEnvValue(v) {
  v = String(v == null ? '' : v);
  if (!v || /[\s#"'\\]/.test(v)) return JSON.stringify(v);
  return v;
}
function writeEnvUpdates(updates) {
  const src = fs.existsSync(ENV_PATH) ? fs.readFileSync(ENV_PATH, 'utf8') : '';
  const lines = src.split(/\r?\n/);
  const used = new Set();
  const next = lines.map(line => {
    const m = line.match(/^(\s*)([\w.-]+)(\s*=\s*)(.*)$/);
    if (!m || !(m[2] in updates)) return line;
    used.add(m[2]);
    return `${m[1]}${m[2]}${m[3]}${quoteEnvValue(updates[m[2]])}`;
  });
  for (const [k, v] of Object.entries(updates)) {
    if (!used.has(k)) next.push(`${k}=${quoteEnvValue(v)}`);
  }
  fs.writeFileSync(ENV_PATH, next.join('\n').replace(/\n{3,}/g, '\n\n'));
  Object.assign(process.env, updates);
}
function maskSecret(v) {
  if (!v) return '';
  v = String(v);
  return v.length <= 8 ? '已設定' : `${v.slice(0, 4)}…${v.slice(-4)}`;
}

// 記憶體快取（開機時會用 DB 內容預先填入）
let cache = { at: 0, payload: null };

async function buildPayload() {
  const [{ cards, seriesHistory, degraded }, yieldCurve] = await Promise.all([
    getIndicatorCards(),
    getYieldCurve(),
  ]);
  const { score, regime, breakdown, signals } = computeScore(cards);
  const groups = GROUPS.map((g) => ({
    key: g.key, title: g.title, titleEn: g.titleEn, icon: g.icon,
    colorKey: g.colorKey, intro: g.intro,
    cards: Object.values(cards).filter((c) => c.group === g.key),
  }));
  const payload = {
    updatedAt: new Date().toISOString(),
    score, regime, breakdown, signals, groups, yieldCurve, degraded,
  };
  return { payload, seriesHistory };
}

// 抓取 → 更新記憶體快取 → 寫入資料庫（序列 + 分數快照）
async function refreshAndCache() {
  const { payload, seriesHistory } = await buildPayload();
  cache = { at: Date.now(), payload };
  try {
    savePayload(payload);
    for (const [key, points] of Object.entries(seriesHistory)) saveSeries(key, points);
    saveScoreSnapshot(payload.score, payload.regime?.label);
  } catch (e) {
    console.warn('寫入資料庫失敗（不影響顯示）：', e.message);
  }
  return payload;
}

app.get('/api/macro', async (req, res) => {
  try {
    const fresh = req.query.fresh === '1';
    if (!fresh && cache.payload && Date.now() - cache.at < CACHE_TTL_MS) {
      return res.json({ ...cache.payload, cached: true });
    }
    const payload = await refreshAndCache();
    res.json({ ...payload, cached: false });
  } catch (err) {
    if (err instanceof MissingKeyError) {
      return res.status(503).json({
        error: 'missing_api_key',
        message: '尚未設定 FRED 金鑰。請到 AI 設定頁填入免費的 FRED_API_KEY，儲存後重新載入總經頁。',
        hint: 'https://fred.stlouisfed.org/docs/api/api_key.html',
      });
    }
    // 若有舊快取，至少先給舊資料
    if (cache.payload) return res.json({ ...cache.payload, cached: true, stale: true });
    console.error('[api/macro] 失敗：', err);
    res.status(502).json({ error: 'fetch_failed', message: '取得資料失敗，請稍後再試。', detail: String(err?.message || err) });
  }
});

// 歷史事件標記 & 危機案例（靜態設定，給走勢標註與「歷史殷鑑」頁用）
app.get('/api/events', (req, res) => res.json({ events: EVENTS, episodes: EPISODES }));

// 單一指標歷史序列（給走勢大圖）
const RANGE_DAYS = { '1m': 30, '6m': 182, '1y': 365, '5y': 1825, '10y': 3650, max: null };
app.get('/api/series/:key', (req, res) => {
  const key = req.params.key;
  const ind = INDICATOR_MAP[key];
  if (!ind) return res.status(404).json({ error: 'unknown_series', message: `查無指標：${key}` });
  const range = RANGE_DAYS[req.query.range] !== undefined ? req.query.range : '1y';
  const days = RANGE_DAYS[range];
  const since = days ? new Date(Date.now() - days * 86400000).toISOString().slice(0, 10) : null;
  const points = getSeries(key, since);
  res.json({
    key, label: ind.label, labelEn: ind.labelEn,
    format: ind.format, decimals: ind.decimals ?? 2,
    inverted: !!ind.inverted, tip: ind.tip, substitute: ind.substitute || null,
    range, points,
  });
});

// 每日健康分數歷史
app.get('/api/score-history', (req, res) => {
  res.json({ points: getScoreHistory() });
});

// ─── 學習教材：知識庫 ───
app.get('/api/knowledge', (req, res) => {
  const k = getKnowledge();
  if (!k) return res.status(503).json({ error: 'knowledge_not_built', message: '知識庫尚未建立，請先執行 npm run build:knowledge。' });
  res.json(k);
});
app.get('/api/note/:kind/:id', (req, res) => {
  const note = getNote(req.params.kind, req.params.id);
  if (!note) return res.status(404).json({ error: 'note_not_found', message: '找不到這篇筆記。' });
  res.json(note);
});

// ─── 財報健檢 ───
app.get('/api/fundamentals/:symbol', async (req, res) => {
  const symbol = String(req.params.symbol || '').trim().toUpperCase();
  if (!/^[A-Z0-9.\-]{1,12}$/.test(symbol)) return res.status(400).json({ error: 'bad_symbol', message: '代號格式不正確。' });
  const cacheKey = 'fund:' + symbol;
  const fresh = req.query.fresh === '1';
  const entry = getCachedEntry(cacheKey); // { value, updatedAt } | null
  try {
    if (!fresh && entry) {
      const hasMetricPayload = entry.value?._metricsVersion >= 2 && (Array.isArray(entry.value?.quarters) || Array.isArray(entry.value?.annual));
      const age = Date.now() - entry.updatedAt;
      // 1) 還很新 → 直接用快取，完全不連網
      if (hasMetricPayload && age < FUND_SOFT_MS) return res.json({ ...entry.value, cached: true });
      // 2) 稍舊 → 用輕量探針確認 SEC 是否有新財報
      const probe = await getLatestFilingInfo(symbol).catch(() => null);
      const known = entry.value._latestFiling;
      const noUpdate = hasMetricPayload && (probe ? (known && probe.accn === known) : (age <= FUND_HARD_MS));
      if (noUpdate) {
        // 沒有新財報（或暫時無法判斷但還沒到硬上限）→ 續用快取，只更新「檢查時間」
        const v = { ...entry.value, _checkedAt: Date.now() };
        putCachedJSON(cacheKey, v); // 更新 updated_at，避免短時間內重複探針
        return res.json({ ...v, cached: true });
      }
    }
    // 3) 首次、或偵測到新財報、或使用者手動更新 → 真正抓取
    const fundamentals = await getFundamentals(symbol);
    const report = buildReport(fundamentals);
    const probe = await getLatestFilingInfo(symbol).catch(() => null);
    const now = Date.now();
    const payload = {
      symbol: fundamentals.symbol, name: fundamentals.name, source: fundamentals.source,
      currency: fundamentals.currency, asOf: fundamentals.asOf, price: fundamentals.price, report,
      peTrailing: fundamentals.peTrailing, marketCap: fundamentals.marketCap,
      sharesOutstanding: fundamentals.sharesOutstanding,
      targetPrice: fundamentals.targetPrice, dividendYield: fundamentals.dividendYield,
      quarters: fundamentals.quarters, annual: fundamentals.annual, balance: fundamentals.balance,
      _metricsVersion: 2,
      _fetchedAt: now, _checkedAt: now,
      _latestFiling: probe ? probe.accn : null, _latestForm: probe ? probe.form : null,
    };
    putCachedJSON(cacheKey, payload);
    res.json({ ...payload, cached: false });
  } catch (err) {
    console.error('[api/fundamentals]', symbol, err?.message || err);
    // 抓取失敗但有舊資料 → 回舊資料（標記 stale），不讓使用者卡住、也避免一直重試燒 API
    if (entry) return res.json({ ...entry.value, cached: true, stale: true, fetchError: String(err?.message || err) });
    res.status(502).json({ error: 'fundamentals_failed', message: String(err?.message || err) });
  }
});

// ─── 歷史股價（價格走勢 + 回測共用，持久 DB 快取）───
const PRICE_RANGE_DAYS = { '3mo': 92, '6mo': 184, '1y': 370, '2y': 740, '5y': 1855, '10y': 3710, max: null };
function trimHistoryRange(payload, range) {
  if (!payload?.points || !PRICE_RANGE_DAYS[range]) return payload;
  const since = new Date(Date.now() - PRICE_RANGE_DAYS[range] * 86400000).toISOString().slice(0, 10);
  return { ...payload, points: payload.points.filter(p => p.date >= since) };
}
function mergeHistory(oldPayload, patchPayload) {
  const map = new Map();
  for (const p of (oldPayload.points || [])) map.set(p.date, p);
  for (const p of (patchPayload.points || [])) map.set(p.date, p);
  const points = [...map.values()].sort((a, b) => a.date < b.date ? -1 : 1);
  return {
    ...oldPayload,
    ...patchPayload,
    points,
    _lastIncrementalAt: Date.now(),
    _incremental: true,
  };
}
async function getHistoryCached(symbol, range, interval, fresh) {
  const key = `hist:${symbol}:${range}:${interval}`;
  const ttl = interval === '1d' ? HIST_TTL_MS : 24 * 3600 * 1000;
  const entry = getCachedEntry(key);
  if (!fresh && entry && Date.now() - entry.updatedAt < ttl) return { ...trimHistoryRange(entry.value, range), cached: true };
  try {
    let hist;
    const oldPoints = entry?.value?.points || [];
    const lastDate = oldPoints.length ? oldPoints[oldPoints.length - 1].date : null;
    if (lastDate) hist = mergeHistory(entry.value, await getHistorySince(symbol, lastDate, range, interval));
    else hist = await getHistory(symbol, range, interval);
    const payload = { ...hist, _fetchedAt: Date.now() };
    putCachedJSON(key, payload);
    return { ...trimHistoryRange(payload, range), cached: false };
  } catch (err) {
    if (entry) return { ...trimHistoryRange(entry.value, range), cached: true, stale: true, fetchError: String(err?.message || err) };
    throw err;
  }
}

app.get('/api/price/:symbol', async (req, res) => {
  const symbol = String(req.params.symbol || '').trim().toUpperCase();
  if (!SYMBOL_RE.test(symbol)) return res.status(400).json({ error: 'bad_symbol', message: '代號格式不正確。' });
  const range = RANGES.includes(req.query.range) ? req.query.range : '5y';
  const interval = INTERVALS.includes(req.query.interval) ? req.query.interval : '1d';
  try {
    const h = await getHistoryCached(symbol, range, interval, req.query.fresh === '1');
    res.json(h);
  } catch (err) {
    console.error('[api/price]', symbol, err?.message || err);
    res.status(502).json({ error: 'price_failed', message: String(err?.message || err) });
  }
});

app.get('/api/quote/:symbol', async (req, res) => {
  const symbol = String(req.params.symbol || '').trim().toUpperCase();
  if (!SYMBOL_RE.test(symbol)) return res.status(400).json({ error: 'bad_symbol', message: '代號格式不正確。' });
  const key = `quote:${symbol}`;
  const entry = getCachedEntry(key);
  const fresh = req.query.fresh === '1';
  try {
    if (!fresh && entry && Date.now() - entry.updatedAt < QUOTE_TTL_MS) {
      return res.json({ ...entry.value, cached: true });
    }
    const quote = await getQuote(symbol);
    const payload = { symbol, ...quote, _fetchedAt: Date.now() };
    putCachedJSON(key, payload);
    res.json({ ...payload, cached: false });
  } catch (err) {
    console.error('[api/quote]', symbol, err?.message || err);
    if (entry) return res.json({ ...entry.value, cached: true, stale: true, fetchError: String(err?.message || err) });
    res.status(502).json({ error: 'quote_failed', message: String(err?.message || err) });
  }
});

app.get('/api/profile/:symbol', async (req, res) => {
  const symbol = String(req.params.symbol || '').trim().toUpperCase();
  if (!SYMBOL_RE.test(symbol)) return res.status(400).json({ error: 'bad_symbol', message: '代號格式不正確。' });
  const key = `profile:${symbol}`;
  const entry = getCachedEntry(key);
  const fresh = req.query.fresh === '1';
  try {
    if (!fresh && entry && Date.now() - entry.updatedAt < PROFILE_TTL_MS) return res.json({ ...entry.value, cached: true });
    const profile = await getCompanyProfile(symbol);
    const payload = { ...profile, _fetchedAt: Date.now() };
    putCachedJSON(key, payload);
    res.json({ ...payload, cached: false });
  } catch (err) {
    console.error('[api/profile]', symbol, err?.message || err);
    if (entry) return res.json({ ...entry.value, cached: true, stale: true, fetchError: String(err?.message || err) });
    res.status(502).json({ error: 'profile_failed', message: String(err?.message || err) });
  }
});

app.get('/api/company-intel/:symbol', async (req, res) => {
  const symbol = String(req.params.symbol || '').trim().toUpperCase();
  if (!SYMBOL_RE.test(symbol)) return res.status(400).json({ error: 'bad_symbol', message: '代號格式不正確。' });
  const key = `intel:${symbol}`;
  const entry = getCachedEntry(key);
  const fresh = req.query.fresh === '1';
  try {
    if (!fresh && entry && Date.now() - entry.updatedAt < INTEL_TTL_MS) return res.json({ ...entry.value, cached: true });
    const profile = getCachedEntry(`profile:${symbol}`)?.value || {};
    const payload = await getCompanyIntel(symbol, profile);
    putCachedJSON(key, payload);
    res.json({ ...payload, cached: false });
  } catch (err) {
    console.error('[api/company-intel]', symbol, err?.message || err);
    if (entry) return res.json({ ...entry.value, cached: true, stale: true, fetchError: String(err?.message || err) });
    res.status(502).json({ error: 'intel_failed', message: String(err?.message || err) });
  }
});

function addDaysISO(base, days) {
  const d = new Date(base + 'T00:00:00Z');
  d.setUTCDate(d.getUTCDate() + days);
  return d.toISOString().slice(0, 10);
}
function calendarSymbols(req) {
  const fromQuery = String(req.query.symbols || '').split(',').map(s => s.trim().toUpperCase()).filter(Boolean);
  return [...new Set(fromQuery)].filter(s => SYMBOL_RE.test(s)).slice(0, 30);
}
app.get('/api/calendar/watchlist', (req, res) => {
  res.json({ symbols: getCalendarWatchlist() });
});
app.put('/api/calendar/watchlist', (req, res) => {
  const raw = Array.isArray(req.body?.symbols) ? req.body.symbols : String(req.body?.symbols || '').split(',');
  const symbols = saveCalendarWatchlist(raw.filter(s => SYMBOL_RE.test(String(s).trim().toUpperCase())));
  res.json({ ok: true, symbols });
});
app.get('/api/calendar', async (req, res) => {
  const today = new Date().toISOString().slice(0, 10);
  const start = /^\d{4}-\d{2}-\d{2}$/.test(req.query.start) ? req.query.start : today;
  const end = /^\d{4}-\d{2}-\d{2}$/.test(req.query.end) ? req.query.end : addDaysISO(today, 60);
  const fromQuery = calendarSymbols(req);
  const stored = getCalendarWatchlist();
  const symbols = fromQuery.length ? fromQuery : stored;
  const forceFresh = req.query.fresh === '1';
  try {
    const payload = await getCalendarPayload({ start, end, symbols, forceFresh });
    res.json(payload);
  } catch (err) {
    console.error('[api/calendar]', err?.message || err);
    res.status(502).json({ error: 'calendar_failed', message: String(err?.message || err) });
  }
});

app.get('/api/backtest/:symbol', async (req, res) => {
  const symbol = String(req.params.symbol || '').trim().toUpperCase();
  if (!SYMBOL_RE.test(symbol)) return res.status(400).json({ error: 'bad_symbol', message: '代號格式不正確。' });
  const strategy = STRATEGIES[req.query.strategy] ? req.query.strategy : 'buyhold';
  const range = RANGES.includes(req.query.range) ? req.query.range : '5y';
  const numQ = (k) => (req.query[k] != null && req.query[k] !== '') ? Number(req.query[k]) : undefined;
  try {
    const h = await getHistoryCached(symbol, range, '1d', false);
    const result = runBacktest(h.points, {
      strategy, monthly: numQ('monthly'), short: numQ('short'), long: numQ('long'), drop: numQ('drop'),
    });
    res.json({ symbol, name: h.name, currency: h.currency, range, cached: h.cached, ...result });
  } catch (err) {
    console.error('[api/backtest]', symbol, err?.message || err);
    res.status(502).json({ error: 'backtest_failed', message: String(err?.message || err) });
  }
});

app.get('/api/graph', (req, res) => {
  try {
    const k = getKnowledge();
    if (!k) return res.status(503).json({ error: 'knowledge_not_built', message: '知識庫尚未建立。' });
    res.json(buildGraph(k, req.query));
  } catch (err) {
    res.status(500).json({ error: 'graph_failed', message: String(err?.message || err) });
  }
});

app.get('/api/investmap', (req, res) => {
  try {
    const k = getKnowledge();
    const byNum = {};
    for (const p of (k?.principles || [])) byNum[p.num] = { title: p.title, id: p.id };
    res.json(getInvestMap(byNum));
  } catch (err) {
    res.status(500).json({ error: 'investmap_failed', message: String(err?.message || err) });
  }
});

// ─── 交易復盤 ───
app.get('/api/trades', (req, res) => res.json({ trades: listTrades() }));
app.get('/api/trades/stats', (req, res) => res.json(tradeStats()));
app.post('/api/trades', (req, res) => {
  try { res.json(insertTrade(req.body || {})); }
  catch (e) { res.status(400).json({ error: 'bad_trade', message: String(e?.message || e) }); }
});
app.put('/api/trades/:id', (req, res) => {
  const row = updateTrade(Number(req.params.id), req.body || {});
  if (!row) return res.status(404).json({ error: 'not_found', message: '查無此交易。' });
  res.json(row);
});
app.delete('/api/trades/:id', (req, res) => {
  deleteTrade(Number(req.params.id));
  res.json({ ok: true });
});

// ─── AI Provider 代理：OpenCode Go / Grok ───
const AI_PROVIDERS = {
  'opencode-go': {
    label: 'OpenCode Go',
    endpoint: 'https://opencode.ai/zen/go/v1/chat/completions',
    modelsEndpoint: 'https://opencode.ai/zen/go/v1/models',
    keyEnv: 'OPENCODE_GO_API_KEY',
    modelEnv: 'OPENCODE_GO_MODEL',
    mode: 'chat',
  },
  grok: {
    label: 'Grok',
    endpoint: 'https://api.x.ai/v1/responses',
    modelsEndpoint: 'https://api.x.ai/v1/models',
    keyEnv: 'GROK_API_KEY',
    modelEnv: 'GROK_MODEL',
    mode: 'responses',
  },
};
function normalizeAIText(data, mode) {
  if (mode === 'responses') {
    if (data?.output_text) return data.output_text;
    const chunks = [];
    for (const item of data?.output || []) {
      for (const c of item?.content || []) {
        if (typeof c?.text === 'string') chunks.push(c.text);
        else if (typeof c?.content === 'string') chunks.push(c.content);
      }
    }
    return chunks.join('\n').trim();
  }
  return data?.choices?.[0]?.message?.content || data?.choices?.[0]?.text || '';
}
function compactForPrompt(v, max = 16000) {
  const s = typeof v === 'string' ? v : JSON.stringify(v, null, 2);
  return s.length > max ? s.slice(0, max) + '\n...（上下文已截斷）' : s;
}
function cachedValue(entry) {
  if (!entry) return null;
  return { ...entry.value, cached: true, cachedAt: new Date(entry.updatedAt).toISOString() };
}
function summarizeMacro(payload, focus) {
  if (!payload) return null;
  const cards = (payload.groups || []).flatMap(g => (g.cards || []).map(c => ({ ...c, groupTitle: g.title })));
  const focusedCard = focus?.key ? cards.find(c => c.key === focus.key) : null;
  const series = focusedCard ? getSeries(focusedCard.key, null).slice(-160) : [];
  return {
    updatedAt: payload.updatedAt,
    cached: true,
    score: payload.score,
    regime: payload.regime,
    signals: payload.signals,
    focusedCard: focusedCard ? {
      key: focusedCard.key,
      group: focusedCard.groupTitle,
      label: focusedCard.label,
      labelEn: focusedCard.labelEn,
      value: focusedCard.value,
      change: focusedCard.change,
      status: focusedCard.status,
      human: focusedCard.human,
      context: focusedCard.context,
      tip: focusedCard.tip,
      series,
    } : null,
  };
}
async function stockAIContext(symbol, focus, allowFetch) {
  if (!SYMBOL_RE.test(symbol || '')) return { symbol, error: 'bad_symbol' };
  const out = { symbol, subPage: focus?.subPage || null, sources: [] };
  let fundEntry = getCachedEntry(`fund:${symbol}`);
  if (!fundEntry && allowFetch) {
    const fundamentals = await getFundamentals(symbol);
    const report = buildReport(fundamentals);
    const payload = {
      _metricsVersion: 2,
      _fetchedAt: Date.now(),
      symbol: fundamentals.symbol, name: fundamentals.name, source: fundamentals.source,
      currency: fundamentals.currency, asOf: fundamentals.asOf, price: fundamentals.price, report,
      peTrailing: fundamentals.peTrailing, marketCap: fundamentals.marketCap,
      sharesOutstanding: fundamentals.sharesOutstanding,
      targetPrice: fundamentals.targetPrice, dividendYield: fundamentals.dividendYield,
      quarters: fundamentals.quarters, annual: fundamentals.annual, balance: fundamentals.balance,
      _latestFiling: await getLatestFilingInfo(symbol).catch(() => null),
    };
    putCachedJSON(`fund:${symbol}`, payload);
    fundEntry = getCachedEntry(`fund:${symbol}`);
    out.sources.push('fundamentals:fetched');
  }
  let quoteEntry = getCachedEntry(`quote:${symbol}`);
  if (!quoteEntry && allowFetch) {
    const quote = await getQuote(symbol);
    putCachedJSON(`quote:${symbol}`, { symbol, ...quote, _fetchedAt: Date.now() });
    quoteEntry = getCachedEntry(`quote:${symbol}`);
    out.sources.push('quote:fetched');
  }
  let histEntry = getCachedEntry(`hist:${symbol}:max:1d`);
  if (!histEntry && allowFetch) {
    await getHistoryCached(symbol, 'max', '1d', false).catch(() => null);
    histEntry = getCachedEntry(`hist:${symbol}:max:1d`);
    if (histEntry) out.sources.push('history:fetched');
  }
  const fundamentals = cachedValue(fundEntry);
  const quote = cachedValue(quoteEntry);
  const history = cachedValue(histEntry);
  out.fundamentals = fundamentals ? {
    symbol: fundamentals.symbol,
    name: fundamentals.name,
    cachedAt: fundamentals.cachedAt,
    asOf: fundamentals.asOf,
    price: fundamentals.price,
    report: fundamentals.report,
    peTrailing: fundamentals.peTrailing,
    marketCap: fundamentals.marketCap,
    dividendYield: fundamentals.dividendYield,
    quarters: (fundamentals.quarters || []).slice(0, 8),
    annual: (fundamentals.annual || []).slice(0, 5),
    balance: fundamentals.balance,
  } : null;
  out.quote = quote;
  out.history = history ? { ...history, points: (history.points || []).slice(-260) } : null;
  out.cacheStatus = {
    fundamentals: !!fundEntry,
    quote: !!quoteEntry,
    history: !!histEntry,
  };
  return out;
}
async function buildAIPageContext({ view, focus = {}, client = {}, allowFetch = true }) {
  const base = {
    mode: ['macro', 'calendar', 'learn', 'stock', 'journal'].includes(view) ? 'page' : 'chat',
    hasPageData: ['macro', 'calendar', 'learn', 'stock', 'journal'].includes(view),
    view,
    focus,
    client,
    dataPolicy: 'cache-first: data.db first; fetch only when DB cache is missing; never force fresh for AI context.',
    collectedAt: new Date().toISOString(),
  };
  if (view === 'macro') {
    let saved = loadPayload();
    if (!saved && allowFetch) {
      const payload = await refreshAndCache();
      saved = { payload, updatedAt: Date.now() };
    }
    base.macro = summarizeMacro(saved?.payload || cache.payload, focus);
  } else if (view === 'stock') {
    const symbol = String(focus.symbol || client.symbol || '').trim().toUpperCase();
    if (symbol) base.stock = await stockAIContext(symbol, focus, allowFetch);
  } else if (view === 'calendar') {
    const symbols = getCalendarWatchlist();
    const today = new Date().toISOString().slice(0, 10);
    const end = new Date(Date.now() + 60 * 86400000).toISOString().slice(0, 10);
    const baseEntry = getCachedEntry(`calendar:base:v5:${today}`);
    const earnEntry = symbols.length ? getCachedEntry(`calendar:earn:v5:${today}:${[...symbols].sort().join(',')}`) : null;
    if (baseEntry?.value) {
      const baseEvents = (baseEntry.value.events || []).filter(e => e.category !== 'earnings');
      const earnEvents = symbols.length ? (earnEntry?.value?.events || []).filter(e => e.category === 'earnings') : [];
      const events = [...baseEvents, ...earnEvents]
        .filter(e => e.date >= today && e.date <= end)
        .sort((a, b) => (a.date + (a.time || '')).localeCompare(b.date + (b.time || '')))
        .slice(0, 80);
      base.calendar = {
        cached: true,
        cachedAt: new Date(baseEntry.updatedAt).toISOString(),
        watchlist: symbols,
        events,
        sources: baseEntry.value.sources || [],
      };
    } else {
      base.calendar = allowFetch
        ? await getCalendarPayload({ start: today, end, symbols, forceFresh: false })
          .then(d => ({ ...d, events: (d.events || []).slice(0, 80) }))
          .catch(e => ({ error: String(e?.message || e), watchlist: symbols }))
        : { cached: false, watchlist: symbols, events: [] };
    }
  } else if (view === 'journal') {
    base.journal = {
      stats: tradeStats(),
      trades: listTrades().slice(0, 80),
    };
  } else if (view === 'learn') {
    const note = focus.kind && focus.id ? getNote(focus.kind, focus.id) : null;
    base.learning = {
      focusedNote: note ? {
        kind: focus.kind,
        id: focus.id,
        title: note.title,
        summary: note.summary,
        body: String(note.body || '').slice(0, 10000),
      } : client.currentNote || null,
      visibleText: client.visibleText || '',
      personalNotes: client.personalNotes || [],
    };
  }
  return base;
}
function normalizeModelList(data) {
  const items = Array.isArray(data?.data) ? data.data : Array.isArray(data?.models) ? data.models : Array.isArray(data) ? data : [];
  return items
    .map(m => (typeof m === 'string' ? { id: m } : { id: m?.id || m?.name || m?.model, created: m?.created, ownedBy: m?.owned_by || m?.ownedBy }))
    .filter(m => m.id);
}
async function listProviderModels(provider, apiKey) {
  const ctrl = new AbortController();
  const timer = setTimeout(() => ctrl.abort(), 30000);
  try {
    const r = await fetch(provider.modelsEndpoint, {
      headers: { Authorization: `Bearer ${apiKey}` },
      signal: ctrl.signal,
    });
    const data = await r.json().catch(() => ({}));
    if (!r.ok) {
      const msg = data?.error?.message || data?.message || `${provider.label} 回傳 ${r.status}`;
      const err = new Error(msg);
      err.status = r.status;
      throw err;
    }
    return normalizeModelList(data);
  } finally {
    clearTimeout(timer);
  }
}
app.post('/api/ai/models', async (req, res) => {
  const providerId = String(req.body?.provider || '').trim();
  const provider = AI_PROVIDERS[providerId];
  if (!provider) return res.status(400).json({ error: 'bad_provider', message: '不支援的 AI provider。' });
  const apiKey = String(req.body?.apiKey || process.env[provider.keyEnv] || '').trim();
  if (!apiKey) return res.status(400).json({ error: 'missing_key', message: '請先在 AI 設定填入 API key。' });
  try {
    const models = await listProviderModels(provider, apiKey);
    res.json({ provider: providerId, models });
  } catch (err) {
    res.status(502).json({ error: 'models_failed', message: String(err?.message || err) });
  }
});
app.post('/api/ai/context', async (req, res) => {
  try {
    const view = String(req.body?.view || '').trim();
    const focus = req.body?.focus || {};
    const client = req.body?.client || {};
    const allowFetch = req.body?.allowFetch !== false;
    const context = await buildAIPageContext({ view, focus, client, allowFetch });
    res.json(context);
  } catch (err) {
    res.status(502).json({ error: 'context_failed', message: String(err?.message || err) });
  }
});
app.post('/api/ai/chat', async (req, res) => {
  const providerId = String(req.body?.provider || '').trim();
  const provider = AI_PROVIDERS[providerId];
  if (!provider) return res.status(400).json({ error: 'bad_provider', message: '不支援的 AI provider。' });
  const apiKey = String(req.body?.apiKey || process.env[provider.keyEnv] || '').trim();
  let model = String(req.body?.model || process.env[provider.modelEnv] || '').trim();
  const question = String(req.body?.question || '').trim();
  const context = req.body?.context || {};
  if (!apiKey) return res.status(400).json({ error: 'missing_key', message: '請先在 AI 設定填入 API key。' });
  if (!model) {
    const models = await listProviderModels(provider, apiKey).catch(() => []);
    model = models[0]?.id || '';
  }
  if (!model) return res.status(400).json({ error: 'missing_model', message: '請先設定模型。' });
  if (!question) return res.status(400).json({ error: 'missing_question', message: '請輸入問題。' });
  const hasPageData = context?.mode === 'page' || context?.hasPageData === true;
  const system = hasPageData ? [
    '你是 MacroScope 的投資學習助理。',
    '使用者正在帶著頁面上下文提問，請根據提供的財報、總經、學習資料或交易復盤做分析與對照。',
    '請用繁體中文回答，先給結論，再列出依據、矛盾點、下一步可以在頁面上檢查什麼。',
    '不要聲稱已即時查網路；若資料不足，要明確說資料不足。',
    '內容僅供學習，不構成投資建議。',
  ].join('\n') : [
    '你是 MacroScope 的 AI 助手。',
    '目前沒有可用頁面資料，請把這次對話當一般聊天或一般投資學習問答處理。',
    '請用繁體中文自然回答；如果使用者問投資或財務判斷，要提醒內容僅供學習，不構成投資建議。',
    '不要聲稱已即時查網路；需要即時資料時，請說明你需要使用者提供資料或切到相關頁面。',
  ].join('\n');
  const user = [
    `使用者問題：${question}`,
    '',
    hasPageData ? '目前頁面上下文：' : '對話狀態：',
    hasPageData ? compactForPrompt(context) : compactForPrompt({ mode: 'chat', view: context?.view || '', collectedAt: context?.collectedAt || '' }),
  ].join('\n');
  try {
    const ctrl = new AbortController();
    const timer = setTimeout(() => ctrl.abort(), 120000);
    const body = provider.mode === 'responses'
      ? { model, store: false, input: [{ role: 'system', content: system }, { role: 'user', content: user }] }
      : { model, messages: [{ role: 'system', content: system }, { role: 'user', content: user }], temperature: 0.2 };
    const r = await fetch(provider.endpoint, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${apiKey}` },
      body: JSON.stringify(body),
      signal: ctrl.signal,
    });
    clearTimeout(timer);
    const data = await r.json().catch(() => ({}));
    if (!r.ok) {
      return res.status(502).json({
        error: 'provider_failed',
        message: data?.error?.message || data?.message || `${provider.label} 回傳 ${r.status}`,
        detail: data,
      });
    }
    res.json({ provider: providerId, model, text: normalizeAIText(data, provider.mode), raw: { id: data.id, usage: data.usage } });
  } catch (err) {
    res.status(502).json({ error: 'ai_failed', message: String(err?.message || err) });
  }
});

app.get('/api/settings/env', (req, res) => {
  const env = { ...readEnvFile(), ...process.env };
  res.json({
    envPath: ENV_PATH,
    fields: SETTINGS_FIELDS.map(f => ({
      ...f,
      value: f.type === 'secret' ? '' : (env[f.key] || ''),
      hasValue: !!env[f.key],
      masked: f.type === 'secret' ? maskSecret(env[f.key]) : '',
    })),
  });
});
app.post('/api/settings/env', (req, res) => {
  const allowed = new Set(SETTINGS_FIELDS.map(f => f.key));
  const secret = new Set(SETTINGS_FIELDS.filter(f => f.type === 'secret').map(f => f.key));
  const body = req.body?.values || {};
  const updates = {};
  for (const [k, raw] of Object.entries(body)) {
    if (!allowed.has(k)) continue;
    const v = String(raw == null ? '' : raw).trim();
    if (secret.has(k) && !v) continue; // 留空代表保留既有 secret
    updates[k] = v;
  }
  try {
    if (Object.keys(updates).length) writeEnvUpdates(updates);
    const env = { ...readEnvFile(), ...process.env };
    res.json({
      ok: true,
      envPath: ENV_PATH,
      updated: Object.keys(updates),
      fields: SETTINGS_FIELDS.map(f => ({
        ...f,
        value: f.type === 'secret' ? '' : (env[f.key] || ''),
        hasValue: !!env[f.key],
        masked: f.type === 'secret' ? maskSecret(env[f.key]) : '',
      })),
    });
  } catch (err) {
    res.status(500).json({ error: 'env_write_failed', message: String(err?.message || err) });
  }
});

app.get('/api/health', (req, res) => res.json({ ok: true, knowledge: knowledgeReady() }));
app.use(express.static(__dirname));

app.listen(PORT, () => {
  console.log(`\nMacroScope 已啟動 → http://localhost:${PORT}\n`);
  if (!hasKey) {
    console.log('提醒：尚未設定 FRED_API_KEY，畫面會顯示設定教學。');
    console.log('申請免費金鑰：https://fred.stlouisfed.org/docs/api/api_key.html\n');
    return;
  }
  // 先用資料庫裡的舊資料填入快取（若有），讓頁面能即時開啟
  const saved = loadPayload();
  if (saved) {
    cache = { at: saved.updatedAt, payload: saved.payload };
    console.log('已從資料庫載入上次資料，頁面可即時開啟。');
  }
  // 背景刷新最新資料（首次或過期時較久，FRED 有流量限制）
  console.log('正在背景抓取最新資料（首次約需 20–40 秒）…');
  const t0 = Date.now();
  refreshAndCache()
    .then((payload) => {
      const ok = payload.groups.reduce((a, g) => a + g.cards.length, 0);
      console.log(`資料就緒：${ok} 個指標，健康分數 ${payload.score}（耗時 ${((Date.now() - t0) / 1000).toFixed(0)} 秒）\n`);
    })
    .catch((err) => console.log('背景抓取失敗（開啟頁面時會再試）：', String(err?.message || err), '\n'));
  warmCalendarCache()
    .then(() => console.log('日曆快取已就緒（資料庫，每日更新）。'))
    .catch(err => console.warn('[calendar warm]', err?.message || err));
});