diff --git a/backend/internal/library/knowledge/synth.go b/backend/internal/library/knowledge/synth.go index 322c0a2..d7c3433 100644 --- a/backend/internal/library/knowledge/synth.go +++ b/backend/internal/library/knowledge/synth.go @@ -118,9 +118,9 @@ func ParseSynthOutput(raw string, in SynthInput, sources []BraveSource) (Graph, if err != nil { return Graph{}, err } - var out rawSynthOutput - if err := json.Unmarshal(payload, &out); err != nil { - return Graph{}, fmt.Errorf("parse knowledge graph json: %w", err) + out, err := unmarshalSynthOutput(payload) + if err != nil { + return Graph{}, err } seed := strings.TrimSpace(in.Seed) @@ -322,6 +322,130 @@ func resolveNodeRef(ref string, labelToID map[string]string, nodes []Node) strin return "" } +// unmarshalSynthOutput parses the knowledge graph payload, retrying once on a +// sanitized copy because LLMs occasionally emit `...` placeholders or +// trailing/duplicate commas that break strict JSON parsing. +func unmarshalSynthOutput(payload []byte) (rawSynthOutput, error) { + var out rawSynthOutput + if err := json.Unmarshal(payload, &out); err == nil { + return out, nil + } else { + if cleaned := sanitizeLLMJSON(payload); cleaned != nil { + var retry rawSynthOutput + if err2 := json.Unmarshal(cleaned, &retry); err2 == nil { + return retry, nil + } + } + return rawSynthOutput{}, fmt.Errorf("parse knowledge graph json: %w", err) + } +} + +// sanitizeLLMJSON removes ellipsis placeholders and normalizes stray commas +// outside of string literals. It returns nil when the input is unchanged so +// callers can skip a redundant re-parse. +func sanitizeLLMJSON(in []byte) []byte { + withoutEllipsis := stripEllipsis(string(in)) + normalized := normalizeCommas(withoutEllipsis) + if normalized == string(in) { + return nil + } + return []byte(normalized) +} + +func stripEllipsis(s string) string { + var b strings.Builder + b.Grow(len(s)) + inString := false + escaped := false + for i := 0; i < len(s); i++ { + ch := s[i] + if inString { + b.WriteByte(ch) + if escaped { + escaped = false + } else if ch == '\\' { + escaped = true + } else if ch == '"' { + inString = false + } + continue + } + if ch == '"' { + inString = true + b.WriteByte(ch) + continue + } + if ch == '.' { + j := i + for j < len(s) && s[j] == '.' { + j++ + } + if j-i >= 2 { + i = j - 1 + continue + } + } + b.WriteByte(ch) + } + return b.String() +} + +func normalizeCommas(s string) string { + var b strings.Builder + b.Grow(len(s)) + inString := false + escaped := false + var lastMeaningful byte + for i := 0; i < len(s); i++ { + ch := s[i] + if inString { + b.WriteByte(ch) + if escaped { + escaped = false + } else if ch == '\\' { + escaped = true + } else if ch == '"' { + inString = false + } + continue + } + if ch == '"' { + inString = true + b.WriteByte(ch) + lastMeaningful = ch + continue + } + if ch == ',' { + if lastMeaningful == '[' || lastMeaningful == '{' || lastMeaningful == ',' || lastMeaningful == 0 { + continue + } + if next := nextNonSpaceByte(s, i+1); next == ']' || next == '}' { + continue + } + b.WriteByte(ch) + lastMeaningful = ch + continue + } + b.WriteByte(ch) + if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' { + lastMeaningful = ch + } + } + return b.String() +} + +func nextNonSpaceByte(s string, from int) byte { + for i := from; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\n', '\r': + continue + default: + return s[i] + } + } + return 0 +} + func extractJSONObject(raw string) ([]byte, error) { text := strings.TrimSpace(raw) if text == "" { diff --git a/backend/internal/library/knowledge/synth_test.go b/backend/internal/library/knowledge/synth_test.go index a2ea17c..868510b 100644 --- a/backend/internal/library/knowledge/synth_test.go +++ b/backend/internal/library/knowledge/synth_test.go @@ -50,6 +50,57 @@ func TestExtractJSONObject(t *testing.T) { } } +func TestParseSynthOutputToleratesEllipsisAndTrailingCommas(t *testing.T) { + // LLM emits `...` placeholders, a trailing comma, and a duplicated comma. + raw := "```json\n" + `{ + "nodes": [ + { + "label": "敏感肌泛紅", + "nodeKind": "symptom", + "type": "symptom", + "layer": 1, + "relation": "溫差導致泛紅", + "evidenceUrls": ["https://example.com/a"], + }, + ..., + { + "label": "保濕不足", + "nodeKind": "cause", + "layer": 2, + "evidenceUrls": [], + } + ], + "edges": [...] +}` + "\n```" + payload, err := extractJSONObject(raw) + if err != nil { + t.Fatalf("extractJSONObject error: %v", err) + } + graph, err := ParseSynthOutput(string(payload), SynthInput{Seed: "敏感肌"}, nil) + if err != nil { + t.Fatalf("ParseSynthOutput error: %v", err) + } + labels := map[string]bool{} + for _, n := range graph.Nodes { + labels[n.Label] = true + } + if !labels["敏感肌泛紅"] || !labels["保濕不足"] { + t.Fatalf("expected both real nodes parsed, got %+v", labels) + } + // The URL containing "//" must survive sanitization. + found := false + for _, n := range graph.Nodes { + for _, ev := range n.Evidence { + if ev.URL == "https://example.com/a" { + found = true + } + } + } + if !found { + t.Fatal("expected evidence URL to survive sanitization") + } +} + func TestParseSynthOutputPlacementReason(t *testing.T) { raw := `{ "nodes": [