fix2

backend/internal/library/knowledge/synth.go

@@ -1,11 +1,11 @@
 package knowledge
 
 import (
-	"encoding/json"
 	"fmt"
 	"regexp"
 	"strings"
 
+	"haixun-backend/internal/library/llmjson"
 	libprompt "haixun-backend/internal/library/prompt"
 
 	"github.com/google/uuid"
@@ -322,128 +322,15 @@ func resolveNodeRef(ref string, labelToID map[string]string, nodes []Node) strin
 	return ""
 }
 
-// unmarshalSynthOutput parses the knowledge graph payload, retrying once on a
+// unmarshalSynthOutput parses the knowledge graph payload, tolerating the small
-// sanitized copy because LLMs occasionally emit `...` placeholders or
+// JSON artifacts (e.g. `...` placeholders, trailing/duplicate commas) that LLMs
-// trailing/duplicate commas that break strict JSON parsing.
+// occasionally emit.
 func unmarshalSynthOutput(payload []byte) (rawSynthOutput, error) {
 	var out rawSynthOutput
-	if err := json.Unmarshal(payload, &out); err == nil {
+	if err := llmjson.Unmarshal(payload, &out); err != nil {
-		return out, nil
-	} else {
-		if cleaned := sanitizeLLMJSON(payload); cleaned != nil {
-			var retry rawSynthOutput
-			if err2 := json.Unmarshal(cleaned, &retry); err2 == nil {
-				return retry, nil
-			}
-		}
 		return rawSynthOutput{}, fmt.Errorf("parse knowledge graph json: %w", err)
 	}
-}
+	return out, nil
-
-// sanitizeLLMJSON removes ellipsis placeholders and normalizes stray commas
-// outside of string literals. It returns nil when the input is unchanged so
-// callers can skip a redundant re-parse.
-func sanitizeLLMJSON(in []byte) []byte {
-	withoutEllipsis := stripEllipsis(string(in))
-	normalized := normalizeCommas(withoutEllipsis)
-	if normalized == string(in) {
-		return nil
-	}
-	return []byte(normalized)
-}
-
-func stripEllipsis(s string) string {
-	var b strings.Builder
-	b.Grow(len(s))
-	inString := false
-	escaped := false
-	for i := 0; i < len(s); i++ {
-		ch := s[i]
-		if inString {
-			b.WriteByte(ch)
-			if escaped {
-				escaped = false
-			} else if ch == '\\' {
-				escaped = true
-			} else if ch == '"' {
-				inString = false
-			}
-			continue
-		}
-		if ch == '"' {
-			inString = true
-			b.WriteByte(ch)
-			continue
-		}
-		if ch == '.' {
-			j := i
-			for j < len(s) && s[j] == '.' {
-				j++
-			}
-			if j-i >= 2 {
-				i = j - 1
-				continue
-			}
-		}
-		b.WriteByte(ch)
-	}
-	return b.String()
-}
-
-func normalizeCommas(s string) string {
-	var b strings.Builder
-	b.Grow(len(s))
-	inString := false
-	escaped := false
-	var lastMeaningful byte
-	for i := 0; i < len(s); i++ {
-		ch := s[i]
-		if inString {
-			b.WriteByte(ch)
-			if escaped {
-				escaped = false
-			} else if ch == '\\' {
-				escaped = true
-			} else if ch == '"' {
-				inString = false
-			}
-			continue
-		}
-		if ch == '"' {
-			inString = true
-			b.WriteByte(ch)
-			lastMeaningful = ch
-			continue
-		}
-		if ch == ',' {
-			if lastMeaningful == '[' || lastMeaningful == '{' || lastMeaningful == ',' || lastMeaningful == 0 {
-				continue
-			}
-			if next := nextNonSpaceByte(s, i+1); next == ']' || next == '}' {
-				continue
-			}
-			b.WriteByte(ch)
-			lastMeaningful = ch
-			continue
-		}
-		b.WriteByte(ch)
-		if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' {
-			lastMeaningful = ch
-		}
-	}
-	return b.String()
-}
-
-func nextNonSpaceByte(s string, from int) byte {
-	for i := from; i < len(s); i++ {
-		switch s[i] {
-		case ' ', '\t', '\n', '\r':
-			continue
-		default:
-			return s[i]
-		}
-	}
-	return 0
 }
 
 func extractJSONObject(raw string) ([]byte, error) {

backend/internal/library/llmjson/sanitize.go

@@ -0,0 +1,130 @@
+// Package llmjson tolerates the small JSON artifacts that LLMs sometimes emit
+// (e.g. `...` placeholders, trailing or duplicated commas) so structured
+// outputs can still be parsed instead of failing the whole job.
+package llmjson
+
+import (
+	"encoding/json"
+	"strings"
+)
+
+// Unmarshal parses payload into v, retrying once on a sanitized copy when the
+// raw bytes are not strictly valid JSON. The original error is returned when
+// the sanitized retry also fails, so callers keep a meaningful message.
+func Unmarshal(payload []byte, v any) error {
+	err := json.Unmarshal(payload, v)
+	if err == nil {
+		return nil
+	}
+	if cleaned := Sanitize(payload); cleaned != nil {
+		if retryErr := json.Unmarshal(cleaned, v); retryErr == nil {
+			return nil
+		}
+	}
+	return err
+}
+
+// Sanitize removes ellipsis placeholders and normalizes stray commas that sit
+// outside of string literals. It returns nil when nothing changed so callers
+// can avoid a redundant re-parse. String/URL contents are never altered.
+func Sanitize(in []byte) []byte {
+	normalized := normalizeCommas(stripEllipsis(string(in)))
+	if normalized == string(in) {
+		return nil
+	}
+	return []byte(normalized)
+}
+
+func stripEllipsis(s string) string {
+	var b strings.Builder
+	b.Grow(len(s))
+	inString := false
+	escaped := false
+	for i := 0; i < len(s); i++ {
+		ch := s[i]
+		if inString {
+			b.WriteByte(ch)
+			if escaped {
+				escaped = false
+			} else if ch == '\\' {
+				escaped = true
+			} else if ch == '"' {
+				inString = false
+			}
+			continue
+		}
+		if ch == '"' {
+			inString = true
+			b.WriteByte(ch)
+			continue
+		}
+		if ch == '.' {
+			j := i
+			for j < len(s) && s[j] == '.' {
+				j++
+			}
+			if j-i >= 2 {
+				i = j - 1
+				continue
+			}
+		}
+		b.WriteByte(ch)
+	}
+	return b.String()
+}
+
+func normalizeCommas(s string) string {
+	var b strings.Builder
+	b.Grow(len(s))
+	inString := false
+	escaped := false
+	var lastMeaningful byte
+	for i := 0; i < len(s); i++ {
+		ch := s[i]
+		if inString {
+			b.WriteByte(ch)
+			if escaped {
+				escaped = false
+			} else if ch == '\\' {
+				escaped = true
+			} else if ch == '"' {
+				inString = false
+			}
+			continue
+		}
+		if ch == '"' {
+			inString = true
+			b.WriteByte(ch)
+			lastMeaningful = ch
+			continue
+		}
+		if ch == ',' {
+			if lastMeaningful == '[' || lastMeaningful == '{' || lastMeaningful == ',' || lastMeaningful == 0 {
+				continue
+			}
+			if next := nextNonSpaceByte(s, i+1); next == ']' || next == '}' {
+				continue
+			}
+			b.WriteByte(ch)
+			lastMeaningful = ch
+			continue
+		}
+		b.WriteByte(ch)
+		if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' {
+			lastMeaningful = ch
+		}
+	}
+	return b.String()
+}
+
+func nextNonSpaceByte(s string, from int) byte {
+	for i := from; i < len(s); i++ {
+		switch s[i] {
+		case ' ', '\t', '\n', '\r':
+			continue
+		default:
+			return s[i]
+		}
+	}
+	return 0
+}

backend/internal/library/llmjson/sanitize_test.go

@@ -0,0 +1,57 @@
+package llmjson
+
+import "testing"
+
+func TestUnmarshalToleratesEllipsisAndCommas(t *testing.T) {
+	raw := []byte(`{
+  "nodes": [
+    {"label": "a", "url": "https://example.com/a"},
+    ...,
+    {"label": "b"},
+  ],
+  "edges": [...]
+}`)
+	var out struct {
+		Nodes []struct {
+			Label string `json:"label"`
+			URL   string `json:"url"`
+		} `json:"nodes"`
+		Edges []any `json:"edges"`
+	}
+	if err := Unmarshal(raw, &out); err != nil {
+		t.Fatalf("Unmarshal error: %v", err)
+	}
+	if len(out.Nodes) != 2 {
+		t.Fatalf("nodes = %d, want 2", len(out.Nodes))
+	}
+	if out.Nodes[0].URL != "https://example.com/a" {
+		t.Fatalf("url mutated: %q", out.Nodes[0].URL)
+	}
+}
+
+func TestSanitizeReturnsNilWhenValid(t *testing.T) {
+	if got := Sanitize([]byte(`{"a":1}`)); got != nil {
+		t.Fatalf("expected nil for clean json, got %q", got)
+	}
+}
+
+func TestSanitizeKeepsDotsInsideStrings(t *testing.T) {
+	raw := []byte(`{"note":"see ... and https://a.b/c", "list":[1, ...]}`)
+	cleaned := Sanitize(raw)
+	if cleaned == nil {
+		t.Fatal("expected sanitized output")
+	}
+	var out struct {
+		Note string `json:"note"`
+		List []int  `json:"list"`
+	}
+	if err := Unmarshal(raw, &out); err != nil {
+		t.Fatalf("Unmarshal error: %v", err)
+	}
+	if out.Note != "see ... and https://a.b/c" {
+		t.Fatalf("string content mutated: %q", out.Note)
+	}
+	if len(out.List) != 1 || out.List[0] != 1 {
+		t.Fatalf("list = %v, want [1]", out.List)
+	}
+}

backend/internal/library/placement/research_map.go

@@ -1,11 +1,12 @@
 package placement
 
 import (
-	"encoding/json"
 	"fmt"
 	"regexp"
 	"strings"
 	"unicode/utf8"
+
+	"haixun-backend/internal/library/llmjson"
 )
 
 type ResearchMap struct {
@@ -220,7 +221,7 @@ func ParseResearchMapOutput(raw string) (ResearchMap, error) {
 		return ResearchMap{}, err
 	}
 	var out ResearchMap
-	if err := json.Unmarshal(payload, &out); err != nil {
+	if err := llmjson.Unmarshal(payload, &out); err != nil {
 		return ResearchMap{}, fmt.Errorf("parse research map json: %w", err)
 	}
 	out.AudienceSummary = strings.TrimSpace(out.AudienceSummary)

backend/internal/library/viral/mission_research_map.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"strings"
+
+	"haixun-backend/internal/library/llmjson"
 )
 
 // MissionResearchMap is the structured copy-mission research map (single LLM call).
@@ -79,7 +81,7 @@ func ParseMissionResearchMapOutput(raw string) (MissionResearchMap, error) {
 		return MissionResearchMap{}, err
 	}
 	var root map[string]json.RawMessage
-	if err := json.Unmarshal(payload, &root); err != nil {
+	if err := llmjson.Unmarshal(payload, &root); err != nil {
 		return MissionResearchMap{}, fmt.Errorf("parse mission research map: %w", err)
 	}
 	out := MissionResearchMap{