From eef4a0f985dbe248f1beb0a7eab6d94e0e868144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=80=A7=E9=A9=8A?= Date: Sun, 28 Jun 2026 19:58:15 +0800 Subject: [PATCH] fix2 --- backend/internal/library/knowledge/synth.go | 125 +---------------- backend/internal/library/llmjson/sanitize.go | 130 ++++++++++++++++++ .../internal/library/llmjson/sanitize_test.go | 57 ++++++++ .../library/placement/research_map.go | 5 +- .../library/viral/mission_research_map.go | 4 +- 5 files changed, 199 insertions(+), 122 deletions(-) create mode 100644 backend/internal/library/llmjson/sanitize.go create mode 100644 backend/internal/library/llmjson/sanitize_test.go diff --git a/backend/internal/library/knowledge/synth.go b/backend/internal/library/knowledge/synth.go index d7c3433..9c3cc6d 100644 --- a/backend/internal/library/knowledge/synth.go +++ b/backend/internal/library/knowledge/synth.go @@ -1,11 +1,11 @@ package knowledge import ( - "encoding/json" "fmt" "regexp" "strings" + "haixun-backend/internal/library/llmjson" libprompt "haixun-backend/internal/library/prompt" "github.com/google/uuid" @@ -322,128 +322,15 @@ func resolveNodeRef(ref string, labelToID map[string]string, nodes []Node) strin return "" } -// unmarshalSynthOutput parses the knowledge graph payload, retrying once on a -// sanitized copy because LLMs occasionally emit `...` placeholders or -// trailing/duplicate commas that break strict JSON parsing. +// unmarshalSynthOutput parses the knowledge graph payload, tolerating the small +// JSON artifacts (e.g. `...` placeholders, trailing/duplicate commas) that LLMs +// occasionally emit. func unmarshalSynthOutput(payload []byte) (rawSynthOutput, error) { var out rawSynthOutput - if err := json.Unmarshal(payload, &out); err == nil { - return out, nil - } else { - if cleaned := sanitizeLLMJSON(payload); cleaned != nil { - var retry rawSynthOutput - if err2 := json.Unmarshal(cleaned, &retry); err2 == nil { - return retry, nil - } - } + if err := llmjson.Unmarshal(payload, &out); err != nil { return rawSynthOutput{}, fmt.Errorf("parse knowledge graph json: %w", err) } -} - -// sanitizeLLMJSON removes ellipsis placeholders and normalizes stray commas -// outside of string literals. It returns nil when the input is unchanged so -// callers can skip a redundant re-parse. -func sanitizeLLMJSON(in []byte) []byte { - withoutEllipsis := stripEllipsis(string(in)) - normalized := normalizeCommas(withoutEllipsis) - if normalized == string(in) { - return nil - } - return []byte(normalized) -} - -func stripEllipsis(s string) string { - var b strings.Builder - b.Grow(len(s)) - inString := false - escaped := false - for i := 0; i < len(s); i++ { - ch := s[i] - if inString { - b.WriteByte(ch) - if escaped { - escaped = false - } else if ch == '\\' { - escaped = true - } else if ch == '"' { - inString = false - } - continue - } - if ch == '"' { - inString = true - b.WriteByte(ch) - continue - } - if ch == '.' { - j := i - for j < len(s) && s[j] == '.' { - j++ - } - if j-i >= 2 { - i = j - 1 - continue - } - } - b.WriteByte(ch) - } - return b.String() -} - -func normalizeCommas(s string) string { - var b strings.Builder - b.Grow(len(s)) - inString := false - escaped := false - var lastMeaningful byte - for i := 0; i < len(s); i++ { - ch := s[i] - if inString { - b.WriteByte(ch) - if escaped { - escaped = false - } else if ch == '\\' { - escaped = true - } else if ch == '"' { - inString = false - } - continue - } - if ch == '"' { - inString = true - b.WriteByte(ch) - lastMeaningful = ch - continue - } - if ch == ',' { - if lastMeaningful == '[' || lastMeaningful == '{' || lastMeaningful == ',' || lastMeaningful == 0 { - continue - } - if next := nextNonSpaceByte(s, i+1); next == ']' || next == '}' { - continue - } - b.WriteByte(ch) - lastMeaningful = ch - continue - } - b.WriteByte(ch) - if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' { - lastMeaningful = ch - } - } - return b.String() -} - -func nextNonSpaceByte(s string, from int) byte { - for i := from; i < len(s); i++ { - switch s[i] { - case ' ', '\t', '\n', '\r': - continue - default: - return s[i] - } - } - return 0 + return out, nil } func extractJSONObject(raw string) ([]byte, error) { diff --git a/backend/internal/library/llmjson/sanitize.go b/backend/internal/library/llmjson/sanitize.go new file mode 100644 index 0000000..9ad9440 --- /dev/null +++ b/backend/internal/library/llmjson/sanitize.go @@ -0,0 +1,130 @@ +// Package llmjson tolerates the small JSON artifacts that LLMs sometimes emit +// (e.g. `...` placeholders, trailing or duplicated commas) so structured +// outputs can still be parsed instead of failing the whole job. +package llmjson + +import ( + "encoding/json" + "strings" +) + +// Unmarshal parses payload into v, retrying once on a sanitized copy when the +// raw bytes are not strictly valid JSON. The original error is returned when +// the sanitized retry also fails, so callers keep a meaningful message. +func Unmarshal(payload []byte, v any) error { + err := json.Unmarshal(payload, v) + if err == nil { + return nil + } + if cleaned := Sanitize(payload); cleaned != nil { + if retryErr := json.Unmarshal(cleaned, v); retryErr == nil { + return nil + } + } + return err +} + +// Sanitize removes ellipsis placeholders and normalizes stray commas that sit +// outside of string literals. It returns nil when nothing changed so callers +// can avoid a redundant re-parse. String/URL contents are never altered. +func Sanitize(in []byte) []byte { + normalized := normalizeCommas(stripEllipsis(string(in))) + if normalized == string(in) { + return nil + } + return []byte(normalized) +} + +func stripEllipsis(s string) string { + var b strings.Builder + b.Grow(len(s)) + inString := false + escaped := false + for i := 0; i < len(s); i++ { + ch := s[i] + if inString { + b.WriteByte(ch) + if escaped { + escaped = false + } else if ch == '\\' { + escaped = true + } else if ch == '"' { + inString = false + } + continue + } + if ch == '"' { + inString = true + b.WriteByte(ch) + continue + } + if ch == '.' { + j := i + for j < len(s) && s[j] == '.' { + j++ + } + if j-i >= 2 { + i = j - 1 + continue + } + } + b.WriteByte(ch) + } + return b.String() +} + +func normalizeCommas(s string) string { + var b strings.Builder + b.Grow(len(s)) + inString := false + escaped := false + var lastMeaningful byte + for i := 0; i < len(s); i++ { + ch := s[i] + if inString { + b.WriteByte(ch) + if escaped { + escaped = false + } else if ch == '\\' { + escaped = true + } else if ch == '"' { + inString = false + } + continue + } + if ch == '"' { + inString = true + b.WriteByte(ch) + lastMeaningful = ch + continue + } + if ch == ',' { + if lastMeaningful == '[' || lastMeaningful == '{' || lastMeaningful == ',' || lastMeaningful == 0 { + continue + } + if next := nextNonSpaceByte(s, i+1); next == ']' || next == '}' { + continue + } + b.WriteByte(ch) + lastMeaningful = ch + continue + } + b.WriteByte(ch) + if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' { + lastMeaningful = ch + } + } + return b.String() +} + +func nextNonSpaceByte(s string, from int) byte { + for i := from; i < len(s); i++ { + switch s[i] { + case ' ', '\t', '\n', '\r': + continue + default: + return s[i] + } + } + return 0 +} diff --git a/backend/internal/library/llmjson/sanitize_test.go b/backend/internal/library/llmjson/sanitize_test.go new file mode 100644 index 0000000..9b2d566 --- /dev/null +++ b/backend/internal/library/llmjson/sanitize_test.go @@ -0,0 +1,57 @@ +package llmjson + +import "testing" + +func TestUnmarshalToleratesEllipsisAndCommas(t *testing.T) { + raw := []byte(`{ + "nodes": [ + {"label": "a", "url": "https://example.com/a"}, + ..., + {"label": "b"}, + ], + "edges": [...] +}`) + var out struct { + Nodes []struct { + Label string `json:"label"` + URL string `json:"url"` + } `json:"nodes"` + Edges []any `json:"edges"` + } + if err := Unmarshal(raw, &out); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if len(out.Nodes) != 2 { + t.Fatalf("nodes = %d, want 2", len(out.Nodes)) + } + if out.Nodes[0].URL != "https://example.com/a" { + t.Fatalf("url mutated: %q", out.Nodes[0].URL) + } +} + +func TestSanitizeReturnsNilWhenValid(t *testing.T) { + if got := Sanitize([]byte(`{"a":1}`)); got != nil { + t.Fatalf("expected nil for clean json, got %q", got) + } +} + +func TestSanitizeKeepsDotsInsideStrings(t *testing.T) { + raw := []byte(`{"note":"see ... and https://a.b/c", "list":[1, ...]}`) + cleaned := Sanitize(raw) + if cleaned == nil { + t.Fatal("expected sanitized output") + } + var out struct { + Note string `json:"note"` + List []int `json:"list"` + } + if err := Unmarshal(raw, &out); err != nil { + t.Fatalf("Unmarshal error: %v", err) + } + if out.Note != "see ... and https://a.b/c" { + t.Fatalf("string content mutated: %q", out.Note) + } + if len(out.List) != 1 || out.List[0] != 1 { + t.Fatalf("list = %v, want [1]", out.List) + } +} diff --git a/backend/internal/library/placement/research_map.go b/backend/internal/library/placement/research_map.go index 7754ce6..c2cff1d 100644 --- a/backend/internal/library/placement/research_map.go +++ b/backend/internal/library/placement/research_map.go @@ -1,11 +1,12 @@ package placement import ( - "encoding/json" "fmt" "regexp" "strings" "unicode/utf8" + + "haixun-backend/internal/library/llmjson" ) type ResearchMap struct { @@ -220,7 +221,7 @@ func ParseResearchMapOutput(raw string) (ResearchMap, error) { return ResearchMap{}, err } var out ResearchMap - if err := json.Unmarshal(payload, &out); err != nil { + if err := llmjson.Unmarshal(payload, &out); err != nil { return ResearchMap{}, fmt.Errorf("parse research map json: %w", err) } out.AudienceSummary = strings.TrimSpace(out.AudienceSummary) diff --git a/backend/internal/library/viral/mission_research_map.go b/backend/internal/library/viral/mission_research_map.go index 01ee564..0826115 100644 --- a/backend/internal/library/viral/mission_research_map.go +++ b/backend/internal/library/viral/mission_research_map.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "strings" + + "haixun-backend/internal/library/llmjson" ) // MissionResearchMap is the structured copy-mission research map (single LLM call). @@ -79,7 +81,7 @@ func ParseMissionResearchMapOutput(raw string) (MissionResearchMap, error) { return MissionResearchMap{}, err } var root map[string]json.RawMessage - if err := json.Unmarshal(payload, &root); err != nil { + if err := llmjson.Unmarshal(payload, &root); err != nil { return MissionResearchMap{}, fmt.Errorf("parse mission research map: %w", err) } out := MissionResearchMap{