thread-master/old/backend/internal/library/llmjson/sanitize.go

// Package llmjson tolerates the small JSON artifacts that LLMs sometimes emit
// (e.g. `...` placeholders, trailing or duplicated commas) so structured
// outputs can still be parsed instead of failing the whole job.
package llmjson

import (
	"encoding/json"
	"strings"
)

// Unmarshal parses payload into v, retrying once on a sanitized copy when the
// raw bytes are not strictly valid JSON. The original error is returned when
// the sanitized retry also fails, so callers keep a meaningful message.
func Unmarshal(payload []byte, v any) error {
	err := json.Unmarshal(payload, v)
	if err == nil {
		return nil
	}
	if cleaned := Sanitize(payload); cleaned != nil {
		if retryErr := json.Unmarshal(cleaned, v); retryErr == nil {
			return nil
		}
	}
	return err
}

// Sanitize removes ellipsis placeholders, escapes unescaped control characters
// inside JSON string literals, and normalizes stray commas that sit outside of
// string literals. It returns nil when nothing changed so callers can avoid a
// redundant re-parse. String meaning is preserved; only invalid JSON escapes
// are added.
func Sanitize(in []byte) []byte {
	normalized := normalizeCommas(stripEllipsis(escapeControlCharsInStrings(string(in))))
	if normalized == string(in) {
		return nil
	}
	return []byte(normalized)
}

func escapeControlCharsInStrings(s string) string {
	var b strings.Builder
	b.Grow(len(s) + 16)
	inString := false
	escaped := false
	for i := 0; i < len(s); i++ {
		ch := s[i]
		if inString {
			if escaped {
				escaped = false
				b.WriteByte(ch)
				continue
			}
			switch ch {
			case '\\':
				escaped = true
				b.WriteByte(ch)
			case '"':
				inString = false
				b.WriteByte(ch)
			case '\n':
				b.WriteString(`\n`)
			case '\r':
				b.WriteString(`\r`)
			case '\t':
				b.WriteString(`\t`)
			default:
				if ch < 0x20 {
					b.WriteString(`\u00`)
					b.WriteByte(hexDigit(ch >> 4))
					b.WriteByte(hexDigit(ch & 0x0f))
				} else {
					b.WriteByte(ch)
				}
			}
			continue
		}
		if ch == '"' {
			inString = true
		}
		b.WriteByte(ch)
	}
	return b.String()
}

func hexDigit(v byte) byte {
	if v < 10 {
		return '0' + v
	}
	return 'a' + (v - 10)
}

func stripEllipsis(s string) string {
	var b strings.Builder
	b.Grow(len(s))
	inString := false
	escaped := false
	for i := 0; i < len(s); i++ {
		ch := s[i]
		if inString {
			b.WriteByte(ch)
			if escaped {
				escaped = false
			} else if ch == '\\' {
				escaped = true
			} else if ch == '"' {
				inString = false
			}
			continue
		}
		if ch == '"' {
			inString = true
			b.WriteByte(ch)
			continue
		}
		if ch == '.' {
			j := i
			for j < len(s) && s[j] == '.' {
				j++
			}
			if j-i >= 2 {
				i = j - 1
				continue
			}
		}
		b.WriteByte(ch)
	}
	return b.String()
}

func normalizeCommas(s string) string {
	var b strings.Builder
	b.Grow(len(s))
	inString := false
	escaped := false
	var lastMeaningful byte
	for i := 0; i < len(s); i++ {
		ch := s[i]
		if inString {
			b.WriteByte(ch)
			if escaped {
				escaped = false
			} else if ch == '\\' {
				escaped = true
			} else if ch == '"' {
				inString = false
			}
			continue
		}
		if ch == '"' {
			inString = true
			b.WriteByte(ch)
			lastMeaningful = ch
			continue
		}
		if ch == ',' {
			if lastMeaningful == '[' || lastMeaningful == '{' || lastMeaningful == ',' || lastMeaningful == 0 {
				continue
			}
			if next := nextNonSpaceByte(s, i+1); next == ']' || next == '}' {
				continue
			}
			b.WriteByte(ch)
			lastMeaningful = ch
			continue
		}
		b.WriteByte(ch)
		if ch != ' ' && ch != '\t' && ch != '\n' && ch != '\r' {
			lastMeaningful = ch
		}
	}
	return b.String()
}

func nextNonSpaceByte(s string, from int) byte {
	for i := from; i < len(s); i++ {
		switch s[i] {
		case ' ', '\t', '\n', '\r':
			continue
		default:
			return s[i]
		}
	}
	return 0
}