Fix UTF-8 insert failures in Postgres

This commit is contained in:
binwiederhier
2026-03-15 21:03:18 -04:00
parent 1afb99db67
commit 4699ed3ffd
8 changed files with 287 additions and 12 deletions
+20
View File
@@ -17,6 +17,7 @@ import (
"strings"
"sync"
"time"
"unicode/utf8"
"github.com/gabriel-vasile/mimetype"
"golang.org/x/term"
@@ -434,3 +435,22 @@ func Int(v int) *int {
func Time(v time.Time) *time.Time {
return &v
}
// SanitizeUTF8 ensures a string is safe to store in PostgreSQL by handling two cases:
//
// 1. Invalid UTF-8 sequences: Some clients send Latin-1/ISO-8859-1 encoded text (e.g. accented
// characters like é, ñ, ß) in HTTP headers or SMTP messages. Go treats these as raw bytes in
// strings, but PostgreSQL rejects them. Any invalid UTF-8 byte is replaced with the Unicode
// replacement character (U+FFFD, "�") so the message is still delivered rather than lost.
//
// 2. NUL bytes (0x00): These are valid in UTF-8 but PostgreSQL TEXT columns reject them.
// They are stripped entirely.
func SanitizeUTF8(s string) string {
if !utf8.ValidString(s) {
s = strings.ToValidUTF8(s, "\xef\xbf\xbd") // U+FFFD
}
if strings.ContainsRune(s, 0) {
s = strings.ReplaceAll(s, "\x00", "")
}
return s
}