fix(feishu): truncate sanitized filenames at UTF-8 rune boundary
Raw byte truncation at 200 bytes splits multi-byte codepoints (Chinese characters are 3 bytes), producing invalid UTF-8 that downstream filename validation (utf8.ValidString) rejects with "文件名包含非法字符". Peel trailing bytes via DecodeLastRuneInString until the last rune is complete. Same latent bug and fix as the Yuque connector's sanitizeFileName.
This commit is contained in:
@@ -7,6 +7,7 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
@@ -357,20 +358,29 @@ func parseFeishuTimestamp(ts string) time.Time {
|
||||
return time.Unix(sec, 0)
|
||||
}
|
||||
|
||||
// sanitizeFileName removes characters that are invalid in filenames.
|
||||
// sanitizeFileName removes characters that are invalid in filenames and
|
||||
// truncates at a UTF-8 rune boundary. Raw byte truncation would split a
|
||||
// multi-byte codepoint (Chinese chars are 3 bytes) and produce invalid UTF-8
|
||||
// that downstream validation (utf8.ValidString) rejects.
|
||||
func sanitizeFileName(name string) string {
|
||||
if name == "" {
|
||||
return "untitled"
|
||||
}
|
||||
// Replace common invalid characters with underscore
|
||||
replacer := strings.NewReplacer(
|
||||
"/", "_", "\\", "_", ":", "_", "*", "_",
|
||||
"?", "_", "\"", "_", "<", "_", ">", "_", "|", "_",
|
||||
)
|
||||
result := replacer.Replace(name)
|
||||
// Limit length
|
||||
if len(result) > 200 {
|
||||
result = result[:200]
|
||||
const maxBytes = 200
|
||||
if len(result) > maxBytes {
|
||||
result = result[:maxBytes]
|
||||
for len(result) > 0 {
|
||||
r, size := utf8.DecodeLastRuneInString(result)
|
||||
if r != utf8.RuneError || size != 1 {
|
||||
break
|
||||
}
|
||||
result = result[:len(result)-1]
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/Tencent/WeKnora/internal/types"
|
||||
)
|
||||
@@ -222,6 +223,22 @@ func TestSanitizeFileName(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSanitizeFileName_TruncatesAtRuneBoundary(t *testing.T) {
|
||||
// Each 测 is 3 bytes; raw byte truncation at 200 would split a rune and
|
||||
// produce invalid UTF-8 that downstream filename validation rejects.
|
||||
long := strings.Repeat("测试", 100)
|
||||
got := sanitizeFileName(long)
|
||||
if !utf8.ValidString(got) {
|
||||
t.Fatalf("sanitizeFileName produced invalid UTF-8: %q", got)
|
||||
}
|
||||
if len(got) > 200 {
|
||||
t.Errorf("len = %d, want ≤ 200", len(got))
|
||||
}
|
||||
if len(got) == 0 {
|
||||
t.Error("result is empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseFeishuTimestamp(t *testing.T) {
|
||||
ts := parseFeishuTimestamp("1711468800") // 2024-03-27 00:00:00 UTC
|
||||
if ts.IsZero() {
|
||||
|
||||
Reference in New Issue
Block a user