diff --git a/internal/datasource/connector/feishu/connector.go b/internal/datasource/connector/feishu/connector.go index 2f45417e..d49da72c 100644 --- a/internal/datasource/connector/feishu/connector.go +++ b/internal/datasource/connector/feishu/connector.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "time" + "unicode/utf8" "github.com/Tencent/WeKnora/internal/types" ) @@ -357,20 +358,29 @@ func parseFeishuTimestamp(ts string) time.Time { return time.Unix(sec, 0) } -// sanitizeFileName removes characters that are invalid in filenames. +// sanitizeFileName removes characters that are invalid in filenames and +// truncates at a UTF-8 rune boundary. Raw byte truncation would split a +// multi-byte codepoint (Chinese chars are 3 bytes) and produce invalid UTF-8 +// that downstream validation (utf8.ValidString) rejects. func sanitizeFileName(name string) string { if name == "" { return "untitled" } - // Replace common invalid characters with underscore replacer := strings.NewReplacer( "/", "_", "\\", "_", ":", "_", "*", "_", "?", "_", "\"", "_", "<", "_", ">", "_", "|", "_", ) result := replacer.Replace(name) - // Limit length - if len(result) > 200 { - result = result[:200] + const maxBytes = 200 + if len(result) > maxBytes { + result = result[:maxBytes] + for len(result) > 0 { + r, size := utf8.DecodeLastRuneInString(result) + if r != utf8.RuneError || size != 1 { + break + } + result = result[:len(result)-1] + } } return result } diff --git a/internal/datasource/connector/feishu/connector_test.go b/internal/datasource/connector/feishu/connector_test.go index b74092cc..5184cde2 100644 --- a/internal/datasource/connector/feishu/connector_test.go +++ b/internal/datasource/connector/feishu/connector_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" "time" + "unicode/utf8" "github.com/Tencent/WeKnora/internal/types" ) @@ -222,6 +223,22 @@ func TestSanitizeFileName(t *testing.T) { } } +func TestSanitizeFileName_TruncatesAtRuneBoundary(t *testing.T) { + // Each 测 is 3 bytes; raw byte truncation at 200 would split a rune and + // produce invalid UTF-8 that downstream filename validation rejects. + long := strings.Repeat("测试", 100) + got := sanitizeFileName(long) + if !utf8.ValidString(got) { + t.Fatalf("sanitizeFileName produced invalid UTF-8: %q", got) + } + if len(got) > 200 { + t.Errorf("len = %d, want ≤ 200", len(got)) + } + if len(got) == 0 { + t.Error("result is empty") + } +} + func TestParseFeishuTimestamp(t *testing.T) { ts := parseFeishuTimestamp("1711468800") // 2024-03-27 00:00:00 UTC if ts.IsZero() {