fix(fetch): UTF-8 sniff boundary must not reject valid documents

validate_magic read exactly 4096 bytes and strict-decoded them; a multibyte
character straddling the boundary made perfectly valid .md/.txt files fail
with invalid_format. Only tolerate 'unexpected end of data' when the file
actually continues past the sniff window; genuinely corrupt bytes still fail.
This commit is contained in:
姚章浩
2026-09-15 09:57:41 +08:00
parent b3c9331966
commit 257be88ecf
2 changed files with 35 additions and 4 deletions
+23
View File
@@ -11,6 +11,29 @@ spec.loader.exec_module(artifact_fetch)
class DingTalkOnlineDocumentExportTest(unittest.TestCase):
def test_text_validation_tolerates_multibyte_char_cut_at_read_boundary(self):
# A valid UTF-8 document whose 4096-byte sniff boundary lands in the
# middle of a multibyte character must still pass validate_magic.
header = "# 标题\n".encode("utf-8") # 7 bytes
payload = header + b"a" * (4096 - len(header) - 1) + "机待\u2705".encode("utf-8") + b"\n"
self.assertEqual(payload[4095:4098], b"\xe6\x9c\xba") # a char straddles the sniff boundary
temp = pathlib.Path("/tmp/kg-md-boundary.md")
temp.write_bytes(payload)
try:
artifact_fetch.validate_magic(str(temp), "md")
finally:
temp.unlink()
def test_text_validation_still_rejects_genuinely_invalid_bytes(self):
temp = pathlib.Path("/tmp/kg-md-invalid.md")
temp.write_bytes(b"# \xe6\xa0\x87\xe9\xa2\x98\n\xff\xfe not utf-8\n" + b"a" * 8192)
try:
with self.assertRaises(artifact_fetch.FetchError) as raised:
artifact_fetch.validate_magic(str(temp), "md")
self.assertEqual(raised.exception.code, "invalid_format")
finally:
temp.unlink()
def test_adoc_uses_export_job_instead_of_ordinary_file_download(self):
calls = []