fix(fetch): UTF-8 sniff boundary must not reject valid documents
validate_magic read exactly 4096 bytes and strict-decoded them; a multibyte character straddling the boundary made perfectly valid .md/.txt files fail with invalid_format. Only tolerate 'unexpected end of data' when the file actually continues past the sniff window; genuinely corrupt bytes still fail.
This commit is contained in:
@@ -11,6 +11,29 @@ spec.loader.exec_module(artifact_fetch)
|
||||
|
||||
|
||||
class DingTalkOnlineDocumentExportTest(unittest.TestCase):
|
||||
def test_text_validation_tolerates_multibyte_char_cut_at_read_boundary(self):
|
||||
# A valid UTF-8 document whose 4096-byte sniff boundary lands in the
|
||||
# middle of a multibyte character must still pass validate_magic.
|
||||
header = "# 标题\n".encode("utf-8") # 7 bytes
|
||||
payload = header + b"a" * (4096 - len(header) - 1) + "机待\u2705".encode("utf-8") + b"\n"
|
||||
self.assertEqual(payload[4095:4098], b"\xe6\x9c\xba") # a char straddles the sniff boundary
|
||||
temp = pathlib.Path("/tmp/kg-md-boundary.md")
|
||||
temp.write_bytes(payload)
|
||||
try:
|
||||
artifact_fetch.validate_magic(str(temp), "md")
|
||||
finally:
|
||||
temp.unlink()
|
||||
|
||||
def test_text_validation_still_rejects_genuinely_invalid_bytes(self):
|
||||
temp = pathlib.Path("/tmp/kg-md-invalid.md")
|
||||
temp.write_bytes(b"# \xe6\xa0\x87\xe9\xa2\x98\n\xff\xfe not utf-8\n" + b"a" * 8192)
|
||||
try:
|
||||
with self.assertRaises(artifact_fetch.FetchError) as raised:
|
||||
artifact_fetch.validate_magic(str(temp), "md")
|
||||
self.assertEqual(raised.exception.code, "invalid_format")
|
||||
finally:
|
||||
temp.unlink()
|
||||
|
||||
def test_adoc_uses_export_job_instead_of_ordinary_file_download(self):
|
||||
calls = []
|
||||
|
||||
|
||||
Reference in New Issue
Block a user