diff --git a/scripts/artifact_fetch.py b/scripts/artifact_fetch.py index 4c0c179..7ce824b 100644 --- a/scripts/artifact_fetch.py +++ b/scripts/artifact_fetch.py @@ -93,11 +93,19 @@ def validate_magic(path, suffix): raise FetchError("downloaded bytes are not a valid %s file" % suffix, retryable=False, code="invalid_format") if suffix in ("md", "txt", "csv", "json"): + with open(path, "rb") as handle: + head = handle.read(4096) + continues = bool(handle.read(1)) try: - with open(path, "rb") as handle: - handle.read(4096).decode("utf-8", errors="strict") - except UnicodeDecodeError: - raise FetchError("text artifact is not valid UTF-8", retryable=False, code="invalid_format") + head.decode("utf-8", errors="strict") + except UnicodeDecodeError as error: + # The sniff boundary can cut a multibyte character in half; that + # is not corruption while the file has bytes beyond the sniff. + boundary_cut = (continues and error.reason == "unexpected end of data" + and error.start >= len(head) - 3) + if not boundary_cut: + raise FetchError("text artifact is not valid UTF-8", retryable=False, code="invalid_format") + head[:error.start].decode("utf-8", errors="strict") def pick_suffix(name, declared_type): diff --git a/scripts/artifact_fetch_test.py b/scripts/artifact_fetch_test.py index a4022a7..f5b2d9a 100644 --- a/scripts/artifact_fetch_test.py +++ b/scripts/artifact_fetch_test.py @@ -11,6 +11,29 @@ spec.loader.exec_module(artifact_fetch) class DingTalkOnlineDocumentExportTest(unittest.TestCase): + def test_text_validation_tolerates_multibyte_char_cut_at_read_boundary(self): + # A valid UTF-8 document whose 4096-byte sniff boundary lands in the + # middle of a multibyte character must still pass validate_magic. + header = "# 标题\n".encode("utf-8") # 7 bytes + payload = header + b"a" * (4096 - len(header) - 1) + "机待\u2705".encode("utf-8") + b"\n" + self.assertEqual(payload[4095:4098], b"\xe6\x9c\xba") # a char straddles the sniff boundary + temp = pathlib.Path("/tmp/kg-md-boundary.md") + temp.write_bytes(payload) + try: + artifact_fetch.validate_magic(str(temp), "md") + finally: + temp.unlink() + + def test_text_validation_still_rejects_genuinely_invalid_bytes(self): + temp = pathlib.Path("/tmp/kg-md-invalid.md") + temp.write_bytes(b"# \xe6\xa0\x87\xe9\xa2\x98\n\xff\xfe not utf-8\n" + b"a" * 8192) + try: + with self.assertRaises(artifact_fetch.FetchError) as raised: + artifact_fetch.validate_magic(str(temp), "md") + self.assertEqual(raised.exception.code, "invalid_format") + finally: + temp.unlink() + def test_adoc_uses_export_job_instead_of_ordinary_file_download(self): calls = []