fix(fetch): UTF-8 sniff boundary must not reject valid documents

validate_magic read exactly 4096 bytes and strict-decoded them; a multibyte
character straddling the boundary made perfectly valid .md/.txt files fail
with invalid_format. Only tolerate 'unexpected end of data' when the file
actually continues past the sniff window; genuinely corrupt bytes still fail.
This commit is contained in:
姚章浩
2026-09-15 09:57:41 +08:00
parent b3c9331966
commit 257be88ecf
2 changed files with 35 additions and 4 deletions
+12 -4
View File
@@ -93,11 +93,19 @@ def validate_magic(path, suffix):
raise FetchError("downloaded bytes are not a valid %s file" % suffix,
retryable=False, code="invalid_format")
if suffix in ("md", "txt", "csv", "json"):
with open(path, "rb") as handle:
head = handle.read(4096)
continues = bool(handle.read(1))
try:
with open(path, "rb") as handle:
handle.read(4096).decode("utf-8", errors="strict")
except UnicodeDecodeError:
raise FetchError("text artifact is not valid UTF-8", retryable=False, code="invalid_format")
head.decode("utf-8", errors="strict")
except UnicodeDecodeError as error:
# The sniff boundary can cut a multibyte character in half; that
# is not corruption while the file has bytes beyond the sniff.
boundary_cut = (continues and error.reason == "unexpected end of data"
and error.start >= len(head) - 3)
if not boundary_cut:
raise FetchError("text artifact is not valid UTF-8", retryable=False, code="invalid_format")
head[:error.start].decode("utf-8", errors="strict")
def pick_suffix(name, declared_type):