fix(fetch): UTF-8 sniff boundary must not reject valid documents
validate_magic read exactly 4096 bytes and strict-decoded them; a multibyte character straddling the boundary made perfectly valid .md/.txt files fail with invalid_format. Only tolerate 'unexpected end of data' when the file actually continues past the sniff window; genuinely corrupt bytes still fail.
This commit is contained in:
@@ -93,11 +93,19 @@ def validate_magic(path, suffix):
|
||||
raise FetchError("downloaded bytes are not a valid %s file" % suffix,
|
||||
retryable=False, code="invalid_format")
|
||||
if suffix in ("md", "txt", "csv", "json"):
|
||||
with open(path, "rb") as handle:
|
||||
head = handle.read(4096)
|
||||
continues = bool(handle.read(1))
|
||||
try:
|
||||
with open(path, "rb") as handle:
|
||||
handle.read(4096).decode("utf-8", errors="strict")
|
||||
except UnicodeDecodeError:
|
||||
raise FetchError("text artifact is not valid UTF-8", retryable=False, code="invalid_format")
|
||||
head.decode("utf-8", errors="strict")
|
||||
except UnicodeDecodeError as error:
|
||||
# The sniff boundary can cut a multibyte character in half; that
|
||||
# is not corruption while the file has bytes beyond the sniff.
|
||||
boundary_cut = (continues and error.reason == "unexpected end of data"
|
||||
and error.start >= len(head) - 3)
|
||||
if not boundary_cut:
|
||||
raise FetchError("text artifact is not valid UTF-8", retryable=False, code="invalid_format")
|
||||
head[:error.start].decode("utf-8", errors="strict")
|
||||
|
||||
|
||||
def pick_suffix(name, declared_type):
|
||||
|
||||
Reference in New Issue
Block a user