From 6817e920df6689ae169fbfe1770f25b2c81f998a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:39:25 +0200 Subject: [PATCH 1/4] gh-155419: honor the file object's current position in `hashlib .file_digest` --- Lib/hashlib.py | 3 +- Lib/test/test_hashlib.py | 72 +++++++++++-------- ...-08-09-13-23-12.gh-issue-155419.J6dSFJ.rst | 3 + 3 files changed, 49 insertions(+), 29 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-08-09-13-23-12.gh-issue-155419.J6dSFJ.rst diff --git a/Lib/hashlib.py b/Lib/hashlib.py index c1e2d07c4357ff9..43175d11f87046f 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -233,7 +233,8 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18): if hasattr(fileobj, "getbuffer"): # io.BytesIO object, use zero-copy buffer - digestobj.update(fileobj.getbuffer()) + with fileobj.getbuffer() as buf: + digestobj.update(buf[fileobj.tell():]) return digestobj # Only binary files implement readinto(). diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 253a8f455853f58..2b67572b412b4ea 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -12,6 +12,7 @@ import itertools import logging import os +import random import re import sys import sysconfig @@ -102,6 +103,20 @@ def read_vectors(hash_name): ) +def make_hash_objects(*digestmods, buf=b"", **kwargs): + objects = [] + for digestmod in digestmods: + try: + if callable(digestmod): + obj = digestmod(buf) + else: + obj = hashlib.new(digestmod, buf, **kwargs) + except ValueError: + continue + objects.append((digestmod, obj)) + return objects + + class HashLibTestCase(unittest.TestCase): supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1', 'sha224', 'SHA224', 'sha256', 'SHA256', @@ -524,7 +539,8 @@ def test_blake2_update_over_4gb(self): self.assertEqual(h.hexdigest(), "8a268e83dd30528bc0907fa2008c91de8f090a0b6e0e60a5ff0d999d8485526f") def check(self, name, data, hexdigest, shake=False, **kwargs): - length = len(hexdigest)//2 + n = len(data) + length = len(hexdigest) // 2 hexdigest = hexdigest.lower() constructors = self.constructors_to_test[name] # 2 is for hashlib.name(...) and hashlib.new(name, ...) @@ -533,47 +549,47 @@ def check(self, name, data, hexdigest, shake=False, **kwargs): m = hash_object_constructor(data, **kwargs) computed = m.hexdigest() if not shake else m.hexdigest(length) self.assertEqual( - computed, hexdigest, - "Hash algorithm %s constructed using %s returned hexdigest" - " %r for %d byte input data that should have hashed to %r." - % (name, hash_object_constructor, - computed, len(data), hexdigest)) + computed, hexdigest, + "Hash algorithm %s constructed using %s returned hexdigest" + " %r for %d byte input data that should have hashed to %r." + % (name, hash_object_constructor, computed, n, hexdigest) + ) computed = m.digest() if not shake else m.digest(length) digest = bytes.fromhex(hexdigest) self.assertEqual(computed, digest) if not shake: self.assertEqual(len(digest), m.digest_size) + def generate_sub_hexdigest(pos=-1): + if pos < 0: + pos = 0 if n == 0 else random.randrange(0, n) + pos += 1 # ensure pos is at least 1 (allowed to be out of range) + buf = data[pos:] + objects = make_hash_objects(name, *constructors, buf=buf, **kwargs) + hexdigests = {obj.hexdigest() for _, obj in objects} + self.assertEqual(len(hexdigests), 1, f"bad digests: {objects}") + return pos, hexdigests.pop() + if not shake and kwargs.get("key") is None: # skip shake and blake2 extended parameter tests self.check_file_digest(name, data, hexdigest) + for pos in sorted({-1, 1, n - 1, n, n + 1}): + with self.subTest(pos=pos): + pos, hexdigest2 = generate_sub_hexdigest(pos=pos) + self.check_file_digest(name, data, hexdigest2, pos) - def check_file_digest(self, name, data, hexdigest): + def check_file_digest(self, name, data, hexdigest, pos=0): hexdigest = hexdigest.lower() - digests = [] - for digest in [name, *self.constructors_to_test[name]]: - try: - if callable(digest): - digest(b"") - else: - hashlib.new(digest) - except ValueError: - # skip, algorithm is blocked by security policy. - continue - digests.append(digest) - + digests = make_hash_objects(name, *self.constructors_to_test[name]) with tempfile.TemporaryFile() as f: f.write(data) + buf = io.BytesIO(data) - for digest in digests: - buf = io.BytesIO(data) - buf.seek(0) - self.assertEqual( - hashlib.file_digest(buf, digest).hexdigest(), hexdigest - ) - f.seek(0) - digestobj = hashlib.file_digest(f, digest) - self.assertEqual(digestobj.hexdigest(), hexdigest) + for digest, _ in digests: + for fobj in [buf, f]: + fobj.seek(pos) + digestobj = hashlib.file_digest(fobj, digest) + self.assertEqual(digestobj.hexdigest(), hexdigest) def check_no_unicode(self, algorithm_name): # Unicode objects are not allowed as input. diff --git a/Misc/NEWS.d/next/Library/2026-08-09-13-23-12.gh-issue-155419.J6dSFJ.rst b/Misc/NEWS.d/next/Library/2026-08-09-13-23-12.gh-issue-155419.J6dSFJ.rst new file mode 100644 index 000000000000000..2117f189d64871c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-09-13-23-12.gh-issue-155419.J6dSFJ.rst @@ -0,0 +1,3 @@ +Ensure that :func:`hashlib.file_digest` honors the current position of the +file object when given an :class:`io.BytesIO` object. Patch by Bénédikt +Tran. From 8b5acaec3eca6070bb035cc33aa0a5a72cee698a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:51:42 -0700 Subject: [PATCH 2/4] gh-155419: use the `hashlib.file_digest` fast path for `io.BytesIO` only Objects exposing `getbuffer()` without being `io.BytesIO` instances are now read through `readinto()`. The `io` module is imported lazily. --- Lib/hashlib.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 43175d11f87046f..fa7132fcb530bfa 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -53,6 +53,8 @@ '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406' """ +lazy import io + # This tuple and __get_builtin_constructor() must be modified if a new # always available algorithm is added. __always_supported = ('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', @@ -231,7 +233,7 @@ def file_digest(fileobj, digest, /, *, _bufsize=2**18): else: digestobj = digest() - if hasattr(fileobj, "getbuffer"): + if isinstance(fileobj, io.BytesIO): # io.BytesIO object, use zero-copy buffer with fileobj.getbuffer() as buf: digestobj.update(buf[fileobj.tell():]) From 644ac60877e0648431101fce52671ee4317ea167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:51:52 -0700 Subject: [PATCH 3/4] gh-155419: test that importing `hashlib` does not import `io` --- Lib/test/test_hashlib.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 2b67572b412b4ea..4d1a448d998887c 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -22,7 +22,7 @@ from test import support from test.support import _4G, bigmemtest from test.support import hashlib_helper -from test.support.import_helper import import_fresh_module +from test.support.import_helper import ensure_lazy_imports, import_fresh_module from test.support import requires_resource from test.support import threading_helper from http.client import HTTPException @@ -1434,5 +1434,11 @@ def scrypt(password=b"password", /, **kwargs): self.assertRaises(numeric_exc_types, scrypt, dklen=MAX_DKLEN + 1) +class LazyImportTest(unittest.TestCase): + @support.cpython_only + def test_lazy_import(self): + ensure_lazy_imports("hashlib", {"io"}) + + if __name__ == "__main__": unittest.main() From 7309e0f99489ec0f92a1543fbd99233a4d09f562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:51:52 -0700 Subject: [PATCH 4/4] gh-155419: test `hashlib.file_digest` on a non-`io.BytesIO` buffer `BytesIOProxy` exposes the `io.BytesIO` interface without being an instance of it, so `hashlib.file_digest` reads it with `readinto()` from its current position. --- Lib/test/test_hashlib.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 4d1a448d998887c..b53b6edca3a7d66 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -117,6 +117,16 @@ def make_hash_objects(*digestmods, buf=b"", **kwargs): return objects +class BytesIOProxy: + """A file-like object with the io.BytesIO interface, not an io.BytesIO.""" + + def __init__(self, data): + self._buffer = io.BytesIO(data) + + def __getattr__(self, name): + return getattr(self._buffer, name) + + class HashLibTestCase(unittest.TestCase): supported_hash_names = ( 'md5', 'MD5', 'sha1', 'SHA1', 'sha224', 'SHA224', 'sha256', 'SHA256', @@ -584,9 +594,10 @@ def check_file_digest(self, name, data, hexdigest, pos=0): with tempfile.TemporaryFile() as f: f.write(data) buf = io.BytesIO(data) + buf_like = BytesIOProxy(data) for digest, _ in digests: - for fobj in [buf, f]: + for fobj in [buf, buf_like, f]: fobj.seek(pos) digestobj = hashlib.file_digest(fobj, digest) self.assertEqual(digestobj.hexdigest(), hexdigest)