diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 9c0679d1d13d23..a75bfef0a69a5a 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -5182,6 +5182,26 @@ def loads(data, **kwargs): self.check_dumps_loads_oob_buffers(dumps, loads) + def test_load_large_from_file_without_readinto(self): + # gh-158841: large data is read by chunks of increasing size. + class BytesSubclass(bytes): + pass + + obj = 'B' * (3 << 20) + for proto in range(1, pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + data = self.dumps(obj, proto) + self.assertEqual(self.load(MinimalIO(data)), obj) + + f = MinimalIO(data) + f.read = lambda n, read=f.read: BytesSubclass(read(n)) + self.assertEqual(self.load(f), obj) + + f = MinimalIO(data) + f.read = lambda n, read=f.read: read(n // 2 if n > 1000 else n) + with self.assertRaises((pickle.UnpicklingError, EOFError)): + self.load(f) + class AbstractPersistentPicklerTests: diff --git a/Misc/NEWS.d/next/Library/2026-10-22-12-00-00.gh-issue-158841.Rd7KcH.rst b/Misc/NEWS.d/next/Library/2026-10-22-12-00-00.gh-issue-158841.Rd7KcH.rst new file mode 100644 index 00000000000000..3168d207c5cc95 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-12-00-00.gh-issue-158841.Rd7KcH.rst @@ -0,0 +1,5 @@ +Fix a crash and reading of uninitialized memory in :func:`pickle.load` and +:class:`pickle.Unpickler` when reading a large amount of data from a file +object without the ``readinto()`` method, +if its ``read()`` method returns an instance of a :class:`bytes` subclass +or less data than requested. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index f37c0a5bba5541..0a27097030e42b 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1445,27 +1445,41 @@ _Unpickler_ReadFromFile(PickleState *state, UnpicklerObject *self, Py_ssize_t n) return n; } } - Py_ssize_t cursize = Py_MIN(n, MIN_READ_BUF_SIZE); - len = PyLong_FromSsize_t(cursize); - if (len == NULL) - return -1; - data = _Pickle_FastCall(self->read, len); - if (data == NULL) { - return -1; - } - while (cursize < n) { - Py_ssize_t prevsize = cursize; - // geometrically double the chunk size to avoid CPU DoS - cursize += Py_MIN(cursize, n - cursize); - if (_PyBytes_Resize(&data, cursize) < 0) { + if (n <= MIN_READ_BUF_SIZE) { + len = PyLong_FromSsize_t(n); + if (len == NULL) + return -1; + data = _Pickle_FastCall(self->read, len); + if (data == NULL) { return -1; } - if (_Unpickler_ReadIntoFromFile(state, self, - PyBytes_AS_STRING(data) + prevsize, cursize - prevsize) < 0) - { - Py_DECREF(data); + } + else { + // Read by chunks of increasing size to avoid memory DoS: + // do not preallocate memory for data which may be truncated. + Py_ssize_t cursize = MIN_READ_BUF_SIZE; + Py_ssize_t prevsize = 0; + data = PyBytes_FromStringAndSize(NULL, cursize); + if (data == NULL) { return -1; } + while (1) { + if (_Unpickler_ReadIntoFromFile(state, self, + PyBytes_AS_STRING(data) + prevsize, + cursize - prevsize) < 0) + { + Py_DECREF(data); + return -1; + } + if (cursize >= n) { + break; + } + prevsize = cursize; + cursize += Py_MIN(cursize, n - cursize); + if (_PyBytes_Resize(&data, cursize) < 0) { + return -1; + } + } } }