Issue #21932: os.read() now uses a :c:func:Py_ssize_t type instead of

:c:type:`int` for the size to support reading more than 2 GB at once. On
Windows, the size is truncted to INT_MAX. As any call to os.read(), the OS
may read less bytes than the number of requested bytes.

Author: vstinner

Lib/test/test_os.py

@@ -43,6 +43,10 @@
     import _winapi
 except ImportError:
     _winapi = None
+try:
+    from _testcapi import INT_MAX
+except ImportError:
+    INT_MAX = 2 ** 31 - 1
 
 from test.script_helper import assert_python_ok
 
@@ -119,6 +123,21 @@ def test_read(self):
             self.assertEqual(type(s), bytes)
             self.assertEqual(s, b"spam")
 
+    def test_large_read(self):
+        with open(support.TESTFN, "wb") as fp:
+            fp.write(b'test')
+        self.addCleanup(support.unlink, support.TESTFN)
+
+        # Issue #21932: Make sure that os.read() does not raise an
+        # OverflowError for size larger than INT_MAX
+        size = INT_MAX + 10
+        with open(support.TESTFN, "rb") as fp:
+            data = os.read(fp.fileno(), size)
+
+        # The test does not try to read more than 2 GB at once because the
+        # operating system is free to return less bytes than requested.
+        self.assertEqual(data, b'test')
+
     def test_write(self):
         # os.write() accepts bytes- and buffer-like objects but not strings
         fd = os.open(support.TESTFN, os.O_CREAT | os.O_WRONLY)

Misc/NEWS

@@ -108,6 +108,11 @@ Core and Builtins
 Library
 -------
 
+- Issue #21932: os.read() now uses a :c:func:`Py_ssize_t` type instead of
+  :c:type:`int` for the size to support reading more than 2 GB at once. On
+  Windows, the size is truncted to INT_MAX. As any call to os.read(), the OS
+  may read less bytes than the number of requested bytes.
+
 - Issue #21942: Fixed source file viewing in pydoc's server mode on Windows.
 
 - Issue #11259: asynchat.async_chat().set_terminator() now raises a ValueError

Modules/posixmodule.c

@@ -7989,24 +7989,31 @@ Read a file descriptor.");
 static PyObject *
 posix_read(PyObject *self, PyObject *args)
 {
-    int fd, size;
+    int fd;
+    Py_ssize_t size;
     Py_ssize_t n;
     PyObject *buffer;
-    if (!PyArg_ParseTuple(args, "ii:read", &fd, &size))
+    if (!PyArg_ParseTuple(args, "in:read", &fd, &size))
         return NULL;
+    if (!_PyVerify_fd(fd))
+        return posix_error();
+#ifdef MS_WINDOWS
+    if (size > INT_MAX)
+        size = INT_MAX;
+#endif
     if (size < 0) {
         errno = EINVAL;
         return posix_error();
     }
     buffer = PyBytes_FromStringAndSize((char *)NULL, size);
     if (buffer == NULL)
         return NULL;
-    if (!_PyVerify_fd(fd)) {
-        Py_DECREF(buffer);
-        return posix_error();
-    }
     Py_BEGIN_ALLOW_THREADS
+#ifdef MS_WINDOWS
+    n = read(fd, PyBytes_AS_STRING(buffer), (int)size);
+#else
     n = read(fd, PyBytes_AS_STRING(buffer), size);
+#endif
     Py_END_ALLOW_THREADS
     if (n < 0) {
         Py_DECREF(buffer);