update http.client

Author: gneverov

python-stdlib/http/client.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: 2023 Python Software Foundation
 # SPDX-License-Identifier: Python-2.0
 
+# https://github.com/python/cpython/blob/31c9f3ced293492b38e784c17c4befe425da5dab/Lib/http/client.py
+
 r"""HTTP/1.1 client library
 
 <intro stuff goes here>
@@ -154,6 +156,13 @@ def _encode(data, name='data'):
     """Call data.encode("latin-1") but show a better error message."""
     return data.encode()
 
+def _strip_ipv6_iface(enc_name: bytes) -> bytes:
+    """Remove interface scope from IPv6 address."""
+    enc_name, percent, _ = enc_name.partition(b"%")
+    if percent:
+        assert enc_name.startswith(b'['), enc_name
+        enc_name += b']'
+    return enc_name
 
 def _read_headers(fp):
     """Reads potential header lines into a list from a file pointer.
@@ -210,7 +219,7 @@ def __init__(self, sock, debuglevel=0, method=None, url=None):
         # happen if a self.fp.read() is done (without a size) whether
         # self.fp is buffered or not.  So, no self.fp.read() by
         # clients unless they know what they are doing.
-        self.fp = sock
+        self.fp = sock.makefile("rb")
         self.debuglevel = debuglevel
         self._method = method
 
@@ -373,8 +382,7 @@ def _close_conn(self):
 
     def close(self):
         try:
-            # super().close() # set "closed" flag
-            pass
+            super().close() # set "closed" flag
         finally:
             if self.fp:
                 self._close_conn()
@@ -618,17 +626,10 @@ def read1(self, n=-1):
             self._close_conn()
         elif self.length is not None:
             self.length -= len(result)
+            if not self.length:
+                self._close_conn()
         return result
 
-    def peek(self, n=-1):
-        # Having this enables IOBase.readline() to read more than one
-        # byte at a time
-        if self.fp is None or self._method == "HEAD":
-            return b""
-        if self.chunked:
-            return self._peek_chunked(n)
-        return self.fp.peek(n)
-
     def readline(self, limit=-1):
         if self.fp is None or self._method == "HEAD":
             return b""
@@ -642,6 +643,8 @@ def readline(self, limit=-1):
             self._close_conn()
         elif self.length is not None:
             self.length -= len(result)
+            if not self.length:
+                self._close_conn()
         return result
 
     def _read1_chunked(self, n):
@@ -658,19 +661,6 @@ def _read1_chunked(self, n):
             raise IncompleteRead(b"")
         return read
 
-    def _peek_chunked(self, n):
-        # Strictly speaking, _get_chunk_left() may cause more than one read,
-        # but that is ok, since that is to satisfy the chunked protocol.
-        try:
-            chunk_left = self._get_chunk_left()
-        except IncompleteRead:
-            return b'' # peek doesn't worry about protocol
-        if chunk_left is None:
-            return b'' # eof
-        # peek is allowed to return more than requested.  Just request the
-        # entire chunk, and truncate what we get.
-        return self.fp.peek(chunk_left)[:chunk_left]
-
     def fileno(self):
         return self.fp.fileno()
 
@@ -810,6 +800,7 @@ def __init__(self, host, port=None, timeout=-1,
         self._tunnel_host = None
         self._tunnel_port = None
         self._tunnel_headers = {}
+        self._raw_proxy_headers = None
 
         (self.host, self.port) = self._get_hostport(host, port)
 
@@ -822,27 +813,39 @@ def __init__(self, host, port=None, timeout=-1,
     def set_tunnel(self, host, port=None, headers=None):
         """Set up host and port for HTTP CONNECT tunnelling.
 
-        In a connection that uses HTTP CONNECT tunneling, the host passed to the
-        constructor is used as a proxy server that relays all communication to
-        the endpoint passed to `set_tunnel`. This done by sending an HTTP
+        In a connection that uses HTTP CONNECT tunnelling, the host passed to
+        the constructor is used as a proxy server that relays all communication
+        to the endpoint passed to `set_tunnel`. This done by sending an HTTP
         CONNECT request to the proxy server when the connection is established.
 
         This method must be called before the HTTP connection has been
         established.
 
         The headers argument should be a mapping of extra HTTP headers to send
         with the CONNECT request.
+
+        As HTTP/1.1 is used for HTTP CONNECT tunnelling request, as per the RFC
+        (https://tools.ietf.org/html/rfc7231#section-4.3.6), a HTTP Host:
+        header must be provided, matching the authority-form of the request
+        target provided as the destination for the CONNECT request. If a
+        HTTP Host: header is not provided via the headers argument, one
+        is generated and transmitted automatically.
         """
 
         if self.sock:
             raise RuntimeError("Can't set up tunnel for established connection")
 
         self._tunnel_host, self._tunnel_port = self._get_hostport(host, port)
         if headers:
-            self._tunnel_headers = headers
+            self._tunnel_headers = headers.copy()
         else:
             self._tunnel_headers.clear()
 
+        if not any(header.lower() == "host" for header in self._tunnel_headers):
+            encoded_host = self._tunnel_host.encode("idna").decode("ascii")
+            self._tunnel_headers["Host"] = "%s:%d" % (
+                encoded_host, self._tunnel_port)
+
     def _get_hostport(self, host, port):
         if port is None:
             i = host.rfind(':')
@@ -858,17 +861,24 @@ def _get_hostport(self, host, port):
                 host = host[:i]
             else:
                 port = self.default_port
-            if host and host[0] == '[' and host[-1] == ']':
-                host = host[1:-1]
+        if host and host[0] == '[' and host[-1] == ']':
+            host = host[1:-1]
 
         return (host, port)
 
     def set_debuglevel(self, level):
         self.debuglevel = level
 
+    def _wrap_ipv6(self, ip):
+        if b':' in ip and ip[0] != b'['[0]:
+            return b"[" + ip + b"]"
+        return ip
+
     def _tunnel(self):
-        connect = b"CONNECT %s:%d HTTP/1.0\r\n" % (
-            self._tunnel_host.encode("ascii"), self._tunnel_port)
+        connect = b"CONNECT %s:%d %s\r\n" % (
+            self._wrap_ipv6(self._tunnel_host.encode("idna")),
+            self._tunnel_port,
+            self._http_vsn_str.encode("ascii"))
         headers = [connect]
         for header, value in self._tunnel_headers.items():
             headers.append(f"{header}: {value}\r\n".encode("latin-1"))
@@ -883,24 +893,33 @@ def _tunnel(self):
         try:
             (version, code, message) = response._read_status()
 
+            self._raw_proxy_headers = _read_headers(response.fp)
+
+            if self.debuglevel > 0:
+                for header in self._raw_proxy_headers:
+                    print('header:', header.decode())
+
             if code != http.HTTPStatus.OK:
                 self.close()
                 raise OSError(f"Tunnel connection failed: {code} {message.strip()}")
-            while True:
-                line = response.fp.readline(_MAXLINE + 1)
-                if len(line) > _MAXLINE:
-                    raise LineTooLong("header line")
-                if not line:
-                    # for sites which EOF without sending a trailer
-                    break
-                if line in (b'\r\n', b'\n', b''):
-                    break
 
-                if self.debuglevel > 0:
-                    print('header:', line.decode())
         finally:
             response.close()
 
+    def get_proxy_response_headers(self):
+        """
+        Returns a dictionary with the headers of the response
+        received from the proxy server to the CONNECT request
+        sent to set the tunnel.
+
+        If the CONNECT request was not sent, the method returns None.
+        """
+        return (
+            _parse_header_lines(self._raw_proxy_headers)
+            if self._raw_proxy_headers is not None
+            else None
+        )
+
     def connect(self):
         """Connect to the host and port specified in __init__."""
         self.sock = self._create_connection(
@@ -931,7 +950,7 @@ def close(self):
                 response.close()
 
     def send(self, data):
-        """Send `data' to the server.
+        """Send 'data' to the server.
         ``data`` can be a string object, a bytes object, an array object, a
         file-like object that supports a .read() method, or an iterable object.
         """
@@ -956,14 +975,14 @@ def send(self, data):
                     break
                 if encode:
                     datablock = datablock.encode("iso-8859-1")
-                self.sock.write(datablock)
+                self.sock.sendall(datablock)
             return
         try:
-            self.sock.write(data)
+            self.sock.sendall(data)
         except TypeError:
             try:
                 for d in data:
-                    self.sock.write(d)
+                    self.sock.sendall(d)
             except TypeError:
                 raise TypeError("data should be a bytes-like object "
                                 "or an iterable, got %r" % type(data))
@@ -1047,10 +1066,10 @@ def putrequest(self, method, url, skip_host=False,
                    skip_accept_encoding=False):
         """Send a request to the server.
 
-        `method' specifies an HTTP request method, e.g. 'GET'.
-        `url' specifies the object being requested, e.g. '/index.html'.
-        `skip_host' if True does not add automatically a 'Host:' header
-        `skip_accept_encoding' if True does not add automatically an
+        'method' specifies an HTTP request method, e.g. 'GET'.
+        'url' specifies the object being requested, e.g. '/index.html'.
+        'skip_host' if True does not add automatically a 'Host:' header
+        'skip_accept_encoding' if True does not add automatically an
            'Accept-Encoding:' header
         """
 
@@ -1121,7 +1140,7 @@ def putrequest(self, method, url, skip_host=False,
                         netloc_enc = netloc.encode("ascii")
                     except UnicodeEncodeError:
                         netloc_enc = netloc.encode("idna")
-                    self.putheader('Host', netloc_enc)
+                    self.putheader('Host', _strip_ipv6_iface(netloc_enc))
                 else:
                     if self._tunnel_host:
                         host = self._tunnel_host
@@ -1137,9 +1156,9 @@ def putrequest(self, method, url, skip_host=False,
 
                     # As per RFC 273, IPv6 address should be wrapped with []
                     # when used as Host header
-
-                    if host.find(':') >= 0:
-                        host_enc = b'[' + host_enc + b']'
+                    host_enc = self._wrap_ipv6(host_enc)
+                    if ":" in host:
+                        host_enc = _strip_ipv6_iface(host_enc)
 
                     if port == self.default_port:
                         self.putheader('Host', host_enc)
@@ -1341,8 +1360,7 @@ def getresponse(self):
 
             if response.will_close:
                 # this effectively passes the connection to the response
-                # self.close()
-                pass
+                self.close()
             else:
                 # remember this, so we can tell when it is complete
                 self.__response = response
@@ -1364,44 +1382,15 @@ class HTTPSConnection(HTTPConnection):
 
         # XXX Should key_file and cert_file be deprecated in favour of context?
 
-        def __init__(self, host, port=None, key_file=None, cert_file=None,
-                     timeout=-1,
-                     source_address=None, *, context=None,
-                     check_hostname=None, blocksize=8192):
+        def __init__(self, host, port=None,
+                     *, timeout=-1,
+                     source_address=None, context=None, blocksize=8192):
             super(HTTPSConnection, self).__init__(host, port, timeout,
                                                   source_address,
                                                   blocksize=blocksize)
-            if (key_file is not None or cert_file is not None or
-                        check_hostname is not None):
-                import warnings
-                warnings.warn("key_file, cert_file and check_hostname are "
-                              "deprecated, use a custom context instead.",
-                              DeprecationWarning, 2)
-            self.key_file = key_file
-            self.cert_file = cert_file
             if context is None:
-                context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
-                # send ALPN extension to indicate HTTP/1.1 protocol
-                # if self._http_vsn == 11:
-                #     context.set_alpn_protocols(['http/1.1'])
-                # enable PHA for TLS 1.3 connections if available
-                # if context.post_handshake_auth is not None:
-                #     context.post_handshake_auth = True
-            will_verify = context.verify_mode != ssl.CERT_NONE
-            if check_hostname is None:
-                check_hostname = False
-            if check_hostname and not will_verify:
-                raise ValueError("check_hostname needs a SSL context with "
-                                 "either CERT_OPTIONAL or CERT_REQUIRED")
-            if key_file or cert_file:
-                context.load_cert_chain(cert_file, key_file)
-                # cert and key file means the user wants to authenticate.
-                # enable TLS 1.3 PHA implicitly even for custom contexts.
-                if context.post_handshake_auth is not None:
-                    context.post_handshake_auth = True
+                context = _create_https_context(self._http_vsn)
             self._context = context
-            # if check_hostname is not None:
-            #     self._context.check_hostname = check_hostname
 
         def connect(self):
             "Connect to a host on a given (SSL) port."