diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index c4159508e4..488562ddd2 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -32,3 +32,23 @@ jobs: - name: Run tests run: | make ci + + no_chardet: + name: "No Character Detection" + runs-on: ubuntu-latest + strategy: + fail-fast: true + + steps: + - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 + - name: 'Set up Python 3.8' + uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d + with: + python-version: '3.8' + - name: Install dependencies + run: | + make + python -m pip uninstall -y "charset_normalizer" "chardet" + - name: Run tests + run: | + make ci diff --git a/HISTORY.md b/HISTORY.md index bbe6dd425b..6676a82e30 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -6,6 +6,22 @@ dev - \[Short description of non-trivial change.\] + +2.31.1 (2024-05-20) +------------------- + +**Dependencies** + +- `charset_normalizer` and `chardet` are now optional dependencies of Requests. + The default behavior will degrade to using `utf-8` when decoding strings in + the `Response.text()` API if `Response.encoding` is not set. + + This will not affect the default installation of Requests but enables + downstream repackaging to remove these dependencies if they're undesired. + Default installations of Requests will still use `charset_normalizer` unless + `chardet` is present. + + 2.31.0 (2023-05-22) ------------------- diff --git a/requests/__init__.py b/requests/__init__.py index 300a16c574..051cda1340 100644 --- a/requests/__init__.py +++ b/requests/__init__.py @@ -83,7 +83,11 @@ def check_compatibility(urllib3_version, chardet_version, charset_normalizer_ver # charset_normalizer >= 2.0.0 < 4.0.0 assert (2, 0, 0) <= (major, minor, patch) < (4, 0, 0) else: - raise Exception("You need either charset_normalizer or chardet installed") + warnings.warn( + "Unable to find acceptable character detection dependency " + "(chardet or charset_normalizer).", + RequestsDependencyWarning, + ) def _check_cryptography(cryptography_version): diff --git a/requests/__version__.py b/requests/__version__.py index 5063c3f8ee..5fe711d46e 100644 --- a/requests/__version__.py +++ b/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = "requests" __description__ = "Python HTTP for Humans." __url__ = "https://requests.readthedocs.io/" -__version__ = "2.31.0" -__build__ = 0x023100 +__version__ = "2.31.1" +__build__ = 0x023101 __author__ = "Kenneth Reitz" __author_email__ = "me@kennethreitz.org" __license__ = "Apache 2.0" diff --git a/requests/compat.py b/requests/compat.py index 6776163c94..095de1b6ca 100644 --- a/requests/compat.py +++ b/requests/compat.py @@ -7,13 +7,28 @@ compatibility until the next major version. """ -try: - import chardet -except ImportError: - import charset_normalizer as chardet - +import importlib import sys +# ------------------- +# Character Detection +# ------------------- + + +def _resolve_char_detection(): + """Find supported character detection libraries.""" + chardet = None + for lib in ("chardet", "charset_normalizer"): + if chardet is None: + try: + chardet = importlib.import_module(lib) + except ImportError: + pass + return chardet + + +chardet = _resolve_char_detection() + # ------- # Pythons # ------- diff --git a/requests/models.py b/requests/models.py index 617a4134e5..b92320232d 100644 --- a/requests/models.py +++ b/requests/models.py @@ -790,7 +790,12 @@ def next(self): @property def apparent_encoding(self): """The apparent encoding, provided by the charset_normalizer or chardet libraries.""" - return chardet.detect(self.content)["encoding"] + if chardet is not None: + return chardet.detect(self.content)["encoding"] + else: + # If no character detection library is available, we'll fall back + # to a standard Python utf-8 str. + return "utf-8" def iter_content(self, chunk_size=1, decode_unicode=False): """Iterates over the response data. When stream=True is set on the diff --git a/requests/packages.py b/requests/packages.py index 77c45c9e90..5ab3d8e250 100644 --- a/requests/packages.py +++ b/requests/packages.py @@ -1,13 +1,6 @@ import sys -try: - import chardet -except ImportError: - import warnings - - import charset_normalizer as chardet - - warnings.filterwarnings("ignore", "Trying to detect", module="charset_normalizer") +from .compat import chardet # This code exists for backwards compatibility reasons. # I don't like it either. Just look the other way. :) @@ -20,9 +13,11 @@ if mod == package or mod.startswith(f"{package}."): sys.modules[f"requests.packages.{mod}"] = sys.modules[mod] -target = chardet.__name__ -for mod in list(sys.modules): - if mod == target or mod.startswith(f"{target}."): - target = target.replace(target, "chardet") - sys.modules[f"requests.packages.{target}"] = sys.modules[mod] -# Kinda cool, though, right? +if chardet is not None: + target = chardet.__name__ + for mod in list(sys.modules): + if mod == target or mod.startswith(f"{target}."): + imported_mod = sys.modules[mod] + sys.modules[f"requests.packages.{mod}"] = imported_mod + mod = mod.replace(target, "chardet") + sys.modules[f"requests.packages.{mod}"] = imported_mod diff --git a/requests/sessions.py b/requests/sessions.py index dbcf2a7b0e..df35bfe2cd 100644 --- a/requests/sessions.py +++ b/requests/sessions.py @@ -326,7 +326,7 @@ def rebuild_proxies(self, prepared_request, proxies): # urllib3 handles proxy authorization for us in the standard adapter. # Avoid appending this to TLS tunneled requests where it may be leaked. - if not scheme.startswith('https') and username and password: + if not scheme.startswith("https") and username and password: headers["Proxy-Authorization"] = _basic_auth_str(username, password) return new_proxies diff --git a/setup.cfg b/setup.cfg index bf21c81cc0..7001049b9c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,7 @@ requires-dist = urllib3>=1.21.1,<1.27 [flake8] -ignore = E203, E501, W503 +ignore = E203, E231, E501, W503 per-file-ignores = requests/__init__.py:E402, F401 requests/compat.py:E402, F401 diff --git a/tests/test_requests.py b/tests/test_requests.py index b420c44d73..5a01f5fbb2 100644 --- a/tests/test_requests.py +++ b/tests/test_requests.py @@ -647,25 +647,26 @@ def test_proxy_authorization_preserved_on_request(self, httpbin): assert sent_headers.get("Proxy-Authorization") == proxy_auth_value - @pytest.mark.parametrize( "url,has_proxy_auth", ( - ('http://example.com/', True), - ('https://example.com/', False), + ("http://example.com/", True), + ("https://example.com/", False), ), ) - def test_proxy_authorization_not_appended_to_https_request(self, url, has_proxy_auth): + def test_proxy_authorization_not_appended_to_https_request( + self, url, has_proxy_auth + ): session = requests.Session() proxies = { - 'http': 'http://test:pass@localhost:8080/', - 'https': 'http://test:pass@localhost:8090/', + "http": "http://test:pass@localhost:8080/", + "https": "http://test:pass@localhost:8090/", } - req = requests.Request('GET', url) + req = requests.Request("GET", url) prep = req.prepare() session.rebuild_proxies(prep, proxies) - assert ('Proxy-Authorization' in prep.headers) is has_proxy_auth + assert ("Proxy-Authorization" in prep.headers) is has_proxy_auth def test_basicauth_with_netrc(self, httpbin): auth = ("user", "pass")