Skip to content

Commit 8efd6d2

Browse files
pawelmhmredapple
pawelmhm
authored and
redapple
committed
[Backport][1.0] response_status_message should not fail on non-standard HTTP codes
utility is used in retry middleware and it was failing to handle non-standard HTTP codes. Instead of raising exceptions when passing through to_native_str it should return "Unknown status" message.
1 parent 38b0908 commit 8efd6d2

File tree

2 files changed

+21
-2
lines changed

2 files changed

+21
-2
lines changed

‎scrapy/utils/response.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ def response_status_message(status):
5555
"""
5656
# Implicit decode/encode is on purpose to force native strings
5757
# This is properly fixed in Scrapy >=1.1 at revision faf9265
58-
reason = http.RESPONSES.get(int(status)).decode('utf8', errors='replace')
58+
reason = http.RESPONSES.get(int(status), "Unknown Status").decode('utf8', errors='replace')
5959
return '{} {}'.format(status, reason)
6060

6161
def response_httprepr(response):

‎tests/test_utils_response.py

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
from six.moves.urllib.parse import urlparse
44

55
from scrapy.http import Response, TextResponse, HtmlResponse
6-
from scrapy.utils.response import response_httprepr, open_in_browser, get_meta_refresh
6+
from scrapy.utils.response import (response_httprepr, open_in_browser,
7+
get_meta_refresh, get_base_url, response_status_message)
78

89
__doctests__ = ['scrapy.utils.response']
910

@@ -61,5 +62,23 @@ def test_get_meta_refresh(self):
6162
self.assertEqual(get_meta_refresh(r2), (None, None))
6263
self.assertEqual(get_meta_refresh(r3), (None, None))
6364

65+
def test_get_base_url(self):
66+
resp = HtmlResponse("http://www.example.com", body=b"""
67+
<html>
68+
<head><base href="http://www.example.com/img/" target="_blank"></head>
69+
<body>blahablsdfsal&amp;</body>
70+
</html>""")
71+
self.assertEqual(get_base_url(resp), "http://www.example.com/img/")
72+
73+
resp2 = HtmlResponse("http://www.example.com", body=b"""
74+
<html><body>blahablsdfsal&amp;</body></html>""")
75+
self.assertEqual(get_base_url(resp2), "http://www.example.com")
76+
77+
def test_response_status_message(self):
78+
self.assertEqual(response_status_message(200), '200 OK')
79+
self.assertEqual(response_status_message(404), '404 Not Found')
80+
self.assertEqual(response_status_message(573), "573 Unknown Status")
81+
82+
6483
if __name__ == "__main__":
6584
unittest.main()

0 commit comments

Comments
 (0)