PY3 fix ChunkedTransferMiddleware

Author: kmike

scrapy/downloadermiddlewares/chunked.py

@@ -7,7 +7,7 @@ class ChunkedTransferMiddleware(object):
     """
 
     def process_response(self, request, response, spider):
-        if response.headers.get('Transfer-Encoding') == 'chunked':
+        if response.headers.get(b'Transfer-Encoding') == b'chunked':
             body = decode_chunked_transfer(response.body)
             return response.replace(body=body)
         return response

scrapy/utils/http.py

@@ -14,10 +14,10 @@ def decode_chunked_transfer(chunked_body):
     http://en.wikipedia.org/wiki/Chunked_transfer_encoding
 
     """
-    body, h, t = '', '', chunked_body
+    body, h, t = b'', b'', chunked_body
     while t:
-        h, t = t.split('\r\n', 1)
-        if h == '0':
+        h, t = t.split(b'\r\n', 1)
+        if h == b'0':
             break
         size = int(h, 16)
         body += t[:size]

tests/test_utils_http.py

@@ -6,15 +6,15 @@ class ChunkedTest(unittest.TestCase):
 
     def test_decode_chunked_transfer(self):
         """Example taken from: http://en.wikipedia.org/wiki/Chunked_transfer_encoding"""
-        chunked_body = "25\r\n" + "This is the data in the first chunk\r\n\r\n"
-        chunked_body += "1C\r\n" + "and this is the second one\r\n\r\n"
-        chunked_body += "3\r\n" + "con\r\n"
-        chunked_body += "8\r\n" + "sequence\r\n"
-        chunked_body += "0\r\n\r\n"
+        chunked_body = b"25\r\n" + b"This is the data in the first chunk\r\n\r\n"
+        chunked_body += b"1C\r\n" + b"and this is the second one\r\n\r\n"
+        chunked_body += b"3\r\n" + b"con\r\n"
+        chunked_body += b"8\r\n" + b"sequence\r\n"
+        chunked_body += b"0\r\n\r\n"
         body = decode_chunked_transfer(chunked_body)
-        self.assertEqual(body, \
-            "This is the data in the first chunk\r\n" +
-            "and this is the second one\r\n" +
-            "consequence")
+        self.assertEqual(body,
+            b"This is the data in the first chunk\r\n"
+            b"and this is the second one\r\n"
+            b"consequence")