Merge pull request scrapy#1419 from scrapy/httprepr-on-file-scheme

Do not fail representing non-http requests

Author: kmike

scrapy/utils/request.py

@@ -79,7 +79,7 @@ def request_httprepr(request):
     parsed = urlparse_cached(request)
     path = urlunparse(('', '', parsed.path or '/', parsed.params, parsed.query, ''))
     s = to_bytes(request.method) + b" " + to_bytes(path) + b" HTTP/1.1\r\n"
-    s += b"Host: " + to_bytes(parsed.hostname) + b"\r\n"
+    s += b"Host: " + to_bytes(parsed.hostname or b'') + b"\r\n"
     if request.headers:
         s += request.headers.to_string() + b"\r\n"
     s += b"\r\n"

tests/test_utils_request.py

@@ -71,5 +71,10 @@ def test_request_httprepr(self):
         r1 = Request("http://www.example.com", method='POST', headers={"Content-type": b"text/html"}, body=b"Some body")
         self.assertEqual(request_httprepr(r1), b'POST / HTTP/1.1\r\nHost: www.example.com\r\nContent-Type: text/html\r\n\r\nSome body')
 
+    def test_request_httprepr_for_non_http_request(self):
+        # the representation is not important but it must not fail.
+        request_httprepr(Request("file:///tmp/foo.txt"))
+        request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
+
 if __name__ == "__main__":
     unittest.main()