Merge pull request scrapy#1800 from redapple/http11-post-content-length

[MRG+1] Add "Content-Length: 0" for body-less HTTP/1.1 POST requests

Author: dangra

scrapy/core/downloader/handlers/http11.py

@@ -211,7 +211,21 @@ def download_request(self, request):
         headers = TxHeaders(request.headers)
         if isinstance(agent, self._TunnelingAgent):
             headers.removeHeader(b'Proxy-Authorization')
-        bodyproducer = _RequestBodyProducer(request.body) if request.body else None
+        if request.body:
+            bodyproducer = _RequestBodyProducer(request.body)
+        else:
+            bodyproducer = None
+            # Setting Content-Length: 0 even for POST requests is not a
+            # MUST per HTTP RFCs, but it's common behavior, and some
+            # servers require this, otherwise returning HTTP 411 Length required
+            #
+            # RFC 7230#section-3.3.2:
+            # "a Content-Length header field is normally sent in a POST
+            # request even when the value is 0 (indicating an empty payload body)."
+            #
+            # Twisted Agent will not add "Content-Length: 0" by itself
+            if method == b'POST':
+                headers.addRawHeader(b'Content-Length', b'0')
 
         start_time = time()
         d = agent.request(

tests/test_downloader_handlers.py

@@ -105,6 +105,15 @@ def test_non_existent(self):
         return self.assertFailure(d, IOError)
 
 
+class ContentLengthHeaderResource(resource.Resource):
+    """
+    A testing resource which renders itself as the value of the Content-Length
+    header from the request.
+    """
+    def render(self, request):
+        return request.requestHeaders.getRawHeaders(b"content-length")[0]
+
+
 class HttpTestCase(unittest.TestCase):
 
     scheme = 'http'
@@ -122,6 +131,7 @@ def setUp(self):
         r.putChild(b"host", HostHeaderResource())
         r.putChild(b"payload", PayloadResource())
         r.putChild(b"broken", BrokenDownloadResource())
+        r.putChild(b"contentlength", ContentLengthHeaderResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
         self.host = 'localhost'
@@ -209,6 +219,28 @@ def _test(response):
         d.addCallback(self.assertEquals, b'example.com')
         return d
 
+    def test_content_length_zero_bodyless_post_request_headers(self):
+        """Tests if "Content-Length: 0" is sent for bodyless POST requests.
+
+        This is not strictly required by HTTP RFCs but can cause trouble
+        for some web servers.
+        See:
+        https://github.com/scrapy/scrapy/issues/823
+        https://issues.apache.org/jira/browse/TS-2902
+        https://github.com/kennethreitz/requests/issues/405
+        https://bugs.python.org/issue14721
+        """
+        def _test(response):
+            self.assertEquals(response.body, b'0')
+
+        request = Request(self.getURL('contentlength'), method='POST', headers={'Host': 'example.com'})
+        return self.download_request(request, Spider('foo')).addCallback(_test)
+
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEquals, b'0')
+        return d
+
     def test_payload(self):
         body = b'1'*100 # PayloadResource requires body length to be 100
         request = Request(self.getURL('payload'), method='POST', body=body)