Explicitly call Twisted transport stopProducing() on HTTP/1.1 timeouts

Author: redapple

scrapy/core/downloader/handlers/http11.py

@@ -193,6 +193,7 @@ def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, poo
         self._pool = pool
         self._maxsize = maxsize
         self._warnsize = warnsize
+        self._txresponse = None
 
     def _get_agent(self, request, timeout):
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
@@ -259,6 +260,11 @@ def _cb_timeout(self, result, request, url, timeout):
         if self._timeout_cl.active():
             self._timeout_cl.cancel()
             return result
+        # needed for HTTPS requests, otherwise _ResponseReader doesn't
+        # receive connectionLost()
+        if self._txresponse:
+            self._txresponse._transport.stopProducing()
+
         raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout))
 
     def _cb_latency(self, result, request, start_time):
@@ -294,6 +300,10 @@ def _cancel(_):
 
         d = defer.Deferred(_cancel)
         txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize))
+
+        # save response for timeouts
+        self._txresponse = txresponse
+
         return d
 
     def _cb_bodydone(self, result, request, url):

tests/test_downloader_handlers.py

@@ -182,17 +182,19 @@ def test_redirect_status_head(self):
         return d
 
     @defer.inlineCallbacks
-    def test_timeout_download_from_spider(self):
-        if self.scheme == 'https':
-            raise unittest.SkipTest(
-                'test_timeout_download_from_spider skipped under https')
+    def test_timeout_download_from_spider_nodata_rcvd(self):
+        # client connects but no data is received
         spider = Spider('foo')
         meta = {'download_timeout': 0.2}
-        # client connects but no data is received
         request = Request(self.getURL('wait'), meta=meta)
         d = self.download_request(request, spider)
         yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
+
+    @defer.inlineCallbacks
+    def test_timeout_download_from_spider_server_hangs(self):
         # client connects, server send headers and some body bytes but hangs
+        spider = Spider('foo')
+        meta = {'download_timeout': 0.2}
         request = Request(self.getURL('hang-after-headers'), meta=meta)
         d = self.download_request(request, spider)
         yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)