Merge pull request scrapy#1804 from redapple/enable-test-dwnld-timeout

Re-enable HTTPS tests for download timeouts

Author: dangra

scrapy/core/downloader/handlers/http11.py

@@ -209,6 +209,7 @@ def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, poo
         self._pool = pool
         self._maxsize = maxsize
         self._warnsize = warnsize
+        self._txresponse = None
 
     def _get_agent(self, request, timeout):
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
@@ -275,6 +276,11 @@ def _cb_timeout(self, result, request, url, timeout):
         if self._timeout_cl.active():
             self._timeout_cl.cancel()
             return result
+        # needed for HTTPS requests, otherwise _ResponseReader doesn't
+        # receive connectionLost()
+        if self._txresponse:
+            self._txresponse._transport.stopProducing()
+
         raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout))
 
     def _cb_latency(self, result, request, start_time):
@@ -310,6 +316,10 @@ def _cancel(_):
 
         d = defer.Deferred(_cancel)
         txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize))
+
+        # save response for timeouts
+        self._txresponse = txresponse
+
         return d
 
     def _cb_bodydone(self, result, request, url):

scrapy/core/downloader/webclient.py

@@ -83,6 +83,11 @@ def handleResponse(self, response):
 
     def timeout(self):
         self.transport.loseConnection()
+
+        # transport cleanup needed for HTTPS connections
+        if self.factory.url.startswith(b'https'):
+            self.transport.stopProducing()
+
         self.factory.noPage(\
                 defer.TimeoutError("Getting %s took longer than %s seconds." % \
                 (self.factory.url, self.factory.timeout)))

tests/test_downloader_handlers.py

@@ -182,17 +182,19 @@ def test_redirect_status_head(self):
         return d
 
     @defer.inlineCallbacks
-    def test_timeout_download_from_spider(self):
-        if self.scheme == 'https':
-            raise unittest.SkipTest(
-                'test_timeout_download_from_spider skipped under https')
+    def test_timeout_download_from_spider_nodata_rcvd(self):
+        # client connects but no data is received
         spider = Spider('foo')
         meta = {'download_timeout': 0.2}
-        # client connects but no data is received
         request = Request(self.getURL('wait'), meta=meta)
         d = self.download_request(request, spider)
         yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
+
+    @defer.inlineCallbacks
+    def test_timeout_download_from_spider_server_hangs(self):
         # client connects, server send headers and some body bytes but hangs
+        spider = Spider('foo')
+        meta = {'download_timeout': 0.2}
         request = Request(self.getURL('hang-after-headers'), meta=meta)
         d = self.download_request(request, spider)
         yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)