nextcall repetitive calls (heartbeats).

Author: sibiryakov

scrapy/core/engine.py

@@ -7,7 +7,7 @@
 import logging
 from time import time
 
-from twisted.internet import defer
+from twisted.internet import defer, task
 from twisted.python.failure import Failure
 
 from scrapy import signals
@@ -30,6 +30,7 @@ def __init__(self, start_requests, close_if_idle, nextcall, scheduler):
         self.close_if_idle = close_if_idle
         self.nextcall = nextcall
         self.scheduler = scheduler
+        self.heartbeat = task.LoopingCall(nextcall.schedule)
 
     def add_request(self, request):
         self.inprogress.add(request)
@@ -47,6 +48,7 @@ def _maybe_fire_closing(self):
         if self.closing and not self.inprogress:
             if self.nextcall:
                 self.nextcall.cancel()
+                self.heartbeat.stop()
             self.closing.callback(None)
 
 
@@ -98,7 +100,6 @@ def _next_request(self, spider):
             return
 
         if self.paused:
-            slot.nextcall.schedule(5)
             return
 
         while not self._needs_backout(spider):
@@ -239,6 +240,7 @@ def open_spider(self, spider, start_requests=(), close_if_idle=True):
         self.crawler.stats.open_spider(spider)
         yield self.signals.send_catch_log_deferred(signals.spider_opened, spider=spider)
         slot.nextcall.schedule()
+        slot.heartbeat.start(5)
 
     def _spider_idle(self, spider):
         """Called when a spider gets idle. This function is called when there
@@ -252,7 +254,6 @@ def _spider_idle(self, spider):
             spider=spider, dont_log=DontCloseSpider)
         if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
                 for _, x in res):
-            self.slot.nextcall.schedule(5)
             return
 
         if self.spider_is_idle(spider):