fix log intermixing in scrapy shell command

Author: alexcepoi

scrapy/commands/shell.py

@@ -43,12 +43,13 @@ def run(self, args, opts):
         url = args[0] if args else None
         spider = crawler.spiders.create(opts.spider) if opts.spider else None
 
-        shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
+        self.crawler_process.start_crawling()
         self._start_crawler_thread()
+
+        shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
         shell.start(url=url, spider=spider)
 
     def _start_crawler_thread(self):
-        self.crawler_process.print_headers()
-        t = Thread(target=self.crawler_process.start, kwargs={'headers': False})
+        t = Thread(target=self.crawler_process.start_reactor)
         t.daemon = True
         t.start()

scrapy/crawler.py

@@ -91,6 +91,9 @@ def __init__(self, *a, **kw):
 
     def start(self):
         self.start_crawling()
+        self.start_reactor()
+
+    def start_reactor(self):
         if self.settings.getbool('DNSCACHE_ENABLED'):
             reactor.installResolver(CachingThreadedResolver(reactor))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
@@ -102,7 +105,7 @@ def start_crawling(self):
     def stop(self):
         raise NotImplementedError
 
-    def _stop_reactor(self, _=None):
+    def stop_reactor(self, _=None):
         try:
             reactor.stop()
         except RuntimeError:  # raised if already stopped or in shutdown stage
@@ -120,7 +123,7 @@ def _signal_kill(self, signum, _):
         signame = signal_names[signum]
         log.msg(format='Received %(signame)s twice, forcing unclean shutdown',
                 level=log.INFO, signame=signame)
-        reactor.callFromThread(self._stop_reactor)
+        reactor.callFromThread(self.stop_reactor)
 
 
 class CrawlerProcess(ProcessMixin):
@@ -140,7 +143,7 @@ def create_crawler(self, name=None):
 
         return self.crawlers[name]
 
-    def start_crawling(self):
+    def start_crawler(self):
         name, crawler = self.crawlers.popitem()
 
         sflo = log.start_from_crawler(crawler)
@@ -157,17 +160,13 @@ def start_crawling(self):
 
     def check_done(self, **kwargs):
         if self.crawlers and not self.stopping:
-            self.start_crawling()
+            self.start_crawler()
         else:
-            self._stop_reactor()
+            self.stop_reactor()
 
-    def print_headers(self):
+    def start_crawling(self):
         log.scrapy_info(self.settings)
-
-    def start(self, headers=True):
-        if headers:
-            self.print_headers()
-        return super(CrawlerProcess, self).start()
+        self.start_crawler()
 
     @defer.inlineCallbacks
     def stop(self):