default to multi crawler commands

compatibility for old single crawler process
deprecated command's `crawler` property
logs about settings only shown once

Author: alexcepoi

scrapy/cmdline.py

@@ -5,7 +5,7 @@
 import pkg_resources
 
 import scrapy
-from scrapy.crawler import CrawlerProcess, MultiCrawlerProcess
+from scrapy.crawler import CrawlerProcess
 from scrapy.xlib import lsprofcalltree
 from scrapy.command import ScrapyCommand
 from scrapy.exceptions import UsageError
@@ -138,14 +138,7 @@ def execute(argv=None, settings=None):
     opts, args = parser.parse_args(args=argv[1:])
     _run_print_help(parser, cmd.process_options, args, opts)
 
-    if cmd.multi_crawlers:
-        process = MultiCrawlerProcess(settings)
-        cmd.process = process
-    else:
-        process = CrawlerProcess(settings)
-        process.install()
-        cmd.set_crawler(process)
-
+    cmd.crawler_process = CrawlerProcess(settings)
     _run_print_help(parser, _run_command, cmd, args, opts)
     sys.exit(cmd.exitcode)
 

scrapy/command.py

@@ -3,17 +3,17 @@
 """
 
 import os
+import warnings
 from optparse import OptionGroup
 from twisted.python import failure
 
-from scrapy import log
 from scrapy.utils.conf import arglist_to_dict
-from scrapy.exceptions import UsageError
+from scrapy.exceptions import UsageError, ScrapyDeprecationWarning
 
 class ScrapyCommand(object):
 
     requires_project = False
-    multi_crawlers = False
+    crawler_process = None
 
     # default settings to be used for this command instead of global defaults
     default_settings = {}
@@ -29,9 +29,24 @@ def set_crawler(self, crawler):
 
     @property
     def crawler(self):
-        if not self.multi_crawlers and not self._crawler.configured:
-            log.start_from_crawler(self._crawler)
-            self._crawler.configure()
+        warnings.warn("Command's default `crawler` is deprecated and will be removed. "
+            "Use `create_crawler` method to instatiate crawlers.",
+            ScrapyDeprecationWarning)
+
+        if not hasattr(self, '_crawler'):
+            crawler = self.crawler_process.create_crawler('default')
+
+            old_start = crawler.start
+            self.crawler_process.started = False
+            def wrapped_start():
+                if self.crawler_process.started:
+                    old_start()
+                else:
+                    self.crawler_process.started = True
+                    self.crawler_process.start()
+            crawler.start = wrapped_start
+
+            self.set_crawler(crawler)
 
         return self._crawler
 

scrapy/commands/check.py

@@ -20,7 +20,6 @@ def wrapper(response):
 
 class Command(ScrapyCommand):
     requires_project = True
-    multi_crawlers = True
     default_settings = {'LOG_ENABLED': False}
 
     def syntax(self):
@@ -59,7 +58,7 @@ def run(self, args, opts):
                 for req in requests:
                     contract_reqs[spider.name].append(req.callback.__name__)
             elif requests:
-                crawler = self.process.create_crawler(spider.name)
+                crawler = self.crawler_process.create_crawler(spider.name)
                 crawler.crawl(spider, requests)
 
         # start checks
@@ -69,7 +68,7 @@ def run(self, args, opts):
                 for method in sorted(methods):
                     print '  * %s' % method
         else:
-            self.process.start()
+            self.crawler_process.start()
             self.results.printErrors()
 
     def get_requests(self, spider):

scrapy/crawler.py

@@ -8,7 +8,6 @@
 from scrapy.signalmanager import SignalManager
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
-from scrapy.settings import overridden_settings
 from scrapy import log, signals
 
 
@@ -35,9 +34,8 @@ def uninstall(self):
     def configure(self):
         if self.configured:
             return
+
         self.configured = True
-        d = dict(overridden_settings(self.settings))
-        log.msg(format="Overridden settings: %(settings)r", settings=d, level=log.DEBUG)
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
         self.extensions = ExtensionManager.from_crawler(self)
@@ -84,11 +82,15 @@ def __init__(self, *a, **kw):
         install_shutdown_handlers(self._signal_shutdown)
 
     def start(self):
+        self.start_crawling()
         if self.settings.getbool('DNSCACHE_ENABLED'):
             reactor.installResolver(CachingThreadedResolver(reactor))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
+    def start_crawling(self):
+        raise NotImplementedError
+
     def stop(self):
         raise NotImplementedError
 
@@ -113,48 +115,34 @@ def _signal_kill(self, signum, _):
         reactor.callFromThread(self._stop_reactor)
 
 
-class CrawlerProcess(Crawler, ProcessMixin):
-    """ A class to run a single Scrapy crawler in a process
-    """
-
-    def __init__(self, *a, **kw):
-        Crawler.__init__(self, *a, **kw)
-        ProcessMixin.__init__(self, *a, **kw)
-        self.signals.connect(self.stop, signals.engine_stopped)
-
-    def start(self):
-        Crawler.start(self)
-        ProcessMixin.start(self)
-
-    def stop(self):
-        d = Crawler.stop(self)
-        d.addBoth(self._stop_reactor)
-        return d
-
-
-class MultiCrawlerProcess(ProcessMixin):
+class CrawlerProcess(ProcessMixin):
     """ A class to run multiple scrapy crawlers in a process sequentially
     """
 
     def __init__(self, settings):
-        super(MultiCrawlerProcess, self).__init__(settings)
+        super(CrawlerProcess, self).__init__(settings)
 
         self.settings = settings
         self.crawlers = {}
         self.stopping = False
 
     def create_crawler(self, name):
         if name not in self.crawlers:
-            self.crawlers[name] = Crawler(self.settings)
+            crawler = Crawler(self.settings)
+            crawler.configure()
+
+            self.crawlers[name] = crawler
 
         return self.crawlers[name]
 
-    def start_crawler(self):
+    def start_crawling(self):
         name, crawler = self.crawlers.popitem()
 
-        crawler.sflo = log.start_from_crawler(crawler)
-        if crawler.sflo:
-            crawler.signals.connect(crawler.sflo.stop, signals.engine_stopped)
+        sflo = log.start_from_crawler(crawler)
+        crawler.install()
+        crawler.signals.connect(crawler.uninstall, signals.engine_stopped)
+        if sflo:
+            crawler.signals.connect(sflo.stop, signals.engine_stopped)
 
         crawler.signals.connect(self.check_done, signals.engine_stopped)
         crawler.start()
@@ -163,13 +151,13 @@ def start_crawler(self):
 
     def check_done(self, **kwargs):
         if self.crawlers and not self.stopping:
-            self.start_crawler()
+            self.start_crawling()
         else:
             self._stop_reactor()
 
     def start(self):
-        self.start_crawler()
-        super(MultiCrawlerProcess, self).start()
+        log.scrapy_info(self.settings)
+        return super(CrawlerProcess, self).start()
 
     @defer.inlineCallbacks
     def stop(self):

scrapy/log.py

@@ -11,6 +11,7 @@
 
 import scrapy
 from scrapy.utils.python import unicode_to_str
+from scrapy.settings import overridden_settings
 
 # Logging levels
 DEBUG = logging.DEBUG
@@ -133,17 +134,24 @@ def err(_stuff=None, _why=None, **kw):
     kw.setdefault('system', 'scrapy')
     log.err(_stuff, _why, **kw)
 
-def start_from_crawler(crawler):
-    settings = crawler.settings
-    if not settings.getbool('LOG_ENABLED'):
-        return
+def start_from_settings(settings, crawler=None):
+    if settings.getbool('LOG_ENABLED'):
+        return start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'],
+            settings['LOG_ENCODING'], crawler)
 
-    sflo = start(settings['LOG_FILE'], settings['LOG_LEVEL'], settings['LOG_STDOUT'],
-        settings['LOG_ENCODING'], crawler)
+def scrapy_info(settings):
+    sflo = start_from_settings(settings)
+    if sflo:
+        msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, \
+            settings['BOT_NAME']))
 
-    msg("Scrapy %s started (bot: %s)" % (scrapy.__version__, \
-        settings['BOT_NAME']))
-    msg("Optional features available: %s" % ", ".join(scrapy.optional_features),
-        level=DEBUG)
+        msg("Optional features available: %s" % ", ".join(scrapy.optional_features),
+            level=DEBUG)
 
-    return sflo
+        d = dict(overridden_settings(settings))
+        msg(format="Overridden settings: %(settings)r", settings=d, level=DEBUG)
+
+        sflo.stop()
+
+def start_from_crawler(crawler, print_headers=False):
+    return start_from_settings(crawler.settings, crawler)