port all scrapy commands to new CrawlerProcess

Author: alexcepoi

scrapy/command.py

@@ -34,7 +34,7 @@ def crawler(self):
             ScrapyDeprecationWarning)
 
         if not hasattr(self, '_crawler'):
-            crawler = self.crawler_process.create_crawler('default')
+            crawler = self.crawler_process.create_crawler()
 
             old_start = crawler.start
             self.crawler_process.started = False

scrapy/commands/bench.py

@@ -16,5 +16,6 @@ def short_desc(self):
     def run(self, args, opts):
         with MockServer():
             spider = FollowAllSpider(total=100000)
-            self.crawler.crawl(spider)
-            self.crawler.start()
+            crawler = self.crawler_process.create_crawler()
+            crawler.crawl(spider)
+            self.crawler_process.start()

scrapy/commands/crawl.py

@@ -43,6 +43,8 @@ def run(self, args, opts):
         elif len(args) > 1:
             raise UsageError("running 'scrapy crawl' with more than one spider is no longer supported")
         spname = args[0]
-        spider = self.crawler.spiders.create(spname, **opts.spargs)
-        self.crawler.crawl(spider)
-        self.crawler.start()
+
+        crawler = self.crawler_process.create_crawler()
+        spider = crawler.spiders.create(spname, **opts.spargs)
+        crawler.crawl(spider)
+        self.crawler_process.start()

scrapy/commands/edit.py

@@ -24,9 +24,11 @@ def _err(self, msg):
     def run(self, args, opts):
         if len(args) != 1:
             raise UsageError()
-        editor = self.crawler.settings['EDITOR']
+
+        crawler = self.crawler_process.create_crawler()
+        editor = crawler.settings['EDITOR']
         try:
-            spider = self.crawler.spiders.create(args[0])
+            spider = crawler.spiders.create(args[0])
         except KeyError:
             return self._err("Spider not found: %s" % args[0])
 

scrapy/commands/fetch.py

@@ -47,12 +47,12 @@ def run(self, args, opts):
         request = Request(args[0], callback=cb, dont_filter=True)
         request.meta['handle_httpstatus_all'] = True
 
+        crawler = self.crawler_process.create_crawler()
         spider = None
         if opts.spider:
-            spider = self.crawler.spiders.create(opts.spider)
+            spider = crawler.spiders.create(opts.spider)
         else:
-            spider = create_spider_for_request(self.crawler.spiders, request, \
+            spider = create_spider_for_request(crawler.spiders, request, \
                 default_spider=BaseSpider('default'))
-        self.crawler.crawl(spider, [request])
-        self.crawler.start()
-
+        crawler.crawl(spider, [request])
+        self.crawler_process.start()

scrapy/commands/genspider.py

@@ -62,7 +62,8 @@ def run(self, args, opts):
             return
 
         try:
-            spider = self.crawler.spiders.create(name)
+            crawler = self.crawler_process.create_crawler()
+            spider = crawler.spiders.create(name)
         except KeyError:
             pass
         else:

scrapy/commands/list.py

@@ -9,5 +9,6 @@ def short_desc(self):
         return "List available spiders"
 
     def run(self, args, opts):
-        for s in self.crawler.spiders.list():
+        crawler = self.crawler_process.create_crawler()
+        for s in crawler.spiders.list():
             print s

scrapy/commands/parse.py

@@ -125,12 +125,12 @@ def get_callback_from_rules(self, response):
     def set_spider(self, url, opts):
         if opts.spider:
             try:
-                self.spider = self.crawler.spiders.create(opts.spider, **opts.spargs)
+                self.spider = self.pcrawler.spiders.create(opts.spider, **opts.spargs)
             except KeyError:
                 log.msg(format='Unable to find spider: %(spider)s',
                         level=log.ERROR, spider=opts.spider)
         else:
-            self.spider = create_spider_for_request(self.crawler.spiders, Request(url), **opts.spargs)
+            self.spider = create_spider_for_request(self.pcrawler.spiders, Request(url), **opts.spargs)
             if not self.spider:
                 log.msg(format='Unable to find spider for: %(url)s',
                         level=log.ERROR, url=url)
@@ -139,8 +139,8 @@ def start_parsing(self, url, opts):
         request = Request(url, opts.callback)
         request = self.prepare_request(request, opts)
 
-        self.crawler.crawl(self.spider, [request])
-        self.crawler.start()
+        self.pcrawler.crawl(self.spider, [request])
+        self.crawler_process.start()
 
         if not self.first_response:
             log.msg(format='No response downloaded for: %(request)s',
@@ -174,7 +174,7 @@ def callback(response):
 
             items, requests = self.run_callback(response, cb)
             if opts.pipelines:
-                itemproc = self.crawler.engine.scraper.itemproc
+                itemproc = self.pcrawler.engine.scraper.itemproc
                 for item in items:
                     itemproc.process_item(item, self.spider)
             self.add_items(depth, items)
@@ -207,6 +207,7 @@ def run(self, args, opts):
             url = args[0]
 
         # prepare spider
+        self.pcrawler = self.crawler_process.create_crawler()
         self.set_spider(url, opts)
 
         if self.spider and opts.depth > 0:

scrapy/commands/runspider.py

@@ -74,5 +74,6 @@ def run(self, args, opts):
             raise UsageError("No spider found in file: %s\n" % filename)
         spider = spclasses.pop()(**opts.spargs)
 
-        self.crawler.crawl(spider)
-        self.crawler.start()
+        crawler = self.crawler_process.create_crawler()
+        crawler.crawl(spider)
+        self.crawler_process.start()

scrapy/commands/settings.py

@@ -25,7 +25,8 @@ def add_options(self, parser):
             help="print setting value, intepreted as an float")
 
     def run(self, args, opts):
-        settings = self.crawler.settings
+        crawler = self.crawler_process.create_crawler()
+        settings = crawler.settings
         if opts.get:
             print settings.get(opts.get)
         elif opts.getbool:

scrapy/commands/shell.py

@@ -8,7 +8,7 @@
 
 from scrapy.command import ScrapyCommand
 from scrapy.shell import Shell
-from scrapy import log
+
 
 class Command(ScrapyCommand):
 
@@ -38,15 +38,17 @@ def update_vars(self, vars):
         pass
 
     def run(self, args, opts):
+        crawler = self.crawler_process.create_crawler()
+
         url = args[0] if args else None
-        spider = None
-        if opts.spider:
-            spider = self.crawler.spiders.create(opts.spider)
-        shell = Shell(self.crawler, update_vars=self.update_vars, code=opts.code)
+        spider = crawler.spiders.create(opts.spider) if opts.spider else None
+
+        shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
         self._start_crawler_thread()
         shell.start(url=url, spider=spider)
 
     def _start_crawler_thread(self):
-        t = Thread(target=self.crawler.start)
+        self.crawler_process.print_headers()
+        t = Thread(target=self.crawler_process.start, kwargs={'headers': False})
         t.daemon = True
         t.start()

scrapy/crawler.py

@@ -126,7 +126,7 @@ def __init__(self, settings):
         self.crawlers = {}
         self.stopping = False
 
-    def create_crawler(self, name):
+    def create_crawler(self, name=None):
         if name not in self.crawlers:
             crawler = Crawler(self.settings)
             crawler.configure()
@@ -155,8 +155,12 @@ def check_done(self, **kwargs):
         else:
             self._stop_reactor()
 
-    def start(self):
+    def print_headers(self):
         log.scrapy_info(self.settings)
+
+    def start(self, headers=True):
+        if headers:
+            self.print_headers()
         return super(CrawlerProcess, self).start()
 
     @defer.inlineCallbacks