Merge branch '0.16' of github.com:scrapy/scrapy into 0.16

Author: pablohoffman

docs/news.rst

@@ -3,6 +3,17 @@
 Release notes
 =============
 
+0.16.2 (released 2012-11-09)
+----------------------------
+
+- scrapy contracts: python2.6 compat (:commit:`a4a9199`)
+- scrapy contracts verbose option (:commit:`ec41673`)
+- proper unittest-like output for scrapy contracts (:commit:`86635e4`)
+- added open_in_browser to debugging doc (:commit:`c9b690d`)
+- removed reference to global scrapy stats from settings doc (:commit:`dd55067`)
+- Fix SpiderState bug in Windows platforms (:commit:`58998f4`)
+
+
 0.16.1 (released 2012-10-26)
 ----------------------------
 

scrapy/__init__.py

@@ -2,8 +2,8 @@
 Scrapy - a screen scraping framework written in Python
 """
 
-version_info = (0, 16, 1)
-__version__ = "0.16.1"
+version_info = (0, 16, 2)
+__version__ = "0.16.2"
 
 import sys, os, warnings
 

scrapy/commands/check.py

@@ -1,11 +1,14 @@
 from collections import defaultdict
 from functools import wraps
+from unittest import TextTestRunner
 
+from scrapy import signals
 from scrapy.command import ScrapyCommand
 from scrapy.contracts import ContractsManager
 from scrapy.utils.misc import load_object
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.conf import build_component_list
+from scrapy.xlib.pydispatch import dispatcher
 
 
 def _generate(cb):
@@ -31,6 +34,8 @@ def add_options(self, parser):
         ScrapyCommand.add_options(self, parser)
         parser.add_option("-l", "--list", dest="list", action="store_true",
             help="only list contracts, without checking them")
+        parser.add_option("-v", "--verbose", dest="verbose", default=1, action="count",
+            help="print all contract hooks")
 
     def run(self, args, opts):
         # load contracts
@@ -39,6 +44,7 @@ def run(self, args, opts):
             self.settings['SPIDER_CONTRACTS'],
         )
         self.conman = ContractsManager([load_object(c) for c in contracts])
+        self.results = TextTestRunner(verbosity=opts.verbose)._makeResult()
 
         # contract requests
         contract_reqs = defaultdict(list)
@@ -61,6 +67,8 @@ def run(self, args, opts):
                 for method in sorted(methods):
                     print '  * %s' % method
         else:
+            dispatcher.connect(self.results.printErrors,
+                    signals.engine_stopped)
             self.crawler.start()
 
     def get_requests(self, spider):
@@ -69,7 +77,7 @@ def get_requests(self, spider):
         for key, value in vars(type(spider)).items():
             if callable(value) and value.__doc__:
                 bound_method = value.__get__(spider, type(spider))
-                request = self.conman.from_method(bound_method)
+                request = self.conman.from_method(bound_method, self.results)
 
                 if request:
                     request.callback = _generate(request.callback)

scrapy/contracts/__init__.py

@@ -1,10 +1,11 @@
+import sys
 import re
 from functools import wraps
+from unittest import TestCase
 
 from scrapy.http import Request
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.python import get_spec
-from scrapy.exceptions import ContractFail
 
 
 class ContractsManager(object):
@@ -27,7 +28,7 @@ def extract_contracts(self, method):
 
         return contracts
 
-    def from_method(self, method, fail=False):
+    def from_method(self, method, results):
         contracts = self.extract_contracts(method)
         if contracts:
             # calculate request args
@@ -43,9 +44,9 @@ def from_method(self, method, fail=False):
 
                 # execute pre and post hooks in order
                 for contract in reversed(contracts):
-                    request = contract.add_pre_hook(request, fail)
+                    request = contract.add_pre_hook(request, results)
                 for contract in contracts:
-                    request = contract.add_post_hook(request, fail)
+                    request = contract.add_post_hook(request, results)
 
                 return request
 
@@ -54,49 +55,67 @@ class Contract(object):
     """ Abstract class for contracts """
 
     def __init__(self, method, *args):
-        self.method = method
+        self.testcase_pre = self.create_testcase(method, 'pre-hook')
+        self.testcase_post = self.create_testcase(method, 'post-hook')
         self.args = args
 
-    def add_pre_hook(self, request, fail=False):
-        cb = request.callback
-
-        @wraps(cb)
-        def wrapper(response):
-            try:
-                self.pre_process(response)
-            except ContractFail as e:
-                if fail:
-                    raise
+    def create_testcase(self, method, hook):
+        spider = method.__self__.name
+
+        class ContractTestCase(TestCase):
+            def __str__(_self):
+                return "[%s] %s (@%s %s)" % (spider, method.__name__, self.name, hook)
+
+        name = '%s_%s' % (spider, method.__name__)
+        setattr(ContractTestCase, name, lambda x: x)
+        return ContractTestCase(name)
+
+    def add_pre_hook(self, request, results):
+        if hasattr(self, 'pre_process'):
+            cb = request.callback
+
+            @wraps(cb)
+            def wrapper(response):
+                try:
+                    results.startTest(self.testcase_pre)
+                    self.pre_process(response)
+                    results.stopTest(self.testcase_pre)
+                except AssertionError:
+                    results.addFailure(self.testcase_pre, sys.exc_info())
+                except Exception:
+                    results.addError(self.testcase_pre, sys.exc_info())
                 else:
-                    print e.format(self.method)
-            return list(iterate_spider_output(cb(response)))
+                    results.addSuccess(self.testcase_pre)
+                finally:
+                    return list(iterate_spider_output(cb(response)))
+
+            request.callback = wrapper
 
-        request.callback = wrapper
         return request
 
-    def add_post_hook(self, request, fail=False):
-        cb = request.callback
-
-        @wraps(cb)
-        def wrapper(response):
-            output = list(iterate_spider_output(cb(response)))
-            try:
-                self.post_process(output)
-            except ContractFail as e:
-                if fail:
-                    raise
+    def add_post_hook(self, request, results):
+        if hasattr(self, 'post_process'):
+            cb = request.callback
+
+            @wraps(cb)
+            def wrapper(response):
+                try:
+                    output = list(iterate_spider_output(cb(response)))
+                    results.startTest(self.testcase_post)
+                    self.post_process(output)
+                    results.stopTest(self.testcase_post)
+                except AssertionError:
+                    results.addFailure(self.testcase_post, sys.exc_info())
+                except Exception:
+                    results.addError(self.testcase_post, sys.exc_info())
                 else:
-                    print e.format(self.method)
-            return output
+                    results.addSuccess(self.testcase_post)
+                finally:
+                    return output
+
+            request.callback = wrapper
 
-        request.callback = wrapper
         return request
 
     def adjust_request_args(self, args):
         return args
-
-    def pre_process(self, response):
-        pass
-
-    def post_process(self, output):
-        pass

scrapy/exceptions.py

@@ -52,7 +52,4 @@ class ScrapyDeprecationWarning(Warning):
 
 class ContractFail(AssertionError):
     """Error raised in case of a failing contract"""
-    
-    def format(self, method):
-        return '[FAILED] %s:%s\n>>> %s\n' % \
-            (method.im_class.name, method.__name__, self)
+    pass

scrapy/tests/test_contracts.py

@@ -1,9 +1,10 @@
+from unittest import TextTestRunner
+
 from twisted.trial import unittest
 
 from scrapy.spider import BaseSpider
 from scrapy.http import Request
 from scrapy.item import Item, Field
-from scrapy.exceptions import ContractFail
 from scrapy.contracts import ContractsManager
 from scrapy.contracts.default import (
     UrlContract,
@@ -71,54 +72,69 @@ def parse_no_url(http://github.com/self,%20response):
 class ContractsManagerTest(unittest.TestCase):
     contracts = [UrlContract, ReturnsContract, ScrapesContract]
 
+    def setUp(self):
+        self.conman = ContractsManager(self.contracts)
+        self.results = TextTestRunner()._makeResult()
+        self.results.stream = None
+
+    def should_succeed(self):
+        self.assertFalse(self.results.failures)
+        self.assertFalse(self.results.errors)
+
+    def should_fail(self):
+        self.assertTrue(self.results.failures)
+        self.assertFalse(self.results.errors)
+
     def test_contracts(self):
-        conman = ContractsManager(self.contracts)
+        spider = TestSpider()
 
         # extract contracts correctly
-        contracts = conman.extract_contracts(TestSpider.returns_request)
+        contracts = self.conman.extract_contracts(spider.returns_request)
         self.assertEqual(len(contracts), 2)
         self.assertEqual(frozenset(map(type, contracts)),
             frozenset([UrlContract, ReturnsContract]))
 
         # returns request for valid method
-        request = conman.from_method(TestSpider.returns_request)
+        request = self.conman.from_method(spider.returns_request, self.results)
         self.assertNotEqual(request, None)
 
         # no request for missing url
-        request = conman.from_method(TestSpider.parse_no_url)
+        request = self.conman.from_method(spider.parse_no_url, self.results)
         self.assertEqual(request, None)
 
     def test_returns(self):
-        conman = ContractsManager(self.contracts)
-
         spider = TestSpider()
         response = ResponseMock()
 
         # returns_item
-        request = conman.from_method(spider.returns_item, fail=True)
+        request = self.conman.from_method(spider.returns_item, self.results)
         output = request.callback(response)
         self.assertEqual(map(type, output), [TestItem])
+        self.should_succeed()
 
         # returns_request
-        request = conman.from_method(spider.returns_request, fail=True)
+        request = self.conman.from_method(spider.returns_request, self.results)
         output = request.callback(response)
         self.assertEqual(map(type, output), [Request])
+        self.should_succeed()
 
         # returns_fail
-        request = conman.from_method(spider.returns_fail, fail=True)
-        self.assertRaises(ContractFail, request.callback, response)
+        request = self.conman.from_method(spider.returns_fail, self.results)
+        request.callback(response)
+        self.should_fail()
 
     def test_scrapes(self):
-        conman = ContractsManager(self.contracts)
-
         spider = TestSpider()
         response = ResponseMock()
 
         # scrapes_item_ok
-        request = conman.from_method(spider.scrapes_item_ok, fail=True)
+        request = self.conman.from_method(spider.scrapes_item_ok, self.results)
         output = request.callback(response)
         self.assertEqual(map(type, output), [TestItem])
+        self.should_succeed()
 
         # scrapes_item_fail
-        request = conman.from_method(spider.scrapes_item_fail, fail=True)
-        self.assertRaises(ContractFail, request.callback, response)
+        request = self.conman.from_method(spider.scrapes_item_fail,
+                self.results)
+        request.callback(response)
+        self.should_fail()