Merge pull request scrapy#1721 from Digenis/offsitemw_subdomains

[MRG+1] tests+doc for subdomains in offsite middleware

Author: eliasdorneles

docs/topics/spider-middleware.rst

@@ -273,6 +273,9 @@ OffsiteMiddleware
 
    This middleware filters out every request whose host names aren't in the
    spider's :attr:`~scrapy.spiders.Spider.allowed_domains` attribute.
+   All subdomains of any domain in the list are also allowed.
+   E.g. the rule ``www.example.org`` will also allow ``bob.www.example.org``
+   but not ``www2.example.com`` nor ``example.com``.
 
    When your spider returns a request for a domain not belonging to those
    covered by the spider, this middleware will log a debug message similar to

docs/topics/spiders.rst

@@ -76,7 +76,7 @@ scrapy.Spider
 
        An optional list of strings containing domains that this spider is
        allowed to crawl. Requests for URLs not belonging to the domain names
-       specified in this list won't be followed if
+       specified in this list (or their subdomains) won't be followed if
        :class:`~scrapy.spidermiddlewares.offsite.OffsiteMiddleware` is enabled.
 
    .. attribute:: start_urls

tests/test_spidermiddleware_offsite.py

@@ -16,21 +16,24 @@ def setUp(self):
         self.mw.spider_opened(self.spider)
 
     def _get_spiderargs(self):
-        return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org'])
+        return dict(name='foo', allowed_domains=['scrapytest.org', 'scrapy.org', 'scrapy.test.org'])
 
     def test_process_spider_output(self):
         res = Response('http://scrapytest.org')
 
         onsite_reqs = [Request('http://scrapytest.org/1'),
                        Request('http://scrapy.org/1'),
                        Request('http://sub.scrapy.org/1'),
-                       Request('http://offsite.tld/letmepass', dont_filter=True)]
+                       Request('http://offsite.tld/letmepass', dont_filter=True),
+                       Request('http://scrapy.test.org/')]
         offsite_reqs = [Request('http://scrapy2.org'),
                        Request('http://offsite.tld/'),
                        Request('http://offsite.tld/scrapytest.org'),
                        Request('http://offsite.tld/rogue.scrapytest.org'),
                        Request('http://rogue.scrapytest.org.haha.com'),
-                       Request('http://roguescrapytest.org')]
+                       Request('http://roguescrapytest.org'),
+                       Request('http://test.org/'),
+                       Request('http://notscrapy.test.org/')]
         reqs = onsite_reqs + offsite_reqs
 
         out = list(self.mw.process_spider_output(res, reqs, self.spider))