Merge pull request scrapy#1415 from scrapy/nyov-py3

nyov's PY3 changes

Author: dangra

requirements-py3.txt

@@ -0,0 +1,6 @@
+Twisted >= 15.1.0
+lxml>=3.2.4
+pyOpenSSL>=0.13.1
+cssselect>=0.9
+queuelib>=1.1.1
+w3lib>=1.8.0

scrapy/cmdline.py

@@ -18,10 +18,10 @@ def _iter_command_classes(module_name):
     # TODO: add `name` attribute to commands and and merge this function with
     # scrapy.utils.spider.iter_spider_classes
     for module in walk_modules(module_name):
-        for obj in vars(module).itervalues():
+        for obj in vars(module).values():
             if inspect.isclass(obj) and \
-               issubclass(obj, ScrapyCommand) and \
-               obj.__module__ == module.__name__:
+                    issubclass(obj, ScrapyCommand) and \
+                    obj.__module__ == module.__name__:
                 yield obj
 
 def _get_commands_from_module(module, inproject):

scrapy/core/downloader/handlers/s3.py

@@ -1,4 +1,4 @@
-from urlparse import unquote
+from six.moves.urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached

scrapy/core/downloader/middleware.py

@@ -3,7 +3,7 @@
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-
+import six
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
@@ -32,7 +32,7 @@ def process_request(request):
                 response = method(request=request, spider=spider)
                 assert response is None or isinstance(response, (Response, Request)), \
                         'Middleware %s.process_request must return None, Response or Request, got %s' % \
-                        (method.im_self.__class__.__name__, response.__class__.__name__)
+                        (six.get_method_self(method).__class__.__name__, response.__class__.__name__)
                 if response:
                     return response
             return download_func(request=request, spider=spider)
@@ -46,7 +46,7 @@ def process_response(response):
                 response = method(request=request, response=response, spider=spider)
                 assert isinstance(response, (Response, Request)), \
                     'Middleware %s.process_response must return Response or Request, got %s' % \
-                    (method.im_self.__class__.__name__, type(response))
+                    (six.get_method_self(method).__class__.__name__, type(response))
                 if isinstance(response, Request):
                     return response
             return response
@@ -57,7 +57,7 @@ def process_exception(_failure):
                 response = method(request=request, exception=exception, spider=spider)
                 assert response is None or isinstance(response, (Response, Request)), \
                     'Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                    (method.im_self.__class__.__name__, type(response))
+                    (six.get_method_self(method).__class__.__name__, type(response))
                 if response:
                     return response
             return _failure

scrapy/core/spidermw.py

@@ -3,7 +3,7 @@
 
 See documentation in docs/topics/spider-middleware.rst
 """
-
+import six
 from twisted.python.failure import Failure
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
@@ -33,7 +33,9 @@ def _add_middleware(self, mw):
             self.methods['process_start_requests'].insert(0, mw.process_start_requests)
 
     def scrape_response(self, scrape_func, response, request, spider):
-        fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__)
+        fname = lambda f:'%s.%s' % (
+                six.get_method_self(f).__class__.__name__,
+                six.get_method_function(f).__name__)
 
         def process_spider_input(response):
             for method in self.methods['process_spider_input']:

scrapy/linkextractors/htmlparser.py

@@ -3,7 +3,7 @@
 """
 
 import warnings
-from HTMLParser import HTMLParser
+from six.moves.html_parser import HTMLParser
 from six.moves.urllib.parse import urljoin
 
 from w3lib.url import safe_url_string

scrapy/pipelines/files.py

@@ -26,6 +26,7 @@
 from scrapy.http import Request
 from scrapy.utils.misc import md5sum
 from scrapy.utils.log import failure_to_exc_info
+from scrapy.utils.python import to_bytes, to_native_str
 
 logger = logging.getLogger(__name__)
 
@@ -198,7 +199,7 @@ def _onsuccess(result):
             if age_days > self.EXPIRES:
                 return  # returning None force download
 
-            referer = request.headers.get('Referer')
+            referer = _get_referer(request)
             logger.debug(
                 'File (uptodate): Downloaded %(medianame)s from %(request)s '
                 'referred in <%(referer)s>',
@@ -224,7 +225,7 @@ def _onsuccess(result):
 
     def media_failed(self, failure, request, info):
         if not isinstance(failure.value, IgnoreRequest):
-            referer = request.headers.get('Referer')
+            referer = _get_referer(request)
             logger.warning(
                 'File (unknown-error): Error downloading %(medianame)s from '
                 '%(request)s referred in <%(referer)s>: %(exception)s',
@@ -236,7 +237,7 @@ def media_failed(self, failure, request, info):
         raise FileException
 
     def media_downloaded(self, response, request, info):
-        referer = request.headers.get('Referer')
+        referer = _get_referer(request)
 
         if response.status != 200:
             logger.warning(
@@ -330,11 +331,19 @@ def _warn():
             return self.file_key(url)
         ## end of deprecation warning block
 
-        media_guid = hashlib.sha1(url).hexdigest()  # change to request.url after deprecation
+        media_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
         media_ext = os.path.splitext(url)[1]  # change to request.url after deprecation
         return 'full/%s%s' % (media_guid, media_ext)
 
     # deprecated
     def file_key(self, url):
         return self.file_path(url)
     file_key._base = True
+
+
+def _get_referer(request):
+    """ Return Referer HTTP header suitable for logging """
+    referrer = request.headers.get('Referer')
+    if referrer is None:
+        return referrer
+    return to_native_str(referrer, errors='replace')

scrapy/pipelines/images.py

@@ -15,6 +15,7 @@
 from PIL import Image
 
 from scrapy.utils.misc import md5sum
+from scrapy.utils.python import to_bytes
 from scrapy.http import Request
 from scrapy.exceptions import DropItem
 #TODO: from scrapy.pipelines.media import MediaPipeline
@@ -138,7 +139,7 @@ def _warn():
             return self.image_key(url)
         ## end of deprecation warning block
 
-        image_guid = hashlib.sha1(url).hexdigest()  # change to request.url after deprecation
+        image_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
         return 'full/%s.jpg' % (image_guid)
 
     def thumb_path(self, request, thumb_id, response=None, info=None):
@@ -163,7 +164,7 @@ def _warn():
             return self.thumb_key(url, thumb_id)
         ## end of deprecation warning block
 
-        thumb_guid = hashlib.sha1(url).hexdigest()  # change to request.url after deprecation
+        thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()  # change to request.url after deprecation
         return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid)
 
     # deprecated

scrapy/utils/testproc.py

@@ -35,8 +35,8 @@ class TestProcessProtocol(protocol.ProcessProtocol):
 
     def __init__(self):
         self.deferred = defer.Deferred()
-        self.out = ''
-        self.err = ''
+        self.out = b''
+        self.err = b''
         self.exitcode = None
 
     def outReceived(self, data):

tests/py3-ignores.txt

@@ -1,12 +1,9 @@
 tests/test_closespider.py
-tests/test_cmdline/__init__.py
 tests/test_command_fetch.py
 tests/test_command_shell.py
 tests/test_commands.py
-tests/test_command_version.py
 tests/test_exporters.py
 tests/test_linkextractors.py
-tests/test_loader.py
 tests/test_crawl.py
 tests/test_crawler.py
 tests/test_downloader_handlers.py

tests/requirements-py3.txt

@@ -0,0 +1,4 @@
+pytest>=2.6.0
+pytest-twisted
+testfixtures
+jmespath

tests/test_cmdline/__init__.py

@@ -1,20 +1,30 @@
+import os
 import sys
+import shutil
+import pstats
+import tempfile
 from subprocess import Popen, PIPE
 import unittest
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from io import StringIO
 
 from scrapy.utils.test import get_testenv
 
+
 class CmdlineTest(unittest.TestCase):
 
     def setUp(self):
         self.env = get_testenv()
         self.env['SCRAPY_SETTINGS_MODULE'] = 'tests.test_cmdline.settings'
 
     def _execute(self, *new_args, **kwargs):
+        encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
         args = (sys.executable, '-m', 'scrapy.cmdline') + new_args
         proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
-        comm = proc.communicate()
-        return comm[0].strip()
+        comm = proc.communicate()[0].strip()
+        return comm.decode(encoding)
 
     def test_default_settings(self):
         self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
@@ -29,3 +39,18 @@ def test_override_settings_using_envvar(self):
         self.assertEqual(self._execute('settings', '--get', 'TEST1'), \
                          'override')
 
+    def test_profiling(self):
+        path = tempfile.mkdtemp()
+        filename = os.path.join(path, 'res.prof')
+        try:
+            self._execute('version', '--profile', filename)
+            self.assertTrue(os.path.exists(filename))
+            out = StringIO()
+            stats = pstats.Stats(filename, stream=out)
+            stats.print_stats()
+            out.seek(0)
+            stats = out.read()
+            self.assertIn('scrapy/commands/version.py', stats)
+            self.assertIn('tottime', stats)
+        finally:
+            shutil.rmtree(path)

tests/test_command_version.py

@@ -1,3 +1,4 @@
+import sys
 from twisted.trial import unittest
 from twisted.internet import defer
 
@@ -11,5 +12,6 @@ class VersionTest(ProcessTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_output(self):
+        encoding = getattr(sys.stdout, 'encoding') or 'utf-8'
         _, out, _ = yield self.execute([])
-        self.assertEqual(out.strip(), "Scrapy %s" % scrapy.__version__)
+        self.assertEqual(out.strip().decode(encoding), "Scrapy %s" % scrapy.__version__)

tests/test_crawl.py

@@ -141,7 +141,7 @@ def test_start_requests_dupes(self):
     def test_unbounded_response(self):
         # Completeness of responses without Content-Length or Transfer-Encoding
         # can not be determined, we treat them as valid but flagged as "partial"
-        from urllib import urlencode
+        from six.moves.urllib.parse import urlencode
         query = urlencode({'raw': '''\
 HTTP/1.1 200 OK
 Server: Apache-Coyote/1.1

tests/test_loader.py

@@ -1,4 +1,5 @@
 import unittest
+import six
 from functools import partial
 
 from scrapy.loader import ItemLoader
@@ -141,7 +142,7 @@ def test_replace_value(self):
 
     def test_get_value(self):
         il = NameItemLoader()
-        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), unicode.upper))
+        self.assertEqual(u'FOO', il.get_value([u'foo', u'bar'], TakeFirst(), six.text_type.upper))
         self.assertEqual([u'foo', u'bar'], il.get_value([u'name:foo', u'name:bar'], re=u'name:(.*)$'))
         self.assertEqual(u'foo', il.get_value([u'name:foo', u'name:bar'], TakeFirst(), re=u'name:(.*)$'))
 
@@ -242,15 +243,15 @@ class IdentityDefaultedItemLoader(DefaultedItemLoader):
 
     def test_extend_custom_input_processors(self):
         class ChildItemLoader(TestItemLoader):
-            name_in = MapCompose(TestItemLoader.name_in, unicode.swapcase)
+            name_in = MapCompose(TestItemLoader.name_in, six.text_type.swapcase)
 
         il = ChildItemLoader()
         il.add_value('name', u'marta')
         self.assertEqual(il.get_output_value('name'), [u'mARTA'])
 
     def test_extend_default_input_processors(self):
         class ChildDefaultedItemLoader(DefaultedItemLoader):
-            name_in = MapCompose(DefaultedItemLoader.default_input_processor, unicode.swapcase)
+            name_in = MapCompose(DefaultedItemLoader.default_input_processor, six.text_type.swapcase)
 
         il = ChildDefaultedItemLoader()
         il.add_value('name', u'marta')
@@ -423,7 +424,7 @@ def test_join(self):
         self.assertRaises(TypeError, proc, [None, '', 'hello', 'world'])
         self.assertEqual(proc(['', 'hello', 'world']), u' hello world')
         self.assertEqual(proc(['hello', 'world']), u'hello world')
-        self.assert_(isinstance(proc(['hello', 'world']), unicode))
+        self.assert_(isinstance(proc(['hello', 'world']), six.text_type))
 
     def test_compose(self):
         proc = Compose(lambda v: v[0], str.upper)
@@ -435,13 +436,13 @@ def test_compose(self):
 
     def test_mapcompose(self):
         filter_world = lambda x: None if x == 'world' else x
-        proc = MapCompose(filter_world, unicode.upper)
+        proc = MapCompose(filter_world, six.text_type.upper)
         self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
                          [u'HELLO', u'THIS', u'IS', u'SCRAPY'])
 
 
 class SelectortemLoaderTest(unittest.TestCase):
-    response = HtmlResponse(url="", body="""
+    response = HtmlResponse(url="", encoding='utf-8', body=b"""
     <html>
     <body>
     <div id="id">marta</div>

tests/test_pipeline_files.py

@@ -12,6 +12,7 @@
 from scrapy.item import Item, Field
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
+from scrapy.utils.python import to_bytes
 
 from tests import mock
 
@@ -103,7 +104,7 @@ def test_file_expired(self):
 
 class DeprecatedFilesPipeline(FilesPipeline):
     def file_key(self, url):
-        media_guid = hashlib.sha1(url).hexdigest()
+        media_guid = hashlib.sha1(to_bytes(url)).hexdigest()
         media_ext = os.path.splitext(url)[1]
         return 'empty/%s%s' % (media_guid, media_ext)
 

tests/test_pipeline_images.py

@@ -10,6 +10,7 @@
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.pipelines.images import ImagesPipeline
+from scrapy.utils.python import to_bytes
 
 skip = False
 try:
@@ -100,11 +101,11 @@ def file_key(self, url):
         return self.image_key(url)
 
     def image_key(self, url):
-        image_guid = hashlib.sha1(url).hexdigest()
+        image_guid = hashlib.sha1(to_bytes(url)).hexdigest()
         return 'empty/%s.jpg' % (image_guid)
 
     def thumb_key(self, url, thumb_id):
-        thumb_guid = hashlib.sha1(url).hexdigest()
+        thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest()
         return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid)
 
 

tests/test_selector_csstranslator.py

@@ -9,7 +9,7 @@
 from cssselect.xpath import ExpressionError
 
 
-HTMLBODY = '''
+HTMLBODY = b'''
 <html>
 <body>
 <div>