PY3 renames (six types)

Author: nyov

scrapy/core/downloader/handlers/s3.py

@@ -1,4 +1,4 @@
-from urlparse import unquote
+from six.moves.urllib.parse import unquote
 
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached

scrapy/core/downloader/middleware.py

@@ -3,7 +3,7 @@
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-
+import six
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
@@ -32,7 +32,7 @@ def process_request(request):
                 response = method(request=request, spider=spider)
                 assert response is None or isinstance(response, (Response, Request)), \
                         'Middleware %s.process_request must return None, Response or Request, got %s' % \
-                        (method.im_self.__class__.__name__, response.__class__.__name__)
+                        (six.get_method_self(method).__class__.__name__, response.__class__.__name__)
                 if response:
                     return response
             return download_func(request=request, spider=spider)
@@ -46,7 +46,7 @@ def process_response(response):
                 response = method(request=request, response=response, spider=spider)
                 assert isinstance(response, (Response, Request)), \
                     'Middleware %s.process_response must return Response or Request, got %s' % \
-                    (method.im_self.__class__.__name__, type(response))
+                    (six.get_method_self(method).__class__.__name__, type(response))
                 if isinstance(response, Request):
                     return response
             return response
@@ -57,7 +57,7 @@ def process_exception(_failure):
                 response = method(request=request, exception=exception, spider=spider)
                 assert response is None or isinstance(response, (Response, Request)), \
                     'Middleware %s.process_exception must return None, Response or Request, got %s' % \
-                    (method.im_self.__class__.__name__, type(response))
+                    (six.get_method_self(method).__class__.__name__, type(response))
                 if response:
                     return response
             return _failure

scrapy/core/spidermw.py

@@ -3,7 +3,7 @@
 
 See documentation in docs/topics/spider-middleware.rst
 """
-
+import six
 from twisted.python.failure import Failure
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.defer import mustbe_deferred
@@ -33,7 +33,9 @@ def _add_middleware(self, mw):
             self.methods['process_start_requests'].insert(0, mw.process_start_requests)
 
     def scrape_response(self, scrape_func, response, request, spider):
-        fname = lambda f:'%s.%s' % (f.im_self.__class__.__name__, f.im_func.__name__)
+        fname = lambda f:'%s.%s' % (
+                six.get_method_self(f).__class__.__name__,
+                six.get_method_function(f).__name__)
 
         def process_spider_input(response):
             for method in self.methods['process_spider_input']:

scrapy/linkextractors/htmlparser.py

@@ -3,7 +3,7 @@
 """
 
 import warnings
-from HTMLParser import HTMLParser
+from six.moves.html_parser import HTMLParser
 from six.moves.urllib.parse import urljoin
 
 from w3lib.url import safe_url_string

tests/test_crawl.py

@@ -141,7 +141,7 @@ def test_start_requests_dupes(self):
     def test_unbounded_response(self):
         # Completeness of responses without Content-Length or Transfer-Encoding
         # can not be determined, we treat them as valid but flagged as "partial"
-        from urllib import urlencode
+        from six.moves.urllib.parse import urlencode
         query = urlencode({'raw': '''\
 HTTP/1.1 200 OK
 Server: Apache-Coyote/1.1