ajax crawling wasn't expanding for unicode urls

Author: dangra

scrapy/http/request/__init__.py

@@ -54,8 +54,7 @@ def _set_url(http://github.com/self,%20url):
             if self.encoding is None:
                 raise TypeError('Cannot convert unicode url - %s has no encoding' %
                     type(self).__name__)
-            unicode_url = url if isinstance(url, unicode) else url.decode(self.encoding)
-            self._url = safe_url_string(unicode_url, self.encoding)
+            self._set_url(url.encode(self.encoding))
         else:
             raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
         if ':' not in self._url:

scrapy/tests/test_http_request.py

@@ -108,8 +108,12 @@ def test_body(self):
         self.assertEqual(r4.body, "Price: \xa3100")
 
     def test_ajax_url(self):
+        # ascii url
         r = self.request_class(url="http://www.example.com/ajax.html#!key=value")
         self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key=value")
+        # unicode url
+        r = self.request_class(url=u"http://www.example.com/ajax.html#!key=value")
+        self.assertEqual(r.url, "http://www.example.com/ajax.html?_escaped_fragment_=key=value")
 
     def test_copy(self):
         """Test Request copy"""