do not fail handling unicode xpaths in libxml2 backed selectors

Author: dangra

scrapy/selector/libxml2sel.py

@@ -37,6 +37,7 @@ def __init__(self, response=None, text=None, node=None, parent=None, expr=None):
     def select(self, xpath):
         if hasattr(self.xmlNode, 'xpathEval'):
             self.doc.xpathContext.setContextNode(self.xmlNode)
+            xpath = unicode_to_str(xpath, 'utf-8')
             try:
                 xpath_result = self.doc.xpathContext.xpathEval(xpath)
             except libxml2.xpathError:

scrapy/tests/test_selector.py

@@ -43,6 +43,12 @@ def test_selector_simple(self):
         self.assertEqual([x.extract() for x in xpath.select("concat(//input[@name='a']/@value, //input[@name='b']/@value)")],
                          [u'12'])
 
+    def test_selector_unicode_query(self):
+        body = u"<p><input name='\xa9' value='1'/></p>"
+        response = TextResponse(url="http://example.com", body=body, encoding='utf8')
+        xpath = self.hxs_cls(response)
+        self.assertEqual(xpath.select(u'//input[@name="\xa9"]/@value').extract(), [u'1'])
+
     @libxml2debug
     def test_selector_same_type(self):
         """Test XPathSelector returning the same type in x() method"""