S3DownloadHandler: fix auth for requests with quoted paths/query params

Author: immerrr

scrapy/core/downloader/handlers/s3.py

@@ -1,3 +1,5 @@
+from urlparse import unquote
+
 from scrapy import optional_features
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
@@ -54,8 +56,8 @@ def download_request(self, request, spider):
         signed_headers = self.conn.make_request(
                 method=request.method,
                 bucket=bucket,
-                key=p.path,
-                query_args=p.query,
+                key=unquote(p.path),
+                query_args=unquote(p.query),
                 headers=request.headers,
                 data=request.body)
         httpreq = request.replace(url=url, headers=signed_headers)

tests/test_downloader_handlers.py

@@ -482,6 +482,21 @@ def test_request_signing6(self):
         self.assertEqual(httpreq.headers['Authorization'], \
                 'AWS 0PN5J17HBGZHT7JJ3X82:C0FlOtU8Ylb9KDTpZqYkZPX91iI=')
 
+    def test_request_signing7(self):
+        # ensure that spaces are quoted properly before signing
+        req = Request(
+            ("s3://johnsmith/photos/my puppy.jpg"
+             "?response-content-disposition=my puppy.jpg"),
+            method='GET',
+            headers={
+                'Date': 'Tue, 27 Mar 2007 19:42:41 +0000',
+            })
+        httpreq = self.download_request(req, self.spider)
+        self.assertEqual(
+            httpreq.headers['Authorization'],
+            'AWS 0PN5J17HBGZHT7JJ3X82:+CfvG8EZ3YccOrRVMXNaK2eKZmM=')
+
+
 class FTPTestCase(unittest.TestCase):
 
     username = "scrapy"