Merge pull request scrapy#1912 from redapple/https-proxy-pool-key

[MRG+1] Fix HTTP Pool key for HTTPS proxy tunneled connections (CONNECT method)

Author: kmike

scrapy/core/downloader/handlers/http11.py

@@ -135,7 +135,8 @@ def processProxyResponse(self, bytes):
             self._tunnelReadyDeferred.callback(self._protocol)
         else:
             self._tunnelReadyDeferred.errback(
-                TunnelError('Could not open CONNECT tunnel.'))
+                TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % (
+                    self._host, self._port)))
 
     def connectFailed(self, reason):
         """Propagates the errback to the appropriate deferred."""
@@ -201,6 +202,14 @@ def _getEndpoint(self, scheme, host, port):
                 self._contextFactory, self._connectTimeout,
                 self._bindAddress)
 
+    def _requestWithEndpoint(self, key, endpoint, method, parsedURI,
+            headers, bodyProducer, requestPath):
+        # proxy host and port are required for HTTP pool `key`
+        # otherwise, same remote host connection request could reuse
+        # a cached tunneled connection to a different proxy
+        key = key + self._proxyConf
+        return super(TunnelingAgent, self)._requestWithEndpoint(key, endpoint, method, parsedURI,
+            headers, bodyProducer, requestPath)
 
 
 class ScrapyAgent(object):