Merge pull request scrapy#1938 from redapple/https-proxy-connect-sni

Set SNI properly when using CONNECT

Author: kmike

scrapy/core/downloader/handlers/http11.py

@@ -122,7 +122,15 @@ def processProxyResponse(self, bytes):
         """
         self._protocol.dataReceived = self._protocolDataReceived
         if  TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes):
-            self._protocol.transport.startTLS(self._contextFactory,
+            try:
+                # this sets proper Server Name Indication extension
+                # but is only available for Twisted>=14.0
+                sslOptions = self._contextFactory.creatorForNetloc(
+                    self._tunneledHost, self._tunneledPort)
+            except AttributeError:
+                # fall back to non-SNI SSL context factory
+                sslOptions = self._contextFactory
+            self._protocol.transport.startTLS(sslOptions,
                                               self._protocolFactory)
             self._tunnelReadyDeferred.callback(self._protocol)
         else: