Merge pull request scrapy#1933 from scrapy/cert-verif-ignore

Ignore HTTPS certificate verification failures

Author: kmike

scrapy/core/downloader/contextfactory.py

@@ -6,11 +6,16 @@
     from zope.interface.declarations import implementer
 
     # the following should be available from Twisted 14.0.0
-    from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
-    from twisted.internet._sslverify import ClientTLSOptions
+    from twisted.internet.ssl import (optionsForClientTLS,
+                                      CertificateOptions,
+                                      platformTrust)
+
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
 
+    from scrapy.core.downloader.tls import ScrapyClientTLSOptions
+
+
     @implementer(IPolicyForHTTPS)
     class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         """
@@ -49,7 +54,7 @@ def getContext(self, hostname=None, port=None):
             return self.getCertificateOptions().getContext()
 
         def creatorForNetloc(self, hostname, port):
-            return ClientTLSOptions(hostname.decode("ascii"), self.getContext())
+            return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext())
 
 
     @implementer(IPolicyForHTTPS)

scrapy/core/downloader/tls.py

@@ -1,6 +1,9 @@
+import logging
 from OpenSSL import SSL
 
 
+logger = logging.getLogger(__name__)
+
 METHOD_SSLv3 = 'SSLv3'
 METHOD_TLS = 'TLS'
 METHOD_TLSv10 = 'TLSv1.0'
@@ -14,3 +17,36 @@
     METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
 }
+
+# ClientTLSOptions requires a recent-enough version of Twisted
+try:
+
+    # taken from twisted/twisted/internet/_sslverify.py
+    try:
+        from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
+    except ImportError:
+        SSL_CB_HANDSHAKE_START = 0x10
+        SSL_CB_HANDSHAKE_DONE = 0x20
+
+    from twisted.internet._sslverify import (ClientTLSOptions,
+                                             _maybeSetHostNameIndication,
+                                             verifyHostname,
+                                             VerificationError)
+
+    class ScrapyClientTLSOptions(ClientTLSOptions):
+        # same as Twisted's ClientTLSOptions,
+        # except that VerificationError is caught
+        # and doesn't close the connection
+        def _identityVerifyingInfoCallback(self, connection, where, ret):
+            if where & SSL_CB_HANDSHAKE_START:
+                _maybeSetHostNameIndication(connection, self._hostnameBytes)
+            elif where & SSL_CB_HANDSHAKE_DONE:
+                try:
+                    verifyHostname(connection, self._hostnameASCII)
+                except VerificationError as e:
+                    logger.warning(e)
+
+except ImportError:
+    # ImportError should not matter for older Twisted versions
+    # as the above is not used in the fallback ScrapyClientContextFactory
+    pass

tests/keys/example-com.cert.pem

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEVTCCAz2gAwIBAgIJANuZ/6fbAJNcMA0GCSqGSIb3DQEBCwUAMH0xCzAJBgNV
+BAYTAlhXMQswCQYDVQQIDAJYVzEVMBMGA1UEBwwMVGhlIEludGVybmV0MQ8wDQYD
+VQQKDAZTY3JhcHkxGDAWBgNVBAMMD3d3dy5leGFtcGxlLmNvbTEfMB0GCSqGSIb3
+DQEJARYQdGVzdEBleGFtcGxlLmNvbTAgFw0xNjA0MjAxMjExNTZaGA8yMTE2MDMy
+NzEyMTE1NlowfTELMAkGA1UEBhMCWFcxCzAJBgNVBAgMAlhXMRUwEwYDVQQHDAxU
+aGUgSW50ZXJuZXQxDzANBgNVBAoMBlNjcmFweTEYMBYGA1UEAwwPd3d3LmV4YW1w
+bGUuY29tMR8wHQYJKoZIhvcNAQkBFhB0ZXN0QGV4YW1wbGUuY29tMIIBIjANBgkq
+hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5r2BzxXivtQXvIwrTGug8l6vjuDhf0WD
+HBU5yIHWvX1rT2MQFuZpu120iFGOK7MBYBBdmAIGsR5cHZ03cqQkAXOGQ5ug5O/u
+d/GZUbcgyv8WCwW71MWLb5CNiDWj/vZq6CUqRg2QkUpkjr/DcQrKsm67yadKLgyd
+G85OyZO6NBuAukQcKrNhspk/Ms55X7RbgYPUbZ0bBee4b3GRnE7PLltIsHo/tloV
+ynC0Sd3T1taYyyG7IJd2LWJELzK0Ww+QUV2qoOdZjl8db1x5c99OR6xY0+Mjf14r
+6kkXOBpPkrJ990qU40+z406u2HPf2abR4D/DUoe9qw+fElCeiuXFFQIDAQABo4HV
+MIHSMB0GA1UdDgQWBBTY3DPInWZxrmQfPHA5w2R+AsbnOjAfBgNVHSMEGDAWgBTY
+3DPInWZxrmQfPHA5w2R+AsbnOjAJBgNVHRMEAjAAMAsGA1UdDwQEAwIFoDBKBgNV
+HREEQzBBggtleGFtcGxlLmNvbYIPd3d3LmV4YW1wbGUuY29tghBtYWlsLmV4YW1w
+bGUuY29tgg9mdHAuZXhhbXBsZS5jb20wLAYJYIZIAYb4QgENBB8WHU9wZW5TU0wg
+R2VuZXJhdGVkIENlcnRpZmljYXRlMA0GCSqGSIb3DQEBCwUAA4IBAQDmnzdIu9XV
+/Tnn5/mt9E98YEsF/eaXBSKG+f2oZc3n2errKwY5qYqULlu8mhajGWTd5EfFCjbD
+lH7nmBdRUAhjzKWntc1G84eaWwHyv+N/5WJrtUfa2A1Ps3Mu9Vz4k2M9HGi/s6KX
+IshezlHe3/TyhIT/WC+MZhpFTL73dpuIgHmp2NjjlJqtG25eC6zmonsc2RApJPa8
+6J0WY/ISH9OwhDSbI9+TIE8QwdC7draiCKK/oid3Jg9fzaEQW+Pr3/4AmYWH1j8s
+iaOVYIXYtt3urQ2Q+qfh34kfRfX5IqAdd1r/fnUjaOLhPNJxGP2KuFaYbdSC9p+n
+bfExzQHUvH2n
+-----END CERTIFICATE-----

tests/keys/example-com.conf

@@ -0,0 +1,84 @@
+# this is copied from http://stackoverflow.com/a/27931596
+[ req ]
+default_bits        = 2048
+default_keyfile     = server-key.pem
+distinguished_name  = subject
+req_extensions      = req_ext
+x509_extensions     = x509_ext
+string_mask         = utf8only
+
+# The Subject DN can be formed using X501 or RFC 4514 (see RFC 4519 for a description).
+#   Its sort of a mashup. For example, RFC 4514 does not provide emailAddress.
+[ subject ]
+countryName         = Country Name (2 letter code)
+countryName_default     = US
+
+stateOrProvinceName     = State or Province Name (full name)
+stateOrProvinceName_default = NY
+
+localityName            = Locality Name (eg, city)
+localityName_default        = New York
+
+organizationName         = Organization Name (eg, company)
+organizationName_default    = Example, LLC
+
+# Use a friendly name here because its presented to the user. The server's DNS
+#   names are placed in Subject Alternate Names. Plus, DNS names here is deprecated
+#   by both IETF and CA/Browser Forums. If you place a DNS name here, then you 
+#   must include the DNS name in the SAN too (otherwise, Chrome and others that
+#   strictly follow the CA/Browser Baseline Requirements will fail).
+commonName          = Common Name (e.g. server FQDN or YOUR name)
+commonName_default      = Example Company
+
+emailAddress            = Email Address
+emailAddress_default        = [email protected]
+
+# Section x509_ext is used when generating a self-signed certificate. I.e., openssl req -x509 ...
+[ x509_ext ]
+
+subjectKeyIdentifier        = hash
+authorityKeyIdentifier  = keyid,issuer
+
+# You only need digitalSignature below. *If* you don't allow
+#   RSA Key transport (i.e., you use ephemeral cipher suites), then
+#   omit keyEncipherment because that's key transport.
+basicConstraints        = CA:FALSE
+keyUsage            = digitalSignature, keyEncipherment
+subjectAltName          = @alternate_names
+nsComment           = "OpenSSL Generated Certificate"
+
+# RFC 5280, Section 4.2.1.12 makes EKU optional
+#   CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused
+#   In either case, you probably only need serverAuth.
+# extendedKeyUsage  = serverAuth, clientAuth
+
+# Section req_ext is used when generating a certificate signing request. I.e., openssl req ...
+[ req_ext ]
+
+subjectKeyIdentifier        = hash
+
+basicConstraints        = CA:FALSE
+keyUsage            = digitalSignature, keyEncipherment
+subjectAltName          = @alternate_names
+nsComment           = "OpenSSL Generated Certificate"
+
+# RFC 5280, Section 4.2.1.12 makes EKU optional
+#   CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused
+#   In either case, you probably only need serverAuth.
+# extendedKeyUsage  = serverAuth, clientAuth
+
+[ alternate_names ]
+
+DNS.1       = example.com
+DNS.2       = www.example.com
+DNS.3       = mail.example.com
+DNS.4       = ftp.example.com
+
+# Add these if you need them. But usually you don't want them or
+#   need them in production. You may need them for development.
+# DNS.5       = localhost
+# DNS.6       = localhost.localdomain
+# DNS.7       = 127.0.0.1
+
+# IPv6 localhost
+# DNS.8     = ::1

tests/keys/example-com.gen.README

@@ -0,0 +1,24 @@
+$ openssl req -config example-com.conf \
+    -new -x509 -sha256 -newkey rsa:2048 -nodes \
+    -keyout example-com.key.pem \
+    -days 36500 \
+    -out example-com.cert.pem
+Generating a 2048 bit RSA private key
+....+++
+.....................+++
+writing new private key to 'example-com.key.pem'
+-----
+You are about to be asked to enter information that will be incorporated
+into your certificate request.
+What you are about to enter is what is called a Distinguished Name or a DN.
+There are quite a few fields but you can leave some blank
+For some fields there will be a default value,
+If you enter '.', the field will be left blank.
+-----
+Country Name (2 letter code) [US]:XW
+State or Province Name (full name) [NY]:XW
+Locality Name (eg, city) [New York]:The Internet
+Organization Name (eg, company) [Example, LLC]:Scrapy
+Common Name (e.g. server FQDN or YOUR name) [Example Company]:www.example.com
+Email Address [[email protected]]:
+

tests/keys/example-com.key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDmvYHPFeK+1Be8
+jCtMa6DyXq+O4OF/RYMcFTnIgda9fWtPYxAW5mm7XbSIUY4rswFgEF2YAgaxHlwd
+nTdypCQBc4ZDm6Dk7+538ZlRtyDK/xYLBbvUxYtvkI2INaP+9mroJSpGDZCRSmSO
+v8NxCsqybrvJp0ouDJ0bzk7Jk7o0G4C6RBwqs2GymT8yznlftFuBg9RtnRsF57hv
+cZGcTs8uW0iwej+2WhXKcLRJ3dPW1pjLIbsgl3YtYkQvMrRbD5BRXaqg51mOXx1v
+XHlz305HrFjT4yN/XivqSRc4Gk+Ssn33SpTjT7PjTq7Yc9/ZptHgP8NSh72rD58S
+UJ6K5cUVAgMBAAECggEAEVxi3vTzmY4Vtx8Ixvg2JOZQ6TwsW0ocoklKjraONLWy
+FEgM1txBSlKzmaohO1J9oP+6Owyz+jGhlqzUljCZcO6DRKT6Bx+yXp3z/jz8H6nv
+u6aTyl+OrTdAHuaCT1W1F4BsXNb6cKQbSs5M4z1/oMtKH9MRdsOGMqhkLzCJSxA+
+E+rRomrP6E9XooLxqpSJooxmX772XPHE7+ZILzRF0viXJ6z0Jd1cOuAQqyIYvmHM
+4313kYJdAKYgJNxe8M8mYLeZcGwImAz/pNQ5R+uVyZlW3kXVzvS7B2m+KcW7Olu8
+r4Ocpdyh20GStpw1f+tk7PLl+SkwFslK+uI7Wl/ygQKBgQD+QsREu1sv507CnvYC
+FZnFryhHUzxMWIX5bvH3YoPbVptqwG5Nj05zIQPCZAEanW4HENCujO9oWbZqLANO
+Th5sNOkBJiC4X6+1NIzQIszaZs4nKIyWNLIOcP4p20k3cR0sS8wLSLwcuW2is27N
+ACKa8u93X1Gb27V0qUhmEqP3NQKBgQDoUY3HRtOVQnpoi5zbTiSKwlBKEkMAawaW
+Q8VSZmrNQZXpcwa2JYN0IeiHnVjctdLul1u9qj5goghTV4XMQ8LSZs0emhvgJxMa
+QpsDLTRr0mBtmduOwZW9a8EcbI2NCth/Irsdl892+y8UVoAO2G6Fgr0DhgXWOJEo
+RcUUkGHyYQKBgHCkT6NLhYhhZykdl0sxGqDTinqey3XfOetZVWUNhfDkG5JdkgBW
+XqDunWW/PCX4XMhQkMLjuSR3qjK7MPO99AhoIFnb9F76NTOIBmInKK4RIX+DnVTm
+H4P6Vv9gIL2pJQ18vva40G3BUGrmJ042ox4WRjSSS+tKmMcIQATIeU+JAoGBAIpr
+TRZW1Oox2c1Aogzo2kzyyfPYPaOaISqW5pLaAviZA0E9D9qnL1OagHmM/s1CaJNQ
+C5FORiw9XsiJdWbnWMUUC1MYb1N29KLI3KNf48P6bQngijjcjuN1uHG/G/fVZnkZ
+sHNJaItzzfFIOLSfr/pMk2HuELw6qAJez7YY8MMBAoGAf5sC2S7duDH2KliMSYhd
+yByHEVTbf7JdJVd/7kq5MPZauJtYztGTSwyyM+gBv+lxdY1jGu/iNu9xAD2DJlE3
+RTPgTIID/PaSmUVotHLq86hbazKPBorx6UWkbGsthEaSF/cTY2eFFFsK/Awoj0yU
+ZhraPWu8S27Pcr9HvZvh5tg=
+-----END PRIVATE KEY-----

tests/mockserver.py

@@ -204,10 +204,10 @@ def __exit__(self, exc_type, exc_value, traceback):
         time.sleep(0.2)
 
 
-def ssl_context_factory():
+def ssl_context_factory(keyfile='keys/cert.pem', certfile='keys/cert.pem'):
     return ssl.DefaultOpenSSLContextFactory(
-         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
-         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
+         os.path.join(os.path.dirname(__file__), keyfile),
+         os.path.join(os.path.dirname(__file__), certfile),
          )
 
 

tests/test_downloader_handlers.py

@@ -119,6 +119,10 @@ class HttpTestCase(unittest.TestCase):
     scheme = 'http'
     download_handler_cls = HTTPDownloadHandler
 
+    # only used for HTTPS tests
+    keyfile = 'keys/cert.pem'
+    certfile = 'keys/cert.pem'
+
     def setUp(self):
         name = self.mktemp()
         os.mkdir(name)
@@ -137,7 +141,8 @@ def setUp(self):
         self.host = 'localhost'
         if self.scheme == 'https':
             self.port = reactor.listenSSL(
-                0, self.wrapper, ssl_context_factory(), interface=self.host)
+                0, self.wrapper, ssl_context_factory(self.keyfile, self.certfile),
+                interface=self.host)
         else:
             self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)
         self.portno = self.port.getHost().port
@@ -318,6 +323,18 @@ class Https11TestCase(Http11TestCase):
     scheme = 'https'
 
 
+class Https11WrongHostnameTestCase(Http11TestCase):
+    scheme = 'https'
+
+    # above tests use a server certificate for "localhost",
+    # client connection to "localhost" too.
+    # here we test that even if the server certificate is for another domain,
+    # "www.example.com" in this case,
+    # the tests still pass
+    keyfile = 'keys/example-com.key.pem'
+    certfile = 'keys/example-com.cert.pem'
+
+
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
     if twisted_version < (11, 1, 0):