Twisted 15.3.0 does not raises expected exception serializing lambda functions

Author: dangra

scrapy/_monkeypatches.py

@@ -1,4 +1,5 @@
 import sys
+from six.moves import copyreg
 
 if sys.version_info[0] == 2:
     from urlparse import urlparse
@@ -12,3 +13,13 @@
     if urlparse('s3://bucket/key?key=value').query != 'key=value':
         from urlparse import uses_query
         uses_query.append('s3')
+
+
+# Undo what Twisted's perspective broker adds to pickle register
+# to prevent bugs like Twisted#7989 while serializing requests
+import twisted.persisted.styles  # NOQA
+# Remove only entries with twisted serializers for non-twisted types.
+for k, v in frozenset(copyreg.dispatch_table.items()):
+    if not getattr(k, '__module__', '').startswith('twisted') \
+            and getattr(v, '__module__', '').startswith('twisted'):
+        copyreg.dispatch_table.pop(k)

tests/test_squeues.py

@@ -31,6 +31,8 @@ def test_serialize(self):
         self.assertEqual(q.pop(), {'a': 'dict'})
 
     def test_nonserializable_object(self):
+        # Trigger Twisted bug #7989
+        import twisted.persisted.styles  # NOQA
         q = self.queue()
         self.assertRaises(ValueError, q.push, lambda x: x)
 
@@ -109,6 +111,8 @@ def test_serialize(self):
         self.assertEqual(q.pop(), 'a')
 
     def test_nonserializable_object(self):
+        # Trigger Twisted bug #7989
+        import twisted.persisted.styles  # NOQA
         q = self.queue()
         self.assertRaises(ValueError, q.push, lambda x: x)