Merge pull request scrapy#1388 from scrapy/dupefilter-persist

Dupefilter persistence

Author: dangra

scrapy/dupefilters.py

@@ -36,6 +36,7 @@ def __init__(self, path=None, debug=False):
         self.logger = logging.getLogger(__name__)
         if path:
             self.file = open(os.path.join(path, 'requests.seen'), 'a+')
+            self.file.seek(0)
             self.fingerprints.update(x.rstrip() for x in self.file)
 
     @classmethod

tests/test_dupefilters.py

@@ -1,5 +1,7 @@
 import hashlib
+import tempfile
 import unittest
+import shutil
 
 from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
@@ -24,6 +26,27 @@ def test_filter(self):
 
         dupefilter.close('finished')
 
+    def test_dupefilter_path(self):
+        r1 = Request('http://scrapytest.org/1')
+        r2 = Request('http://scrapytest.org/2')
+
+        path = tempfile.mkdtemp()
+        try:
+            df = RFPDupeFilter(path)
+            df.open()
+            assert not df.request_seen(r1)
+            assert df.request_seen(r1)
+            df.close('finished')
+
+            df2 = RFPDupeFilter(path)
+            df2.open()
+            assert df2.request_seen(r1)
+            assert not df2.request_seen(r2)
+            assert df2.request_seen(r2)
+            df2.close('finished')
+        finally:
+            shutil.rmtree(path)
+
     def test_request_fingerprint(self):
         """Test if customization of request_fingerprint method will change
         output of request_seen.