Skip to content

Commit 8da9836

Browse files
committed
don't access settings from downloader.Slot
1 parent 38e5bfb commit 8da9836

File tree

1 file changed

+5
-3
lines changed

1 file changed

+5
-3
lines changed

scrapy/core/downloader/__init__.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,11 @@
1717
class Slot(object):
1818
"""Downloader slot"""
1919

20-
def __init__(self, concurrency, delay, settings):
20+
def __init__(self, concurrency, delay, randomize_delay):
2121
self.concurrency = concurrency
2222
self.delay = delay
23-
self.randomize_delay = settings.getbool('RANDOMIZE_DOWNLOAD_DELAY')
23+
self.randomize_delay = randomize_delay
24+
2425
self.active = set()
2526
self.queue = deque()
2627
self.transferring = set()
@@ -66,6 +67,7 @@ def __init__(self, crawler):
6667
self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS')
6768
self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
6869
self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP')
70+
self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY')
6971
self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
7072
self._slot_gc_loop = task.LoopingCall(self._slot_gc)
7173
self._slot_gc_loop.start(60)
@@ -87,7 +89,7 @@ def _get_slot(self, request, spider):
8789
if key not in self.slots:
8890
conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
8991
conc, delay = _get_concurrency_delay(conc, spider, self.settings)
90-
self.slots[key] = Slot(conc, delay, self.settings)
92+
self.slots[key] = Slot(conc, delay, self.randomize_delay)
9193

9294
return key, self.slots[key]
9395

0 commit comments

Comments
 (0)