17
17
class Slot (object ):
18
18
"""Downloader slot"""
19
19
20
- def __init__ (self , concurrency , delay , settings ):
20
+ def __init__ (self , concurrency , delay , randomize_delay ):
21
21
self .concurrency = concurrency
22
22
self .delay = delay
23
- self .randomize_delay = settings .getbool ('RANDOMIZE_DOWNLOAD_DELAY' )
23
+ self .randomize_delay = randomize_delay
24
+
24
25
self .active = set ()
25
26
self .queue = deque ()
26
27
self .transferring = set ()
@@ -66,6 +67,7 @@ def __init__(self, crawler):
66
67
self .total_concurrency = self .settings .getint ('CONCURRENT_REQUESTS' )
67
68
self .domain_concurrency = self .settings .getint ('CONCURRENT_REQUESTS_PER_DOMAIN' )
68
69
self .ip_concurrency = self .settings .getint ('CONCURRENT_REQUESTS_PER_IP' )
70
+ self .randomize_delay = self .settings .getbool ('RANDOMIZE_DOWNLOAD_DELAY' )
69
71
self .middleware = DownloaderMiddlewareManager .from_crawler (crawler )
70
72
self ._slot_gc_loop = task .LoopingCall (self ._slot_gc )
71
73
self ._slot_gc_loop .start (60 )
@@ -87,7 +89,7 @@ def _get_slot(self, request, spider):
87
89
if key not in self .slots :
88
90
conc = self .ip_concurrency if self .ip_concurrency else self .domain_concurrency
89
91
conc , delay = _get_concurrency_delay (conc , spider , self .settings )
90
- self .slots [key ] = Slot (conc , delay , self .settings )
92
+ self .slots [key ] = Slot (conc , delay , self .randomize_delay )
91
93
92
94
return key , self .slots [key ]
93
95
0 commit comments