Set CONCURRENT_SPIDERS=1 in Scrapyd to force one spider per process

Author: pablohoffman

scrapyd/environ.py

@@ -30,6 +30,7 @@ def get_environment(self, message, slot):
         dbpath = os.path.join(self.dbs_dir, '%s.db' % project)
         env['SCRAPY_SQLITE_DB'] = dbpath
         env['SCRAPY_LOG_FILE'] = self._get_log_file(message)
+        env['SCRAPY_CONCURRENT_SPIDERS'] = '1' # scrapyd runs one spider per process
         return env
 
     def _get_log_file(self, message):

scrapyd/tests/test_environ.py

@@ -29,6 +29,7 @@ def test_get_environment_with_eggfile(self):
         self.assertEqual(env['SCRAPY_SLOT'], '3')
         self.assertEqual(env['SCRAPY_SPIDER'], 'myspider')
         self.assertEqual(env['SCRAPY_JOB'], 'ID')
+        self.assertEqual(env['SCRAPY_CONCURRENT_SPIDERS'], '1')
         self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db'))
         self.assert_(env['SCRAPY_LOG_FILE'].endswith('/mybot/myspider/ID.log'))
         self.failIf('SCRAPY_SETTINGS_MODULE' in env)