Merge pull request scrapy#445 from darkrho/import-module

Use `importlib.import_module` instead of `__import__`

Author: dangra

scrapy/commands/genspider.py

@@ -2,6 +2,8 @@
 import os
 import shutil
 import string
+
+from importlib import import_module
 from os.path import join, dirname, abspath, exists, splitext
 
 import scrapy
@@ -90,7 +92,7 @@ def _genspider(self, module, name, domain, template_name, template_file):
             'classname': '%sSpider' % ''.join([s.capitalize() \
                 for s in module.split('_')])
         }
-        spiders_module = __import__(self.settings['NEWSPIDER_MODULE'], {}, {}, [''])
+        spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
         spiders_dir = abspath(dirname(spiders_module.__file__))
         spider_file = "%s.py" % join(spiders_dir, module)
         shutil.copyfile(template_file, spider_file)

scrapy/commands/runspider.py

@@ -1,5 +1,6 @@
 import sys
 import os
+from importlib import import_module
 
 from scrapy.utils.spider import iter_spider_classes
 from scrapy.command import ScrapyCommand
@@ -15,7 +16,7 @@ def _import_file(filepath):
     if dirname:
         sys.path = [dirname] + sys.path
     try:
-        module = __import__(fname, {}, {}, [''])
+        module = import_module(fname)
     finally:
         if dirname:
             sys.path.pop(0)

scrapy/contrib/httpcache.py

@@ -1,6 +1,7 @@
 from __future__ import print_function
 import os
 import cPickle as pickle
+from importlib import import_module
 from time import time
 from weakref import WeakKeyDictionary
 from email.utils import mktime_tz, parsedate_tz
@@ -164,7 +165,7 @@ class DbmCacheStorage(object):
     def __init__(self, settings):
         self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
         self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
-        self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'], {}, {}, [''])
+        self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE'])
         self.db = None
 
     def open_spider(self, spider):

scrapy/contrib/memusage.py

@@ -6,6 +6,7 @@
 
 import socket
 from pprint import pformat
+from importlib import import_module
 
 from twisted.internet import task
 
@@ -20,7 +21,8 @@ def __init__(self, crawler):
         if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
             raise NotConfigured
         try:
-            self.resource = __import__('resource')
+            # stdlib's resource module is only availabe on unix platforms.
+            self.resource = import_module('resource')
         except ImportError:
             raise NotConfigured
 

scrapy/settings/default_settings.py

@@ -15,6 +15,7 @@
 
 import os
 import sys
+from importlib import import_module
 from os.path import join, abspath, dirname
 
 BOT_NAME = 'scrapybot'
@@ -229,7 +230,7 @@
 
 URLLENGTH_LIMIT = 2083
 
-USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % __import__('scrapy').__version__
+USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__
 
 TELNETCONSOLE_ENABLED = 1
 TELNETCONSOLE_PORT = [6023, 6073]

scrapy/tests/test_dependencies.py

@@ -1,9 +1,10 @@
+from importlib import import_module
 from twisted.trial import unittest
 
 class ScrapyUtilsTest(unittest.TestCase):
     def test_required_openssl_version(self):
         try:
-            module = __import__('OpenSSL', {}, {}, [''])
+            module = import_module('OpenSSL')
         except ImportError as ex:
             raise unittest.SkipTest("OpenSSL is not available")
 

scrapy/utils/misc.py

@@ -2,6 +2,8 @@
 
 import re
 import hashlib
+
+from importlib import import_module
 from pkgutil import iter_modules
 
 from w3lib.html import remove_entities
@@ -35,7 +37,7 @@ def load_object(path):
 
     module, name = path[:dot], path[dot+1:]
     try:
-        mod = __import__(module, {}, {}, [''])
+        mod = import_module(module)
     except ImportError as e:
         raise ImportError("Error loading object '%s': %s" % (path, e))
 
@@ -55,15 +57,15 @@ def walk_modules(path, load=False):
     """
 
     mods = []
-    mod = __import__(path, {}, {}, [''])
+    mod = import_module(path)
     mods.append(mod)
     if hasattr(mod, '__path__'):
         for _, subpath, ispkg in iter_modules(mod.__path__):
             fullpath = path + '.' + subpath
             if ispkg:
                 mods += walk_modules(fullpath)
             else:
-                submod = __import__(fullpath, {}, {}, [''])
+                submod = import_module(fullpath)
                 mods.append(submod)
     return mods
 

scrapy/utils/project.py

@@ -1,8 +1,10 @@
 import os
-from os.path import join, dirname, abspath, isabs, exists
 import cPickle as pickle
 import warnings
 
+from importlib import import_module
+from os.path import join, dirname, abspath, isabs, exists
+
 from scrapy.utils.conf import closest_scrapy_cfg, get_config, init_env
 from scrapy.settings import CrawlerSettings
 from scrapy.exceptions import NotConfigured
@@ -14,7 +16,7 @@ def inside_project():
     scrapy_module = os.environ.get('SCRAPY_SETTINGS_MODULE')
     if scrapy_module is not None:
         try:
-            __import__(scrapy_module)
+            import_module(scrapy_module)
         except ImportError as exc:
             warnings.warn("Cannot import scrapy settings module %s: %s" % (scrapy_module, exc))
         else:
@@ -53,7 +55,7 @@ def get_project_settings():
         init_env(project)
     settings_module_path = os.environ.get(ENVVAR)
     if settings_module_path:
-        settings_module = __import__(settings_module_path, {}, {}, [''])
+        settings_module = import_module(settings_module_path)
     else:
         settings_module = None
     settings = CrawlerSettings(settings_module)

scrapy/utils/test.py

@@ -4,6 +4,7 @@
 
 import os
 
+from importlib import import_module
 from twisted.trial.unittest import SkipTest
 
 
@@ -39,7 +40,7 @@ class SettingsModuleMock(object):
 def get_pythonpath():
     """Return a PYTHONPATH suitable to use in processes so that they find this
     installation of Scrapy"""
-    scrapy_path = __import__('scrapy').__path__[0]
+    scrapy_path = import_module('scrapy').__path__[0]
     return os.path.dirname(scrapy_path) + os.pathsep + os.environ.get('PYTHONPATH', '')
 
 def get_testenv():