Merge pull request scrapy#1286 from scrapy/configure_logging

configure_logging: change the meaning of settings=None

Author: curita

docs/topics/logging.rst

@@ -194,42 +194,33 @@ scrapy.utils.log module
 .. module:: scrapy.utils.log
    :synopsis: Logging utils
 
-.. function:: configure_logging(settings=None)
+.. autofunction:: configure_logging
 
-    This function initializes logging defaults for Scrapy.
-
-    It's automatically called when using Scrapy commands, but needs to be
-    called explicitely when running custom scripts. In that case, its usage is
-    not required but it's recommended.
-
-    This function does:
-      - Route warnings and Twisted logging through Python standard logging
-      - Set a filter on Scrapy logger for formatting Twisted failures
-      - Assign DEBUG and ERROR levels to Scrapy and Twisted loggers
-        respectively
-
-    If `settings` is not ``None``, it will also create a root handler based on
-    the settings listed in :ref:`topics-logging-settings`.
+    ``configure_logging`` is automatically called when using Scrapy commands,
+    but needs to be called explicitly when running custom scripts. In that
+    case, its usage is not required but it's recommended.
 
     If you plan on configuring the handlers yourself is still recommended you
-    call this function, keeping `settings` as ``None``. Bear in mind there
-    won't be any log output set by default in that case.
+    call this function, passing `install_root_handler=False`. Bear in mind
+    there won't be any log output set by default in that case.
 
     To get you started on manually configuring logging's output, you can use
-    `logging.basicConfig()`_ to set a basic root handler. This is an example on
-    how to redirect ``INFO`` or higher messages to a file::
+    `logging.basicConfig()`_ to set a basic root handler. This is an example
+    on how to redirect ``INFO`` or higher messages to a file::
 
         import logging
         from scrapy.utils.log import configure_logging
 
-        configure_logging()  # Note we aren't providing settings in this case
-        logging.basicConfig(filename='log.txt', format='%(levelname)s: %(message)s', level=logging.INFO)
+        configure_logging(install_root_handler=False)
+        logging.basicConfig(
+            filename='log.txt',
+            format='%(levelname)s: %(message)s',
+            level=logging.INFO
+        )
 
     Refer to :ref:`run-from-script` for more details about using Scrapy this
     way.
 
-    :param settings: settings used to create and configure a handler for the
-        root logger.
-    :type settings: :class:`~scrapy.settings.Settings` object or ``None``
-
 .. _logging.basicConfig(): https://docs.python.org/2/library/logging.html#logging.basicConfig
+
+

docs/topics/practices.rst

@@ -148,7 +148,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
         # Your second spider definition
         ...
 
-    configure_logging({})
+    configure_logging()
     runner = CrawlerRunner()
     runner.crawl(MySpider1)
     runner.crawl(MySpider2)
@@ -173,7 +173,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
         # Your second spider definition
         ...
 
-    configure_logging({})
+    configure_logging()
     runner = CrawlerRunner()
 
     @defer.inlineCallbacks

scrapy/utils/log.py

@@ -56,14 +56,29 @@ def filter(self, record):
 }
 
 
-def configure_logging(settings=None):
-    """Initialize and configure default loggers
+def configure_logging(settings=None, install_root_handler=True):
+    """
+    Initialize logging defaults for Scrapy.
+
+    :param settings: settings used to create and configure a handler for the
+        root logger (default: None).
+    :type settings: dict, :class:`~scrapy.settings.Settings` object or ``None``
+
+    :param install_root_handler: whether to install root logging handler
+        (default: True)
+    :type install_root_handler: bool
 
     This function does:
-      - Route warnings and twisted logging through Python standard logging
-      - Set FailureFormatter filter on Scrapy logger
-      - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
-      - Create a handler for the root logger according to given settings
+
+    - Route warnings and twisted logging through Python standard logging
+    - Assign DEBUG and ERROR level to Scrapy and Twisted loggers respectively
+    - Route stdout to log if LOG_STDOUT setting is True
+
+    When ``install_root_handler`` is True (default), this function also
+    creates a handler for the root logger according to given settings
+    (see :ref:`topics-logging-settings`). You can override default options
+    using ``settings`` argument. When ``settings`` is empty or None, defaults
+    are used.
     """
     if not sys.warnoptions:
         # Route warnings through python logging
@@ -74,35 +89,39 @@ def configure_logging(settings=None):
 
     dictConfig(DEFAULT_LOGGING)
 
-    if isinstance(settings, dict):
+    if isinstance(settings, dict) or settings is None:
         settings = Settings(settings)
 
-    if settings:
-        logging.root.setLevel(logging.NOTSET)
+    if settings.getbool('LOG_STDOUT'):
+        sys.stdout = StreamLogger(logging.getLogger('stdout'))
 
-        if settings.getbool('LOG_STDOUT'):
-            sys.stdout = StreamLogger(logging.getLogger('stdout'))
-
-        # Set up the default log handler
-        filename = settings.get('LOG_FILE')
-        if filename:
-            encoding = settings.get('LOG_ENCODING')
-            handler = logging.FileHandler(filename, encoding=encoding)
-        elif settings.getbool('LOG_ENABLED'):
-            handler = logging.StreamHandler()
-        else:
-            handler = logging.NullHandler()
-
-        formatter = logging.Formatter(
-            fmt=settings.get('LOG_FORMAT'),
-            datefmt=settings.get('LOG_DATEFORMAT')
-        )
-        handler.setFormatter(formatter)
-        handler.setLevel(settings.get('LOG_LEVEL'))
-        handler.addFilter(TopLevelFormatter(['scrapy']))
+    if install_root_handler:
+        logging.root.setLevel(logging.NOTSET)
+        handler = _get_handler(settings)
         logging.root.addHandler(handler)
 
 
+def _get_handler(settings):
+    """ Return a log handler object according to settings """
+    filename = settings.get('LOG_FILE')
+    if filename:
+        encoding = settings.get('LOG_ENCODING')
+        handler = logging.FileHandler(filename, encoding=encoding)
+    elif settings.getbool('LOG_ENABLED'):
+        handler = logging.StreamHandler()
+    else:
+        handler = logging.NullHandler()
+
+    formatter = logging.Formatter(
+        fmt=settings.get('LOG_FORMAT'),
+        datefmt=settings.get('LOG_DATEFORMAT')
+    )
+    handler.setFormatter(formatter)
+    handler.setLevel(settings.get('LOG_LEVEL'))
+    handler.addFilter(TopLevelFormatter(['scrapy']))
+    return handler
+
+
 def log_scrapy_info(settings):
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})