PY3 nicer log messages in FilesPipeline

Author: kmike

scrapy/pipelines/files.py

@@ -26,7 +26,7 @@
 from scrapy.http import Request
 from scrapy.utils.misc import md5sum
 from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.python import to_bytes
+from scrapy.utils.python import to_bytes, to_native_str
 
 logger = logging.getLogger(__name__)
 
@@ -199,7 +199,7 @@ def _onsuccess(result):
             if age_days > self.EXPIRES:
                 return  # returning None force download
 
-            referer = request.headers.get('Referer')
+            referer = _get_referer(request)
             logger.debug(
                 'File (uptodate): Downloaded %(medianame)s from %(request)s '
                 'referred in <%(referer)s>',
@@ -225,7 +225,7 @@ def _onsuccess(result):
 
     def media_failed(self, failure, request, info):
         if not isinstance(failure.value, IgnoreRequest):
-            referer = request.headers.get('Referer')
+            referer = _get_referer(request)
             logger.warning(
                 'File (unknown-error): Error downloading %(medianame)s from '
                 '%(request)s referred in <%(referer)s>: %(exception)s',
@@ -237,7 +237,7 @@ def media_failed(self, failure, request, info):
         raise FileException
 
     def media_downloaded(self, response, request, info):
-        referer = request.headers.get('Referer')
+        referer = _get_referer(request)
 
         if response.status != 200:
             logger.warning(
@@ -339,3 +339,11 @@ def _warn():
     def file_key(self, url):
         return self.file_path(url)
     file_key._base = True
+
+
+def _get_referer(request):
+    """ Return Referer HTTP header suitable for logging """
+    referrer = request.headers.get('Referer')
+    if referrer is None:
+        return referrer
+    return to_native_str(referrer, errors='replace')