Python 3 compatible syntax: print, except, raise, octal numbers; removed Python 2.2 boolean compatibility code in xlib/pydispatch/dispatcher.py

Author: alexanderlukanin13

extras/scrapy-ws.py

@@ -29,9 +29,9 @@ def get_commands():
 
 def cmd_help(args, opts):
     """help - list available commands"""
-    print "Available commands:"
+    print("Available commands:")
     for _, func in sorted(get_commands().items()):
-        print "  ", func.__doc__
+        print("  ", func.__doc__)
 
 def cmd_stop(args, opts):
     """stop <spider> - stop a running spider"""
@@ -40,29 +40,29 @@ def cmd_stop(args, opts):
 def cmd_list_running(args, opts):
     """list-running - list running spiders"""
     for x in json_get(opts, 'crawler/engine/open_spiders'):
-        print x
+        print(x)
 
 def cmd_list_available(args, opts):
     """list-available - list name of available spiders"""
     for x in jsonrpc_call(opts, 'crawler/spiders', 'list'):
-        print x
+        print(x)
 
 def cmd_list_resources(args, opts):
     """list-resources - list available web service resources"""
     for x in json_get(opts, '')['resources']:
-        print x
+        print(x)
 
 def cmd_get_spider_stats(args, opts):
     """get-spider-stats <spider> - get stats of a running spider"""
     stats = jsonrpc_call(opts, 'stats', 'get_stats', args[0])
     for name, value in stats.items():
-        print "%-40s %s" % (name, value)
+        print("%-40s %s" % (name, value))
 
 def cmd_get_global_stats(args, opts):
     """get-global-stats - get global stats"""
     stats = jsonrpc_call(opts, 'stats', 'get_stats')
     for name, value in stats.items():
-        print "%-40s %s" % (name, value)
+        print("%-40s %s" % (name, value))
 
 def get_wsurl(opts, path):
     return urljoin("http://%s:%s/"% (opts.host, opts.port), path)
@@ -101,12 +101,12 @@ def main():
     try:
         cmd(args, opts)
     except IndexError:
-        print cmd.__doc__
-    except JsonRpcError, e:
-        print str(e)
+        print(cmd.__doc__)
+    except JsonRpcError as e:
+        print(str(e))
         if e.data:
-            print "Server Traceback below:"
-            print e.data
+            print("Server Traceback below:")
+            print(e.data)
 
 
 if __name__ == '__main__':

scrapy/__init__.py

@@ -8,7 +8,7 @@
 import sys, os, warnings
 
 if sys.version_info < (2, 6):
-    print "Scrapy %s requires Python 2.6 or above" % __version__
+    print("Scrapy %s requires Python 2.6 or above" % __version__)
     sys.exit(1)
 
 # ignore noisy twisted deprecation warnings

scrapy/cmdline.py

@@ -59,34 +59,34 @@ def _pop_command_name(argv):
 
 def _print_header(settings, inproject):
     if inproject:
-        print "Scrapy %s - project: %s\n" % (scrapy.__version__, \
-            settings['BOT_NAME'])
+        print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
+            settings['BOT_NAME']))
     else:
-        print "Scrapy %s - no active project\n" % scrapy.__version__
+        print("Scrapy %s - no active project\n" % scrapy.__version__)
 
 def _print_commands(settings, inproject):
     _print_header(settings, inproject)
-    print "Usage:"
-    print "  scrapy <command> [options] [args]\n"
-    print "Available commands:"
+    print("Usage:")
+    print("  scrapy <command> [options] [args]\n")
+    print("Available commands:")
     cmds = _get_commands_dict(settings, inproject)
     for cmdname, cmdclass in sorted(cmds.iteritems()):
-        print "  %-13s %s" % (cmdname, cmdclass.short_desc())
+        print("  %-13s %s" % (cmdname, cmdclass.short_desc()))
     if not inproject:
-        print
-        print "  [ more ]      More commands available when run from project directory"
-    print
-    print 'Use "scrapy <command> -h" to see more info about a command'
+        print()
+        print("  [ more ]      More commands available when run from project directory")
+    print()
+    print('Use "scrapy <command> -h" to see more info about a command')
 
 def _print_unknown_command(settings, cmdname, inproject):
     _print_header(settings, inproject)
-    print "Unknown command: %s\n" % cmdname
-    print 'Use "scrapy" to see available commands'
+    print("Unknown command: %s\n" % cmdname)
+    print('Use "scrapy" to see available commands')
 
 def _run_print_help(parser, func, *a, **kw):
     try:
         func(*a, **kw)
-    except UsageError, e:
+    except UsageError as e:
         if str(e):
             parser.error(str(e))
         if e.print_help:

scrapy/commands/check.py

@@ -64,9 +64,9 @@ def run(self, args, opts):
         # start checks
         if opts.list:
             for spider, methods in sorted(contract_reqs.iteritems()):
-                print spider
+                print(spider)
                 for method in sorted(methods):
-                    print '  * %s' % method
+                    print('  * %s' % method)
         else:
             self.crawler_process.start()
             self.results.printErrors()

scrapy/commands/deploy.py

@@ -72,7 +72,7 @@ def run(self, args, opts):
 
         if opts.list_targets:
             for name, target in _get_targets().items():
-                print "%-20s %s" % (name, target['url'])
+                print("%-20s %s" % (name, target['url']))
             return
 
         if opts.list_projects:
@@ -81,7 +81,7 @@ def run(self, args, opts):
             _add_auth_header(req, target)
             f = urllib2.urlopen(req)
             projects = json.loads(f.read())['projects']
-            print os.linesep.join(projects)
+            print(os.linesep.join(projects))
             return
 
         tmpdir = None
@@ -208,12 +208,12 @@ def _http_post(request):
     try:
         f = urllib2.urlopen(request)
         _log("Server response (%s):" % f.code)
-        print f.read()
+        print(f.read())
         return True
-    except urllib2.HTTPError, e:
+    except urllib2.HTTPError as e:
         _log("Deploy failed (%s):" % e.code)
-        print e.read()
-    except urllib2.URLError, e:
+        print(e.read())
+    except urllib2.URLError as e:
         _log("Deploy failed: %s" % e)
 
 def _build_egg():

scrapy/commands/fetch.py

@@ -30,15 +30,15 @@ def add_options(self, parser):
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
             for value in values:
-                print '%s %s: %s' % (prefix, key, value)
+                print('%s %s: %s' % (prefix, key, value))
 
     def _print_response(self, response, opts):
         if opts.headers:
             self._print_headers(response.request.headers, '>')
-            print '>'
+            print('>')
             self._print_headers(response.headers, '<')
         else:
-            print response.body
+            print(response.body)
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):

scrapy/commands/genspider.py

@@ -49,7 +49,7 @@ def run(self, args, opts):
         if opts.dump:
             template_file = self._find_template(opts.dump)
             if template_file:
-                print open(template_file, 'r').read()
+                print(open(template_file, 'r').read())
             return
         if len(args) != 2:
             raise UsageError()
@@ -58,7 +58,7 @@ def run(self, args, opts):
         module = sanitize_module_name(name)
 
         if self.settings.get('BOT_NAME') == module:
-            print "Cannot create a spider with the same name as your project"
+            print("Cannot create a spider with the same name as your project")
             return
 
         try:
@@ -69,8 +69,8 @@ def run(self, args, opts):
         else:
             # if spider already exists and not --force then halt
             if not opts.force:
-                print "Spider %r already exists in module:" % name
-                print "  %s" % spider.__module__
+                print("Spider %r already exists in module:" % name)
+                print("  %s" % spider.__module__)
                 return
         template_file = self._find_template(opts.template)
         if template_file:
@@ -94,22 +94,22 @@ def _genspider(self, module, name, domain, template_name, template_file):
         spider_file = "%s.py" % join(spiders_dir, module)
         shutil.copyfile(template_file, spider_file)
         render_templatefile(spider_file, **tvars)
-        print "Created spider %r using template %r in module:" % (name, \
-            template_name)
-        print "  %s.%s" % (spiders_module.__name__, module)
+        print("Created spider %r using template %r in module:" % (name, \
+            template_name))
+        print("  %s.%s" % (spiders_module.__name__, module))
 
     def _find_template(self, template):
         template_file = join(self.templates_dir, '%s.tmpl' % template)
         if exists(template_file):
             return template_file
-        print "Unable to find template: %s\n" % template
-        print 'Use "scrapy genspider --list" to see all available templates.'
+        print("Unable to find template: %s\n" % template)
+        print('Use "scrapy genspider --list" to see all available templates.')
 
     def _list_templates(self):
-        print "Available templates:"
+        print("Available templates:")
         for filename in sorted(os.listdir(self.templates_dir)):
             if filename.endswith('.tmpl'):
-                print "  %s" % splitext(filename)[0]
+                print("  %s" % splitext(filename)[0])
 
     @property
     def templates_dir(self):

scrapy/commands/list.py

@@ -11,4 +11,4 @@ def short_desc(self):
     def run(self, args, opts):
         crawler = self.crawler_process.create_crawler()
         for s in crawler.spiders.list():
-            print s
+            print(s)

scrapy/commands/parse.py

@@ -68,7 +68,7 @@ def print_items(self, lvl=None, colour=True):
         else:
             items = self.items.get(lvl, [])
 
-        print "# Scraped Items ", "-"*60
+        print("# Scraped Items ", "-"*60)
         display.pprint([dict(x) for x in items], colorize=colour)
 
     def print_requests(self, lvl=None, colour=True):
@@ -81,21 +81,21 @@ def print_requests(self, lvl=None, colour=True):
         else:
             requests = self.requests.get(lvl, [])
 
-        print "# Requests ", "-"*65
+        print("# Requests ", "-"*65)
         display.pprint(requests, colorize=colour)
 
     def print_results(self, opts):
         colour = not opts.nocolour
 
         if opts.verbose:
             for level in xrange(1, self.max_level+1):
-                print '\n>>> DEPTH LEVEL: %s <<<' % level
+                print('\n>>> DEPTH LEVEL: %s <<<' % level)
                 if not opts.noitems:
                     self.print_items(level, colour)
                 if not opts.nolinks:
                     self.print_requests(level, colour)
         else:
-            print '\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level
+            print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level)
             if not opts.noitems:
                 self.print_items(colour=colour)
             if not opts.nolinks:

scrapy/commands/runspider.py

@@ -67,7 +67,7 @@ def run(self, args, opts):
             raise UsageError("File not found: %s\n" % filename)
         try:
             module = _import_file(filename)
-        except (ImportError, ValueError), e:
+        except (ImportError, ValueError) as e:
             raise UsageError("Unable to load %r: %s\n" % (filename, e))
         spclasses = list(iter_spider_classes(module))
         if not spclasses:

scrapy/commands/settings.py

@@ -27,12 +27,12 @@ def add_options(self, parser):
     def run(self, args, opts):
         settings = self.crawler_process.settings
         if opts.get:
-            print settings.get(opts.get)
+            print(settings.get(opts.get))
         elif opts.getbool:
-            print settings.getbool(opts.getbool)
+            print(settings.getbool(opts.getbool))
         elif opts.getint:
-            print settings.getint(opts.getint)
+            print(settings.getint(opts.getint))
         elif opts.getfloat:
-            print settings.getfloat(opts.getfloat)
+            print(settings.getfloat(opts.getfloat))
         elif opts.getlist:
-            print settings.getlist(opts.getlist)
+            print(settings.getlist(opts.getlist))

scrapy/commands/startproject.py

@@ -36,11 +36,11 @@ def run(self, args, opts):
             raise UsageError()
         project_name = args[0]
         if not re.search(r'^[_a-zA-Z]\w*$', project_name):
-            print 'Error: Project names must begin with a letter and contain only\n' \
-                'letters, numbers and underscores'
+            print('Error: Project names must begin with a letter and contain only\n' \
+                'letters, numbers and underscores')
             sys.exit(1)
         elif exists(project_name):
-            print "Error: directory %r already exists" % project_name
+            print("Error: directory %r already exists" % project_name)
             sys.exit(1)
 
         moduletpl = join(TEMPLATES_PATH, 'module')

scrapy/commands/version.py

@@ -25,11 +25,11 @@ def run(self, args, opts):
             import lxml.etree
             lxml_version = ".".join(map(str, lxml.etree.LXML_VERSION))
             libxml2_version = ".".join(map(str, lxml.etree.LIBXML_VERSION))
-            print "Scrapy  : %s" % scrapy.__version__
-            print "lxml    : %s" % lxml_version
-            print "libxml2 : %s" % libxml2_version
-            print "Twisted : %s" % twisted.version.short()
-            print "Python  : %s" % sys.version.replace("\n", "- ")
-            print "Platform: %s" % platform.platform()
+            print("Scrapy  : %s" % scrapy.__version__)
+            print("lxml    : %s" % lxml_version)
+            print("libxml2 : %s" % libxml2_version)
+            print("Twisted : %s" % twisted.version.short())
+            print("Python  : %s" % sys.version.replace("\n", "- "))
+            print("Platform: %s" % platform.platform())
         else:
-            print "Scrapy %s" % scrapy.__version__
+            print("Scrapy %s" % scrapy.__version__)

scrapy/contrib/djangoitem.py

@@ -51,12 +51,12 @@ def _get_errors(self, exclude=None):
 
         try:
             self.instance.clean_fields(exclude=exclude)
-        except ValidationError, e:
+        except ValidationError as e:
             self._errors = e.update_error_dict(self._errors)
 
         try:
             self.instance.clean()
-        except ValidationError, e:
+        except ValidationError as e:
             self._errors = e.update_error_dict(self._errors)
 
         # uniqueness is not checked, because it is faster to check it when

scrapy/contrib/loader/__init__.py

@@ -81,7 +81,7 @@ def get_output_value(self, field_name):
         proc = wrap_loader_context(proc, self.context)
         try:
             return proc(self._values[field_name])
-        except Exception, e:
+        except Exception as e:
             raise ValueError("Error with output processor: field=%r value=%r error='%s: %s'" % \
                 (field_name, self._values[field_name], type(e).__name__, str(e)))
 

scrapy/core/downloader/handlers/__init__.py

@@ -18,7 +18,7 @@ def __init__(self, crawler):
             cls = load_object(clspath)
             try:
                 dh = cls(crawler.settings)
-            except NotConfigured, ex:
+            except NotConfigured as ex:
                 self._notconfigured[scheme] = str(ex)
             else:
                 self._handlers[scheme] = dh

scrapy/core/downloader/handlers/s3.py

@@ -41,7 +41,7 @@ def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None,
 
         try:
             self.conn = _S3Connection(aws_access_key_id, aws_secret_access_key)
-        except Exception, ex:
+        except Exception as ex:
             raise NotConfigured(str(ex))
         self._download_http = httpdownloadhandler(settings).download_request
 

scrapy/core/engine.py

@@ -111,7 +111,7 @@ def _next_request(self, spider):
                 request = slot.start_requests.next()
             except StopIteration:
                 slot.start_requests = None
-            except Exception, exc:
+            except Exception as exc:
                 log.err(None, 'Obtaining request from start requests', \
                         spider=spider)
             else: