Merge pull request scrapy#431 from alexanderlukanin13/syntax

Python 3 compatible syntax: print, except, raise, octal numbers; removed...

Author: kmike

extras/qps-bench-server.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import print_function
 from time import time
 from collections import deque
 from twisted.web.server import Site, NOT_DONE_YET

extras/scrapy-ws.py

@@ -11,6 +11,7 @@
 
 """
 
+from __future__ import print_function
 import sys, optparse, urllib, json
 from urlparse import urljoin
 
@@ -29,9 +30,9 @@ def get_commands():
 
 def cmd_help(args, opts):
     """help - list available commands"""
-    print "Available commands:"
+    print("Available commands:")
     for _, func in sorted(get_commands().items()):
-        print "  ", func.__doc__
+        print("  ", func.__doc__)
 
 def cmd_stop(args, opts):
     """stop <spider> - stop a running spider"""
@@ -40,29 +41,29 @@ def cmd_stop(args, opts):
 def cmd_list_running(args, opts):
     """list-running - list running spiders"""
     for x in json_get(opts, 'crawler/engine/open_spiders'):
-        print x
+        print(x)
 
 def cmd_list_available(args, opts):
     """list-available - list name of available spiders"""
     for x in jsonrpc_call(opts, 'crawler/spiders', 'list'):
-        print x
+        print(x)
 
 def cmd_list_resources(args, opts):
     """list-resources - list available web service resources"""
     for x in json_get(opts, '')['resources']:
-        print x
+        print(x)
 
 def cmd_get_spider_stats(args, opts):
     """get-spider-stats <spider> - get stats of a running spider"""
     stats = jsonrpc_call(opts, 'stats', 'get_stats', args[0])
     for name, value in stats.items():
-        print "%-40s %s" % (name, value)
+        print("%-40s %s" % (name, value))
 
 def cmd_get_global_stats(args, opts):
     """get-global-stats - get global stats"""
     stats = jsonrpc_call(opts, 'stats', 'get_stats')
     for name, value in stats.items():
-        print "%-40s %s" % (name, value)
+        print("%-40s %s" % (name, value))
 
 def get_wsurl(opts, path):
     return urljoin("http://%s:%s/"% (opts.host, opts.port), path)
@@ -101,12 +102,12 @@ def main():
     try:
         cmd(args, opts)
     except IndexError:
-        print cmd.__doc__
-    except JsonRpcError, e:
-        print str(e)
+        print(cmd.__doc__)
+    except JsonRpcError as e:
+        print(str(e))
         if e.data:
-            print "Server Traceback below:"
-            print e.data
+            print("Server Traceback below:")
+            print(e.data)
 
 
 if __name__ == '__main__':

scrapy/__init__.py

@@ -1,14 +1,15 @@
 """
 Scrapy - a screen scraping framework written in Python
 """
+from __future__ import print_function
 import pkgutil
 __version__ = pkgutil.get_data(__package__, 'VERSION').strip()
 version_info = tuple(__version__.split('.')[:3])
 
 import sys, os, warnings
 
 if sys.version_info < (2, 6):
-    print "Scrapy %s requires Python 2.6 or above" % __version__
+    print("Scrapy %s requires Python 2.6 or above" % __version__)
     sys.exit(1)
 
 # ignore noisy twisted deprecation warnings

scrapy/cmdline.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import sys
 import optparse
 import cProfile
@@ -59,34 +60,34 @@ def _pop_command_name(argv):
 
 def _print_header(settings, inproject):
     if inproject:
-        print "Scrapy %s - project: %s\n" % (scrapy.__version__, \
-            settings['BOT_NAME'])
+        print("Scrapy %s - project: %s\n" % (scrapy.__version__, \
+            settings['BOT_NAME']))
     else:
-        print "Scrapy %s - no active project\n" % scrapy.__version__
+        print("Scrapy %s - no active project\n" % scrapy.__version__)
 
 def _print_commands(settings, inproject):
     _print_header(settings, inproject)
-    print "Usage:"
-    print "  scrapy <command> [options] [args]\n"
-    print "Available commands:"
+    print("Usage:")
+    print("  scrapy <command> [options] [args]\n")
+    print("Available commands:")
     cmds = _get_commands_dict(settings, inproject)
     for cmdname, cmdclass in sorted(cmds.iteritems()):
-        print "  %-13s %s" % (cmdname, cmdclass.short_desc())
+        print("  %-13s %s" % (cmdname, cmdclass.short_desc()))
     if not inproject:
-        print
-        print "  [ more ]      More commands available when run from project directory"
-    print
-    print 'Use "scrapy <command> -h" to see more info about a command'
+        print()
+        print("  [ more ]      More commands available when run from project directory")
+    print()
+    print('Use "scrapy <command> -h" to see more info about a command')
 
 def _print_unknown_command(settings, cmdname, inproject):
     _print_header(settings, inproject)
-    print "Unknown command: %s\n" % cmdname
-    print 'Use "scrapy" to see available commands'
+    print("Unknown command: %s\n" % cmdname)
+    print('Use "scrapy" to see available commands')
 
 def _run_print_help(parser, func, *a, **kw):
     try:
         func(*a, **kw)
-    except UsageError, e:
+    except UsageError as e:
         if str(e):
             parser.error(str(e))
         if e.print_help:

scrapy/commands/check.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 from collections import defaultdict
 from functools import wraps
 from unittest import TextTestRunner
@@ -64,9 +65,9 @@ def run(self, args, opts):
         # start checks
         if opts.list:
             for spider, methods in sorted(contract_reqs.iteritems()):
-                print spider
+                print(spider)
                 for method in sorted(methods):
-                    print '  * %s' % method
+                    print('  * %s' % method)
         else:
             self.crawler_process.start()
             self.results.printErrors()

scrapy/commands/deploy.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import sys
 import os
 import glob
@@ -72,7 +73,7 @@ def run(self, args, opts):
 
         if opts.list_targets:
             for name, target in _get_targets().items():
-                print "%-20s %s" % (name, target['url'])
+                print("%-20s %s" % (name, target['url']))
             return
 
         if opts.list_projects:
@@ -81,7 +82,7 @@ def run(self, args, opts):
             _add_auth_header(req, target)
             f = urllib2.urlopen(req)
             projects = json.loads(f.read())['projects']
-            print os.linesep.join(projects)
+            print(os.linesep.join(projects))
             return
 
         tmpdir = None
@@ -208,12 +209,12 @@ def _http_post(request):
     try:
         f = urllib2.urlopen(request)
         _log("Server response (%s):" % f.code)
-        print f.read()
+        print(f.read())
         return True
-    except urllib2.HTTPError, e:
+    except urllib2.HTTPError as e:
         _log("Deploy failed (%s):" % e.code)
-        print e.read()
-    except urllib2.URLError, e:
+        print(e.read())
+    except urllib2.URLError as e:
         _log("Deploy failed: %s" % e)
 
 def _build_egg():

scrapy/commands/fetch.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 from w3lib.url import is_url
 
 from scrapy.command import ScrapyCommand
@@ -30,15 +31,15 @@ def add_options(self, parser):
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
             for value in values:
-                print '%s %s: %s' % (prefix, key, value)
+                print('%s %s: %s' % (prefix, key, value))
 
     def _print_response(self, response, opts):
         if opts.headers:
             self._print_headers(response.request.headers, '>')
-            print '>'
+            print('>')
             self._print_headers(response.headers, '<')
         else:
-            print response.body
+            print(response.body)
 
     def run(self, args, opts):
         if len(args) != 1 or not is_url(args[0]):

scrapy/commands/genspider.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import os
 import shutil
 import string
@@ -49,7 +50,7 @@ def run(self, args, opts):
         if opts.dump:
             template_file = self._find_template(opts.dump)
             if template_file:
-                print open(template_file, 'r').read()
+                print(open(template_file, 'r').read())
             return
         if len(args) != 2:
             raise UsageError()
@@ -58,7 +59,7 @@ def run(self, args, opts):
         module = sanitize_module_name(name)
 
         if self.settings.get('BOT_NAME') == module:
-            print "Cannot create a spider with the same name as your project"
+            print("Cannot create a spider with the same name as your project")
             return
 
         try:
@@ -69,8 +70,8 @@ def run(self, args, opts):
         else:
             # if spider already exists and not --force then halt
             if not opts.force:
-                print "Spider %r already exists in module:" % name
-                print "  %s" % spider.__module__
+                print("Spider %r already exists in module:" % name)
+                print("  %s" % spider.__module__)
                 return
         template_file = self._find_template(opts.template)
         if template_file:
@@ -94,22 +95,22 @@ def _genspider(self, module, name, domain, template_name, template_file):
         spider_file = "%s.py" % join(spiders_dir, module)
         shutil.copyfile(template_file, spider_file)
         render_templatefile(spider_file, **tvars)
-        print "Created spider %r using template %r in module:" % (name, \
-            template_name)
-        print "  %s.%s" % (spiders_module.__name__, module)
+        print("Created spider %r using template %r in module:" % (name, \
+            template_name))
+        print("  %s.%s" % (spiders_module.__name__, module))
 
     def _find_template(self, template):
         template_file = join(self.templates_dir, '%s.tmpl' % template)
         if exists(template_file):
             return template_file
-        print "Unable to find template: %s\n" % template
-        print 'Use "scrapy genspider --list" to see all available templates.'
+        print("Unable to find template: %s\n" % template)
+        print('Use "scrapy genspider --list" to see all available templates.')
 
     def _list_templates(self):
-        print "Available templates:"
+        print("Available templates:")
         for filename in sorted(os.listdir(self.templates_dir)):
             if filename.endswith('.tmpl'):
-                print "  %s" % splitext(filename)[0]
+                print("  %s" % splitext(filename)[0])
 
     @property
     def templates_dir(self):

scrapy/commands/list.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 from scrapy.command import ScrapyCommand
 
 class Command(ScrapyCommand):
@@ -11,4 +12,4 @@ def short_desc(self):
     def run(self, args, opts):
         crawler = self.crawler_process.create_crawler()
         for s in crawler.spiders.list():
-            print s
+            print(s)

scrapy/commands/parse.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 from w3lib.url import is_url
 from scrapy.command import ScrapyCommand
 from scrapy.http import Request
@@ -68,7 +69,7 @@ def print_items(self, lvl=None, colour=True):
         else:
             items = self.items.get(lvl, [])
 
-        print "# Scraped Items ", "-"*60
+        print("# Scraped Items ", "-"*60)
         display.pprint([dict(x) for x in items], colorize=colour)
 
     def print_requests(self, lvl=None, colour=True):
@@ -81,21 +82,21 @@ def print_requests(self, lvl=None, colour=True):
         else:
             requests = self.requests.get(lvl, [])
 
-        print "# Requests ", "-"*65
+        print("# Requests ", "-"*65)
         display.pprint(requests, colorize=colour)
 
     def print_results(self, opts):
         colour = not opts.nocolour
 
         if opts.verbose:
             for level in xrange(1, self.max_level+1):
-                print '\n>>> DEPTH LEVEL: %s <<<' % level
+                print('\n>>> DEPTH LEVEL: %s <<<' % level)
                 if not opts.noitems:
                     self.print_items(level, colour)
                 if not opts.nolinks:
                     self.print_requests(level, colour)
         else:
-            print '\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level
+            print('\n>>> STATUS DEPTH LEVEL %s <<<' % self.max_level)
             if not opts.noitems:
                 self.print_items(colour=colour)
             if not opts.nolinks:

scrapy/commands/runspider.py

@@ -67,7 +67,7 @@ def run(self, args, opts):
             raise UsageError("File not found: %s\n" % filename)
         try:
             module = _import_file(filename)
-        except (ImportError, ValueError), e:
+        except (ImportError, ValueError) as e:
             raise UsageError("Unable to load %r: %s\n" % (filename, e))
         spclasses = list(iter_spider_classes(module))
         if not spclasses:

scrapy/commands/settings.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 from scrapy.command import ScrapyCommand
 
 class Command(ScrapyCommand):
@@ -27,12 +28,12 @@ def add_options(self, parser):
     def run(self, args, opts):
         settings = self.crawler_process.settings
         if opts.get:
-            print settings.get(opts.get)
+            print(settings.get(opts.get))
         elif opts.getbool:
-            print settings.getbool(opts.getbool)
+            print(settings.getbool(opts.getbool))
         elif opts.getint:
-            print settings.getint(opts.getint)
+            print(settings.getint(opts.getint))
         elif opts.getfloat:
-            print settings.getfloat(opts.getfloat)
+            print(settings.getfloat(opts.getfloat))
         elif opts.getlist:
-            print settings.getlist(opts.getlist)
+            print(settings.getlist(opts.getlist))

scrapy/commands/startproject.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 import sys
 import string
 import re
@@ -36,11 +37,11 @@ def run(self, args, opts):
             raise UsageError()
         project_name = args[0]
         if not re.search(r'^[_a-zA-Z]\w*$', project_name):
-            print 'Error: Project names must begin with a letter and contain only\n' \
-                'letters, numbers and underscores'
+            print('Error: Project names must begin with a letter and contain only\n' \
+                'letters, numbers and underscores')
             sys.exit(1)
         elif exists(project_name):
-            print "Error: directory %r already exists" % project_name
+            print("Error: directory %r already exists" % project_name)
             sys.exit(1)
 
         moduletpl = join(TEMPLATES_PATH, 'module')