Python3 compatibility

object_storage/client.py

@@ -11,6 +11,7 @@
 
 from object_storage import errors
 
+import six
 import logging
 logger = logging.getLogger(__name__)
 
@@ -20,7 +21,7 @@ def __init__(self, controller, headers={}):
         _headers = {}
 
         # Lowercase headers
-        for key, value in headers.iteritems():
+        for key, value in headers.items():
             _key = key.lower()
             _headers[_key] = value
         self.headers = _headers
@@ -42,7 +43,7 @@ def __init__(self, controller, headers={}):
         _properties['url'] = controller.url
 
         meta = {}
-        for key, value in self.headers.iteritems():
+        for key, value in self.headers.items():
             if key.startswith('meta_'):
                 meta[key[5:]] = value
             elif key.startswith('x-account-meta-'):
@@ -53,6 +54,12 @@ def __init__(self, controller, headers={}):
         self.properties = _properties
         self.data = self.properties
 
+    def __len__(self):
+        return len(self.properties)
+
+    def __iter__(self):
+        return iter(self.properties)
+
 
 class Client(object):
     """
@@ -160,7 +167,7 @@ def search(self, q=None, options=None, **kwargs):
         params = {}
         options = options or {}
         options.update(kwargs)
-        for key, val in options.iteritems():
+        for key, val in options.items():
             if key.startswith('q_'):
                 params["q.%s" % key[2:]] = val
             else:
@@ -176,7 +183,7 @@ def search(self, q=None, options=None, **kwargs):
         def _formatter(response):
             """ Formats search results. """
             headers = response.headers
-            items = json.loads(response.content)
+            items = json.loads(response.content if isinstance(response.content, six.string_types) else response.content.decode('utf8'))
             objs = []
             for item in items:
                 if 'type' not in item or item['type'] == 'container':
@@ -234,9 +241,9 @@ def set_metadata(self, meta, headers={}):
         @raises ResponseError
         """
         meta_headers = {}
-        for k, v in headers.iteritems():
+        for k, v in headers.items():
             meta_headers[k] = v
-        for k, v in meta.iteritems():
+        for k, v in meta.items():
             meta_headers["x-account-meta-%s" % (k, )] = v
         self.make_request('POST', headers=meta_headers)
 
@@ -262,7 +269,7 @@ def delete_container(self, name, recursive=False):
         try:
             return self.make_request('DELETE', [name],
                                      params=params, formatter=lambda r: True)
-        except errors.ResponseError, ex:
+        except errors.ResponseError as ex:
             if ex.status == 409:
                 raise errors.ContainerNotEmpty(ex.status,
                                                "ContainerNotEmpty Error")
@@ -282,7 +289,7 @@ def containers(self, marker=None, headers=None):
         def _formatter(res):
             containers = []
             if res.content:
-                items = json.loads(res.content)
+                items = json.loads(res.content if isinstance(res.content, six.string_types) else res.content.decode('utf8'))
                 for item in items:
                     name = item.get('name', None)
                     containers.append(self.container(name, item))

object_storage/container.py

@@ -4,6 +4,8 @@
     See COPYING for license information
 """
 import os
+import six
+
 from object_storage.utils import json, Model
 from object_storage import errors
 from object_storage.storage_object import StorageObject
@@ -16,7 +18,7 @@ def __init__(self, controller, name, headers={}):
         _headers = {}
 
         # Lowercase headers
-        for key, value in headers.iteritems():
+        for key, value in headers.items():
             _key = key.lower()
             _headers[_key] = value
         self.headers = _headers
@@ -43,7 +45,7 @@ def __init__(self, controller, name, headers={}):
 
         meta = {}
 
-        for key, value in self.headers.iteritems():
+        for key, value in self.headers.items():
             if key.startswith('meta_'):
                 meta[key[5:]] = value
             elif key.startswith('x-container-meta-'):
@@ -54,6 +56,12 @@ def __init__(self, controller, name, headers={}):
         self.properties = _properties
         self.data = self.properties
 
+    def __len__(self):
+        return len(self.properties)
+
+    def __iter__(self):
+        return iter(self.properties)
+
 
 class Container:
     """ Container class. Encapsulates Storage containers. """
@@ -150,7 +158,7 @@ def set_metadata(self, meta):
         @raises ResponseError
         """
         meta_headers = {}
-        for k, v in meta.iteritems():
+        for k, v in meta.items():
             meta_headers["x-container-meta-%s" % (k, )] = v
         return self.make_request('POST', headers=meta_headers)
 
@@ -227,7 +235,7 @@ def objects(self, limit=None, marker=None, base_only=False, headers=None):
         def _formatter(res):
             objects = {}
             if res.content:
-                items = json.loads(res.content)
+                items = json.loads(res.content if isinstance(res.content, six.string_types) else res.content.decode('utf8'))
                 for item in items:
                     if 'name' in item:
                         objects[item['name']] = self.storage_object(

object_storage/storage_object.py

@@ -6,8 +6,9 @@
 from object_storage.utils import json, Model
 import mimetypes
 import os
+import six
 import logging
-
+from io import IOBase
 try:
     import StringIO
 except ImportError:
@@ -30,7 +31,7 @@ def __init__(self, controller, container, name, headers={}):
         _headers = {}
 
         # Lowercase headers
-        for key, value in headers.iteritems():
+        for key, value in headers.items():
             _key = key.lower()
             _headers[_key] = value
         self.headers = _headers
@@ -59,7 +60,7 @@ def __init__(self, controller, container, name, headers={}):
         _properties['url'] = controller.url
 
         meta = {}
-        for key, value in self.headers.iteritems():
+        for key, value in self.headers.items():
             if key.startswith('meta_'):
                 meta[key[5:]] = value
             elif key.startswith('x-object-meta-'):
@@ -70,6 +71,11 @@ def __init__(self, controller, container, name, headers={}):
         self.properties = _properties
         self.data = self.properties
 
+    def __len__(self):
+        return len(self.properties)
+
+    def __iter__(self):
+        return iter(self.properties)
 
 class StorageObject:
     """
@@ -185,7 +191,7 @@ def list(self, limit=None, marker=None, base_only=False):
         def _formatter(res):
             objects = {}
             if res.content:
-                items = json.loads(res.content)
+                items = json.loads(res.content if isinstance(res.content, six.string_types) else res.content.decode('utf8'))
                 for item in items:
                     if 'name' in item:
                         objects[item['name']] = self.client.storage_object(
@@ -222,7 +228,7 @@ def set_metadata(self, meta):
         @raises ResponseError
         """
         meta_headers = {}
-        for k, v in meta.iteritems():
+        for k, v in meta.items():
             meta_headers["X-Object-Meta-%s" % (k, )] = v
         return self.make_request('POST', headers=meta_headers)
 
@@ -327,7 +333,7 @@ def send(self, data, check_md5=True):
         @return: StorageObject, self
         """
         size = None
-        if isinstance(data, file):
+        if isinstance(data, IOBase):
             try:
                 data.flush()
             except IOError:
@@ -337,7 +343,7 @@ def send(self, data, check_md5=True):
             if hasattr(data, '__len__'):
                 size = len(data)
 
-        if isinstance(data, basestring):
+        if isinstance(data, six.binary_type):
             data = StringIO.StringIO(data)
 
         headers = {}
@@ -364,7 +370,7 @@ def send(self, data, check_md5=True):
         res = conn.finish()
 
         if check_md5:
-            assert checksum.hexdigest() == res.headers['etag'], \
+            assert checksum.hexdigest() == res.headers.get('etag'), \
                 'md5 hashes do not match'
         res.headers['content-length'] = transfered
         self.model = StorageObjectModel(
@@ -480,6 +486,9 @@ def make_request(self, method, path=None, *args, **kwargs):
     def fileno(self):
         return 1
 
+    def close(self):
+        pass
+
     def __len__(self):
         if not self.model:
             self.load()

object_storage/transport/__init__.py

@@ -3,12 +3,18 @@
 
     See COPYING for license information
 """
-import httplib
 from socket import timeout
-from urlparse import urlparse
 
-import urllib2
+import sys
 import re
+if sys.version_info.major == 2:
+    from urllib2 import Request, urlopen
+    from urlparse import urlparse
+    from httplib import HTTPConnection, HTTPSConnection
+else:
+    from urllib.request import Request, urlopen
+    from urllib.parse import urlparse
+    from http.client import HTTPConnection, HTTPSConnection
 
 from object_storage.errors import ResponseError, NotFound
 from object_storage import consts
@@ -45,8 +51,7 @@ def _authenticate(self):
 
     def get_headers(self):
         """ Get default headers for this connection """
-        return dict([('User-Agent', consts.USER_AGENT)] +
-                    self.auth_headers.items())
+        return dict([('User-Agent', consts.USER_AGENT)] + list(self.auth_headers.items()))
 
     def chunk_upload(self, method, url, size=None, headers=None):
         """ Returns new ChunkedConnection """
@@ -58,10 +63,10 @@ def chunk_upload(self, method, url, size=None, headers=None):
     def chunk_download(self, url, chunk_size=10 * 1024):
         """ Returns new ChunkedConnection """
         headers = self.get_headers()
-        req = urllib2.Request(url)
-        for k, v in headers.iteritems():
+        req = Request(url)
+        for k, v in headers.items():
             req.add_header(k, v)
-        r = urllib2.urlopen(req)
+        r = urlopen(req)
         while True:
             buff = r.read(chunk_size)
             if not buff:
@@ -158,16 +163,16 @@ def __init__(self, conn, method, url, size=None, headers=None):
         port = int(port)
 
         if scheme == 'https':
-            self.req = httplib.HTTPSConnection(host, port)
+            self.req = HTTPSConnection(host, port)
         else:
-            self.req = httplib.HTTPConnection(host, port)
+            self.req = HTTPConnection(host, port)
         try:
             self.req.putrequest('PUT', path)
-            for key, value in headers.iteritems():
+            for key, value in headers.items():
                 self.req.putheader(key, value)
             self.req.endheaders()
-        except Exception:
-            raise ResponseError(0, 'Disconnected')
+        except Exception as e:
+            raise ResponseError(0, 'Disconnected: %s' % e)
 
     def send(self, chunk):
         """ Sends a chunk of data. """
@@ -178,7 +183,7 @@ def send(self, chunk):
                 self.req.send("\r\n")
             else:
                 self.req.send(chunk)
-        except timeout, err:
+        except timeout as err:
             raise err
         except:
             raise ResponseError(0, 'Disconnected')
@@ -188,7 +193,7 @@ def finish(self):
         try:
             if self._chunked_encoding:
                 self.req.send("0\r\n\r\n")
-        except timeout, err:
+        except timeout as err:
             raise err
         except:
             raise ResponseError(0, 'Disconnected')
@@ -199,7 +204,7 @@ def finish(self):
         r = Response()
         r.status_code = res.status
         r.version = res.version
-        r.headers = dict(res.getheaders())
+        r.headers = dict([(k.lower(), v) for k,v in res.getheaders()])
         r.content = content
         r.raise_for_status()
         return r

object_storage/transport/httplib2conn.py

@@ -3,13 +3,13 @@
 
     See COPYING for license information
 """
-import urllib
+import six
 from object_storage import errors
 from object_storage.transport import BaseAuthentication, \
     BaseAuthenticatedConnection, Response
 import httplib2
 
-from object_storage.utils import json
+from object_storage.utils import json, unicode_urlencode
 
 import logging
 logger = logging.getLogger(__name__)
@@ -39,7 +39,7 @@ def make_request(self, method, url=None, headers=None, formatter=None,
         headers.update(self.get_headers())
 
         if params:
-            url = "%s?%s" % (url, urllib.urlencode(params))
+            url = "%s?%s" % (url, unicode_urlencode(params))
 
         def _make_request(headers):
             logger.debug("%s %s %s" % (method, url, headers))
@@ -100,7 +100,7 @@ def authenticate(self):
             raise errors.AuthenticationError('Invalid Credentials')
         response.raise_for_status()
         try:
-            storage_options = json.loads(response.content)['storage']
+            storage_options = json.loads(response.content if isinstance(response.content, six.string_types) else response.content.decode('utf8'))['storage']
         except ValueError:
             raise errors.StorageURLNotFound("Could not parse services JSON.")