Fix py33 compatibility errors

Part of blueprint py33

Change-Id: Ib24bb7e5147a6d241d0392889832ed34e1c856eb

stackalytics/processor/mls.py

@@ -14,9 +14,7 @@
 # limitations under the License.
 
 from email import utils as email_utils
-import gzip
 import re
-import StringIO
 
 import six
 from six.moves import http_client
@@ -85,8 +83,8 @@ def _retrieve_mails(uri):
     if not content:
         LOG.error('Error reading mail archive from uri: %s', uri)
         return
-    gzip_fd = gzip.GzipFile(fileobj=StringIO.StringIO(content))
-    content = gzip_fd.read()
+
+    content = utils.gzip_decompress(content)
     LOG.debug('Mail archive is loaded, start processing')
 
     content += TRAILING_RECORD

stackalytics/processor/mps.py

@@ -33,7 +33,7 @@ CNT_EMPTY_MEMBERS = 50
 
 
 def _convert_str_fields_to_unicode(result):
-    for field, value in result.iteritems():
+    for field, value in six.iteritems(result):
         if type(value) is str:
             try:
                 value = six.text_type(value, 'utf8')

stackalytics/processor/normalizer.py

@@ -33,9 +33,9 @@ def _normalize_user(user):
         elif y["end_date"] == 0:
             return -1
         else:
-            return cmp(x["end_date"], y["end_date"])
+            return x["end_date"] - y["end_date"]
 
-    user['companies'].sort(cmp=end_date_comparator)
+    user['companies'].sort(key=utils.cmp_to_key(end_date_comparator))
     user['user_id'] = user['launchpad_id']
 
 

stackalytics/processor/rcs.py

@@ -108,7 +108,7 @@ class Gerrit(Rcs):
             return False
 
     def _poll_reviews(self, project_organization, module, branch,
-                      start_id=None, last_id=None, is_open=False,
+                      start_id=0, last_id=0, is_open=False,
                       grab_comments=False):
         sort_key = start_id
 

stackalytics/processor/runtime_storage.py

@@ -127,23 +127,29 @@ class MemcachedStorage(RuntimeStorage):
         return self.memcached.incr('user:count')
 
     def get_all_users(self):
-        for n in xrange(0, self.get_by_key('user:count') + 1):
+        for n in six.moves.range(0, self.get_by_key('user:count') + 1):
             user = self.get_by_key('user:%s' % n)
             if user:
                 yield user
 
     def get_by_key(self, key):
-        return self.memcached.get(key.encode('utf8'))
+        if six.PY2:
+            key = key.encode('utf8')
+        return self.memcached.get(key)
 
     def set_by_key(self, key, value):
-        if not self.memcached.set(key.encode('utf8'), value):
+        if six.PY2:
+            key = key.encode('utf8')
+        if not self.memcached.set(key, value):
             LOG.critical('Failed to store data in memcached: '
                          'key %(key)s, value %(value)s',
                          {'key': key, 'value': value})
             raise Exception('Memcached set failed')
 
     def delete_by_key(self, key):
-        if not self.memcached.delete(key.encode('utf8')):
+        if six.PY2:
+            key = key.encode('utf8')
+        if not self.memcached.delete(key):
             LOG.critical('Failed to delete data from memcached: key %s', key)
             raise Exception('Memcached delete failed')
 

stackalytics/processor/utils.py

@@ -15,14 +15,13 @@
 
 import cgi
 import datetime
+import gzip
 import json
 import re
 import time
 
 import iso8601
 import six
-from six.moves.urllib import parse
-from six.moves.urllib import request
 
 from stackalytics.openstack.common import log as logging
 
@@ -89,8 +88,8 @@ def check_email_validity(email):
 
 def read_uri(uri):
     try:
-        fd = request.urlopen(uri)
-        raw = fd.read()
+        fd = six.moves.urllib.request.urlopen(uri)
+        raw = fd.read().decode('utf8')
         fd.close()
         return raw
     except Exception as e:
@@ -106,12 +105,50 @@ def read_json_from_uri(uri):
                  {'error': e, 'uri': uri})
 
 
+def gzip_decompress(content):
+    if six.PY3:
+        return gzip.decompress(content)
+    else:
+        gzip_fd = gzip.GzipFile(fileobj=six.moves.StringIO.StringIO(content))
+        return gzip_fd.read()
+
+
+def cmp_to_key(mycmp):  # ported from python 3
+    """Convert a cmp= function into a key= function."""
+    class K(object):
+        __slots__ = ['obj']
+
+        def __init__(self, obj):
+            self.obj = obj
+
+        def __lt__(self, other):
+            return mycmp(self.obj, other.obj) < 0
+
+        def __gt__(self, other):
+            return mycmp(self.obj, other.obj) > 0
+
+        def __eq__(self, other):
+            return mycmp(self.obj, other.obj) == 0
+
+        def __le__(self, other):
+            return mycmp(self.obj, other.obj) <= 0
+
+        def __ge__(self, other):
+            return mycmp(self.obj, other.obj) >= 0
+
+        def __ne__(self, other):
+            return mycmp(self.obj, other.obj) != 0
+
+        __hash__ = None
+    return K
+
+
 def make_range(start, stop, step):
     last_full = stop - ((stop - start) % step)
-    for i in six.moves.xrange(start, last_full, step):
-        yield six.moves.xrange(i, i + step)
+    for i in six.moves.range(start, last_full, step):
+        yield six.moves.range(i, i + step)
     if stop > last_full:
-        yield six.moves.xrange(last_full, stop)
+        yield six.moves.range(last_full, stop)
 
 
 def store_user(runtime_storage_inst, user):
@@ -200,7 +237,7 @@ def add_index(sequence, start=1, item_filter=lambda x: True):
 
 
 def safe_encode(s):
-    return parse.quote(s.encode('utf-8'))
+    return six.moves.urllib.parse.quote(s.encode('utf-8'))
 
 
 def make_module_group(module_group_id, name=None, modules=None, tag='module'):

stackalytics/processor/vcs.py

@@ -190,18 +190,18 @@ class Git(Vcs):
         try:
             output = sh.git('log', '--pretty=' + GIT_LOG_FORMAT, '--shortstat',
                             '-M', '--no-merges', commit_range, _tty_out=False,
-                            _decode_errors='ignore')
+                            _decode_errors='ignore', _encoding='utf8')
         except sh.ErrorReturnCode as e:
             LOG.error('Unable to get log of git repo %s. Ignore it',
                       self.repo['uri'])
             LOG.exception(e)
             return
 
-        for rec in re.finditer(GIT_LOG_PATTERN, str(output)):
+        for rec in re.finditer(GIT_LOG_PATTERN, six.text_type(output)):
             i = 1
             commit = {}
             for param in GIT_LOG_PARAMS:
-                commit[param[0]] = six.text_type(rec.group(i), 'utf8')
+                commit[param[0]] = rec.group(i)
                 i += 1
 
             if not utils.check_email_validity(commit['author_email']):