Fix py33 compatibility errors

Part of blueprint py33

Change-Id: Ib24bb7e5147a6d241d0392889832ed34e1c856eb

stackalytics/dashboard/reports.py

@@ -20,6 +20,7 @@ import operator
 import time
 
 import flask
+import six
 
 from stackalytics.dashboard import decorators
 from stackalytics.dashboard import helpers
@@ -159,15 +160,15 @@ def members():
 
 def _get_punch_card_data(records):
     punch_card_raw = []  # matrix days x hours
-    for wday in xrange(0, 7):
+    for wday in six.moves.range(0, 7):
         punch_card_raw.append([0] * 24)
     for record in records:
         tt = datetime.datetime.fromtimestamp(record['date']).timetuple()
         punch_card_raw[tt.tm_wday][tt.tm_hour] += 1
 
     punch_card_data = []  # format for jqplot bubble renderer
-    for wday in xrange(0, 7):
+    for wday in six.moves.range(0, 7):
-        for hour in xrange(0, 24):
+        for hour in six.moves.range(0, 24):
             v = punch_card_raw[wday][hour]
             if v:
                 punch_card_data.append([hour, wday, v, v])

stackalytics/processor/mls.py

@@ -14,9 +14,7 @@
 # limitations under the License.
 
 from email import utils as email_utils
-import gzip
 import re
-import StringIO
 
 import six
 from six.moves import http_client
@@ -85,8 +83,8 @@ def _retrieve_mails(uri):
     if not content:
         LOG.error('Error reading mail archive from uri: %s', uri)
         return
-    gzip_fd = gzip.GzipFile(fileobj=StringIO.StringIO(content))
+
-    content = gzip_fd.read()
+    content = utils.gzip_decompress(content)
     LOG.debug('Mail archive is loaded, start processing')
 
     content += TRAILING_RECORD

stackalytics/processor/mps.py

@@ -33,7 +33,7 @@ CNT_EMPTY_MEMBERS = 50
 
 
 def _convert_str_fields_to_unicode(result):
-    for field, value in result.iteritems():
+    for field, value in six.iteritems(result):
         if type(value) is str:
             try:
                 value = six.text_type(value, 'utf8')

stackalytics/processor/normalizer.py

@@ -33,9 +33,9 @@ def _normalize_user(user):
         elif y["end_date"] == 0:
             return -1
         else:
-            return cmp(x["end_date"], y["end_date"])
+            return x["end_date"] - y["end_date"]
 
-    user['companies'].sort(cmp=end_date_comparator)
+    user['companies'].sort(key=utils.cmp_to_key(end_date_comparator))
     user['user_id'] = user['launchpad_id']
 
 

stackalytics/processor/rcs.py

@@ -108,7 +108,7 @@ class Gerrit(Rcs):
             return False
 
     def _poll_reviews(self, project_organization, module, branch,
-                      start_id=None, last_id=None, is_open=False,
+                      start_id=0, last_id=0, is_open=False,
                       grab_comments=False):
         sort_key = start_id
 

stackalytics/processor/runtime_storage.py

@@ -127,23 +127,29 @@ class MemcachedStorage(RuntimeStorage):
         return self.memcached.incr('user:count')
 
     def get_all_users(self):
-        for n in xrange(0, self.get_by_key('user:count') + 1):
+        for n in six.moves.range(0, self.get_by_key('user:count') + 1):
             user = self.get_by_key('user:%s' % n)
             if user:
                 yield user
 
     def get_by_key(self, key):
-        return self.memcached.get(key.encode('utf8'))
+        if six.PY2:
+            key = key.encode('utf8')
+        return self.memcached.get(key)
 
     def set_by_key(self, key, value):
-        if not self.memcached.set(key.encode('utf8'), value):
+        if six.PY2:
+            key = key.encode('utf8')
+        if not self.memcached.set(key, value):
             LOG.critical('Failed to store data in memcached: '
                          'key %(key)s, value %(value)s',
                          {'key': key, 'value': value})
             raise Exception('Memcached set failed')
 
     def delete_by_key(self, key):
-        if not self.memcached.delete(key.encode('utf8')):
+        if six.PY2:
+            key = key.encode('utf8')
+        if not self.memcached.delete(key):
             LOG.critical('Failed to delete data from memcached: key %s', key)
             raise Exception('Memcached delete failed')
 

stackalytics/processor/utils.py

@@ -15,14 +15,13 @@
 
 import cgi
 import datetime
+import gzip
 import json
 import re
 import time
 
 import iso8601
 import six
-from six.moves.urllib import parse
-from six.moves.urllib import request
 
 from stackalytics.openstack.common import log as logging
 
@@ -89,8 +88,8 @@ def check_email_validity(email):
 
 def read_uri(uri):
     try:
-        fd = request.urlopen(uri)
+        fd = six.moves.urllib.request.urlopen(uri)
-        raw = fd.read()
+        raw = fd.read().decode('utf8')
         fd.close()
         return raw
     except Exception as e:
@@ -106,12 +105,50 @@ def read_json_from_uri(uri):
                  {'error': e, 'uri': uri})
 
 
+def gzip_decompress(content):
+    if six.PY3:
+        return gzip.decompress(content)
+    else:
+        gzip_fd = gzip.GzipFile(fileobj=six.moves.StringIO.StringIO(content))
+        return gzip_fd.read()
+
+
+def cmp_to_key(mycmp):  # ported from python 3
+    """Convert a cmp= function into a key= function."""
+    class K(object):
+        __slots__ = ['obj']
+
+        def __init__(self, obj):
+            self.obj = obj
+
+        def __lt__(self, other):
+            return mycmp(self.obj, other.obj) < 0
+
+        def __gt__(self, other):
+            return mycmp(self.obj, other.obj) > 0
+
+        def __eq__(self, other):
+            return mycmp(self.obj, other.obj) == 0
+
+        def __le__(self, other):
+            return mycmp(self.obj, other.obj) <= 0
+
+        def __ge__(self, other):
+            return mycmp(self.obj, other.obj) >= 0
+
+        def __ne__(self, other):
+            return mycmp(self.obj, other.obj) != 0
+
+        __hash__ = None
+    return K
+
+
 def make_range(start, stop, step):
     last_full = stop - ((stop - start) % step)
-    for i in six.moves.xrange(start, last_full, step):
+    for i in six.moves.range(start, last_full, step):
-        yield six.moves.xrange(i, i + step)
+        yield six.moves.range(i, i + step)
     if stop > last_full:
-        yield six.moves.xrange(last_full, stop)
+        yield six.moves.range(last_full, stop)
 
 
 def store_user(runtime_storage_inst, user):
@@ -200,7 +237,7 @@ def add_index(sequence, start=1, item_filter=lambda x: True):
 
 
 def safe_encode(s):
-    return parse.quote(s.encode('utf-8'))
+    return six.moves.urllib.parse.quote(s.encode('utf-8'))
 
 
 def make_module_group(module_group_id, name=None, modules=None, tag='module'):

stackalytics/processor/vcs.py

@@ -190,18 +190,18 @@ class Git(Vcs):
         try:
             output = sh.git('log', '--pretty=' + GIT_LOG_FORMAT, '--shortstat',
                             '-M', '--no-merges', commit_range, _tty_out=False,
-                            _decode_errors='ignore')
+                            _decode_errors='ignore', _encoding='utf8')
         except sh.ErrorReturnCode as e:
             LOG.error('Unable to get log of git repo %s. Ignore it',
                       self.repo['uri'])
             LOG.exception(e)
             return
 
-        for rec in re.finditer(GIT_LOG_PATTERN, str(output)):
+        for rec in re.finditer(GIT_LOG_PATTERN, six.text_type(output)):
             i = 1
             commit = {}
             for param in GIT_LOG_PARAMS:
-                commit[param[0]] = six.text_type(rec.group(i), 'utf8')
+                commit[param[0]] = rec.group(i)
                 i += 1
 
             if not utils.check_email_validity(commit['author_email']):

tests/unit/test_config_files.py

@@ -13,9 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import functools
 import json
 
 import jsonschema
+import six
 import testtools
 
 
@@ -24,17 +26,25 @@ class TestConfigFiles(testtools.TestCase):
         super(TestConfigFiles, self).setUp()
 
     def _read_file(self, file_name):
-        with open(file_name, 'r') as content_file:
+        if six.PY3:
+            opener = functools.partial(open, encoding='utf8')
+        else:
+            opener = open
+        with opener(file_name, 'r') as content_file:
             content = content_file.read()
             return json.loads(content)
 
     def _verify_ordering(self, array, key, msg):
-        sorted_array = sorted(array, key=key)
+        comparator = lambda x, y: (x > y) - (x < y)
-        diff_msg = None
+
-        for i in range(len(array)):
+        diff_msg = ''
-            if array[i] != sorted_array[i]:
+        for i in range(len(array) - 1):
-                diff_msg = ('First differing element %s:\n%s\n%s' %
+            if comparator(key(array[i]), key(array[i + 1])) > 0:
-                            (i, array[i], sorted_array[i]))
+                diff_msg = ('Order fails at index %(index)s, '
+                            'elements:\n%(first)s:\n%(second)s' %
+                            {'index': i, 'first': array[i],
+                             'second': array[i + 1]})
+                break
         if diff_msg:
             self.fail(msg + '\n' + diff_msg)
 

tests/unit/test_record_processor.py

@@ -743,11 +743,11 @@ class TestRecordProcessor(testtools.TestCase):
                 'user_name': 'John Doe',
                 'emails': ['john_doe@ibm.com', 'john_doe@gmail.com'],
                 'companies': [{'company_name': 'IBM', 'end_date': 0}]}
-        self.assertEqual(user, utils.load_user(
+        self.assertUsersMatch(user, utils.load_user(
             record_processor_inst.runtime_storage_inst, 'john_doe'))
-        self.assertEqual(user, utils.load_user(
+        self.assertUsersMatch(user, utils.load_user(
             record_processor_inst.runtime_storage_inst, 'john_doe@gmail.com'))
-        self.assertEqual(user, utils.load_user(
+        self.assertUsersMatch(user, utils.load_user(
             record_processor_inst.runtime_storage_inst, 'john_doe@ibm.com'))
 
     def test_merge_users(self):
@@ -866,10 +866,10 @@ class TestRecordProcessor(testtools.TestCase):
                   'companies': [{'company_name': '*independent',
                                  'end_date': 0}]}
         runtime_storage_inst = record_processor_inst.runtime_storage_inst
-        self.assertEqual(user_1, utils.load_user(runtime_storage_inst,
+        self.assertUsersMatch(user_1, utils.load_user(runtime_storage_inst,
-                                                 'john_doe'))
+                                                      'john_doe'))
-        self.assertEqual(user_2, utils.load_user(runtime_storage_inst,
+        self.assertUsersMatch(user_2, utils.load_user(runtime_storage_inst,
-                                                 'homer'))
+                                                      'homer'))
 
     def test_process_commit_with_coauthors(self):
         record_processor_inst = self.make_record_processor(
@@ -1337,6 +1337,16 @@ class TestRecordProcessor(testtools.TestCase):
             self.assertEqual(value, actual[key],
                              'Values for key %s do not match' % key)
 
+    def assertUsersMatch(self, expected, actual):
+        match = True
+        for key, value in six.iteritems(expected):
+            if key == 'emails':
+                match = (set(value) == set(actual[key]))
+            else:
+                match = (value == actual[key])
+
+        self.assertTrue(match, 'User %s should match %s' % (actual, expected))
+
     # Helpers
 
     def make_record_processor(self, users=None, companies=None, releases=None,
@@ -1415,7 +1425,7 @@ def make_runtime_storage(users=None, companies=None, releases=None,
         return count
 
     def get_all_users():
-        for n in six.moves.xrange(
+        for n in six.moves.range(
                 0, (runtime_storage_cache.get('user:count') or 0) + 1):
             u = runtime_storage_cache.get('user:%s' % n)
             if u:

tests/unit/test_vcs.py

@@ -150,9 +150,12 @@ diff_stat:
         self.assertEqual(0, commits[4]['lines_deleted'])
         self.assertFalse('coauthor' in commits[4])
 
-        self.assertEqual(
+        self.assertIn(
-            [{'author_name': 'Tupac Shakur',
+            {'author_name': 'Tupac Shakur',
-              'author_email': 'tupac.shakur@openstack.com'},
+             'author_email': 'tupac.shakur@openstack.com'},
-             {'author_name': 'Bob Dylan',
+            commits[5]['coauthor'])
-              'author_email': 'bob.dylan@openstack.com'}],
+
+        self.assertIn(
+            {'author_name': 'Bob Dylan',
+             'author_email': 'bob.dylan@openstack.com'},
             commits[5]['coauthor'])

tox.ini

@@ -21,13 +21,17 @@ deps = -r{toxinidir}/requirements-py3.txt
        -r{toxinidir}/test-requirements.txt
 # to be removed once all tests passed
 commands = python -m testtools.run \
-               tests.unit.test_utils \
+               tests.unit.test_config_files \
+               tests.unit.test_default_data_processor \
                tests.unit.test_mps \
-               tests.unit.test_record_processor
+               tests.unit.test_mls \
+               tests.unit.test_record_processor \
+               tests.unit.test_utils \
+               tests.unit.test_vcs
 
 [testenv:pep8]
 commands = flake8
-           {toxinidir}/tools/requirements_style_check.sh requirements.txt test-requirements.txt
+           {toxinidir}/tools/requirements_style_check.sh requirements.txt requirements-py3.txt test-requirements.txt
 distribute = false
 
 [testenv:venv]