Support python3 in tricircle

1.What is the problem?
Tricircle doesn't support python3 yet, but python2 support will
be stopped in 2020. OpenStack community has put the support of
python3 as the community wide goal in Pike release, Tricircle
needs to be ready for this.

2.What is the solution to the problem?
Port the code to be compatible with both python2 and python3, for
python3, only the python3.5 version will be supported.

After this patch is merged, a new gate/check job for python3.5
should be enabled too.

3.What the features need to be implemented to the Tricircle
  to realize the solution?
No new features.

Change-Id: I18cb59cadb7a1c06f6cd729c4bda2c8e95d41e1e
Signed-off-by: joehuang <joehuang@huawei.com>

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 minversion = 1.6
-envlist = py34,py27,pypy,pep8
+envlist = py35,py27,pypy,pep8
 skipsdist = True
 
 [testenv]

tricircle/api/controllers/routing.py

@@ -13,6 +13,7 @@
 import pecan
 from pecan import expose
 from pecan import rest
+import six
 
 from oslo_log import log as logging
 
@@ -113,7 +114,7 @@ class RoutingController(rest.RestController):
         filters = self._get_filters(kwargs)
         filters = [{'key': key,
                     'comparator': 'eq',
-                    'value': value} for key, value in filters.iteritems()]
+                    'value': value} for key, value in six.iteritems(filters)]
 
         try:
             return {'routings': db_api.list_resource_routings(context,

tricircle/common/client.py

@@ -17,6 +17,7 @@ import collections
 import functools
 import inspect
 import six
+from six.moves import xrange
 import uuid
 
 from keystoneclient.auth.identity import v3 as auth_identity

tricircle/common/httpclient.py

@@ -13,8 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import urllib
+from six.moves.urllib import parse as urlparse
-import urlparse
 
 from requests import Request
 from requests import Session
@@ -104,7 +103,7 @@ def get_bottom_url(t_ver, t_url, b_ver, b_endpoint):
         if k == 'availability_zone':
             continue
         query_filters.append((k, v))
-    query = urllib.urlencode(query_filters)
+    query = urlparse.urlencode(query_filters)
 
     fragment = t_parse.fragment
 

tricircle/common/lock_handle.py

@@ -15,6 +15,7 @@
 
 import datetime
 import eventlet
+from six.moves import xrange
 
 import oslo_db.exception as db_exc
 

tricircle/common/restapp.py

@@ -17,8 +17,8 @@ from oslo_config import cfg
 from oslo_middleware import request_id
 from oslo_service import service
 
-import exceptions as t_exc
+from tricircle.common import exceptions as t_exc
-from i18n import _
+from tricircle.common.i18n import _
 
 
 def auth_app(app):

tricircle/common/xrpcapi.py

@@ -19,11 +19,10 @@ Client side of the job daemon RPC API.
 from oslo_config import cfg
 import oslo_messaging as messaging
 
-import rpc
-from serializer import TricircleSerializer as Serializer
-import topics
-
 from tricircle.common import constants
+from tricircle.common import rpc
+from tricircle.common import serializer as t_serializer
+from tricircle.common import topics
 
 
 CONF = cfg.CONF
@@ -59,7 +58,7 @@ class XJobAPI(object):
         else:
             version_cap = self.VERSION_ALIASES.get(upgrade_level,
                                                    upgrade_level)
-        serializer = Serializer()
+        serializer = t_serializer.TricircleSerializer()
         self.client = rpc.get_client(target,
                                      version_cap=version_cap,
                                      serializer=serializer)

tricircle/network/central_plugin.py

@@ -15,6 +15,7 @@
 
 import collections
 import copy
+import six
 
 from oslo_config import cfg
 from oslo_db.sqlalchemy import utils as sa_utils
@@ -549,7 +550,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
             port['id'] = port_id
             if fields:
                 port = dict(
-                    [(k, v) for k, v in port.iteritems() if k in fields])
+                    [(k, v) for k, v in six.iteritems(port) if k in fields])
             if 'network_id' not in port and 'fixed_ips' not in port:
                 return port
 
@@ -589,7 +590,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
             query = query.filter(
                 models_v2.IPAllocation.subnet_id.in_(subnet_ids))
 
-        for key, value in filters.iteritems():
+        for key, value in six.iteritems(filters):
             column = getattr(model, key, None)
             if column is not None:
                 if not value:
@@ -870,7 +871,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2,
                     continue
                 _filters = []
                 if filters:
-                    for key, value in filters.iteritems():
+                    for key, value in six.iteritems(filters):
                         if key == 'fixed_ips':
                             if 'ip_address' in value:
                                 _filters.append(

tricircle/network/helper.py

@@ -14,6 +14,7 @@
 #    under the License.
 
 import netaddr
+import six
 
 from neutron_lib import constants
 import neutronclient.common.exceptions as q_cli_exceptions
@@ -376,7 +377,7 @@ class NetworkHelper(object):
             subnet_dhcp_map[subnet['id']] = subnet['enable_dhcp']
 
         # dhcp port
-        for t_subnet_id, b_subnet_id in subnet_map.iteritems():
+        for t_subnet_id, b_subnet_id in six.iteritems(subnet_map):
             if not subnet_dhcp_map[t_subnet_id]:
                 continue
             self.prepare_dhcp_port(t_ctx, project_id, pod, t_net['id'],

tricircle/tests/unit/api/controllers/test_pod.py

@@ -15,6 +15,7 @@
 
 import mock
 from mock import patch
+import six
 import unittest
 
 import pecan
@@ -88,7 +89,7 @@ class PodsControllerTest(unittest.TestCase):
         actual = [(pod['region_name'],
                    pod['az_name']) for pod in pods['pods']]
         expect = [('TopPod', ''), ('BottomPod', 'TopAZ')]
-        self.assertItemsEqual(expect, actual)
+        six.assertCountEqual(self, expect, actual)
 
     @patch.object(pecan, 'response', new=mock.Mock)
     @patch.object(context, 'extract_context_from_environ')

tricircle/tests/unit/api/controllers/test_routing.py

@@ -13,6 +13,7 @@
 import mock
 from mock import patch
 from oslo_utils import uuidutils
+import six
 import unittest
 
 import pecan
@@ -41,7 +42,7 @@ class RoutingControllerTest(unittest.TestCase):
         policy.populate_default_rules()
 
     def _validate_error_code(self, res, code):
-        self.assertEqual(res[res.keys()[0]]['code'], code)
+        self.assertEqual(res[list(res.keys())[0]]['code'], code)
 
     @patch.object(pecan, 'response', new=FakeResponse)
     @patch.object(context, 'extract_context_from_environ')
@@ -234,7 +235,7 @@ class RoutingControllerTest(unittest.TestCase):
                   for routing in routings['routings']]
         expect = [('c7f641c9-8462-4007-84b2-3035d8cfb7a3', pod_id1),
                   ('b669a2da-ca95-47db-a2a9-ba9e546d82ee', pod_id2)]
-        self.assertItemsEqual(expect, actual)
+        six.assertCountEqual(self, expect, actual)
 
         # apply a resource type filter to the retrieved routings.
         kw_filter1 = {'resource_type': 'port'}
@@ -243,7 +244,7 @@ class RoutingControllerTest(unittest.TestCase):
                   routing['resource_type'])
                   for routing in routings['routings']]
         expect = [('b669a2da-ca95-47db-a2a9-ba9e546d82ee', pod_id2, 'port')]
-        self.assertItemsEqual(expect, actual)
+        six.assertCountEqual(self, expect, actual)
 
         # apply a filter and if it doesn't match with any of the retrieved
         # routings, then all of them will be discarded and the method returns
@@ -259,7 +260,7 @@ class RoutingControllerTest(unittest.TestCase):
                   for routing in routings['routings']]
         expect = [('c7f641c9-8462-4007-84b2-3035d8cfb7a3', pod_id1),
                   ('b669a2da-ca95-47db-a2a9-ba9e546d82ee', pod_id2)]
-        self.assertItemsEqual(expect, actual)
+        six.assertCountEqual(self, expect, actual)
 
         # failure case, only admin can show all resource routings
         self.context.is_admin = False

tricircle/tests/unit/db/test_api.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from six.moves import xrange
 import unittest
 
 from tricircle.common import context

tricircle/tests/unit/network/test_central_plugin.py

@@ -18,6 +18,8 @@ import copy
 import mock
 from mock import patch
 import netaddr
+import six
+from six.moves import xrange
 import unittest
 
 from sqlalchemy.orm import attributes
@@ -217,7 +219,7 @@ class FakePool(driver.Pool):
 class DotDict(dict):
     def __init__(self, normal_dict=None):
         if normal_dict:
-            for key, value in normal_dict.iteritems():
+            for key, value in six.iteritems(normal_dict):
                 self[key] = value
 
     def __getattr__(self, item):
@@ -255,7 +257,7 @@ class FakeNeutronClient(object):
             return_list = []
             for port in port_list:
                 is_selected = True
-                for key, value in params['filters'].iteritems():
+                for key, value in six.iteritems(params['filters']):
                     if key not in port or not port[key] or (
                             port[key] not in value):
                         is_selected = False
@@ -725,7 +727,7 @@ class FakeQuery(object):
         filtered_list = []
         for record in self.records:
             selected = True
-            for key, value in kwargs.iteritems():
+            for key, value in six.iteritems(kwargs):
                 if key not in record or record[key] != value:
                     selected = False
                     break
@@ -761,6 +763,8 @@ class FakeQuery(object):
         self.index += 1
         return self.records[self.index - 1]
 
+    __next__ = next
+
     def one(self):
         if len(self.records) == 0:
             raise exc.NoResultFound()
@@ -774,7 +778,7 @@ class FakeQuery(object):
 
     def update(self, values):
         for record in self.records:
-            for key, value in values.iteritems():
+            for key, value in six.iteritems(values):
                 record[key] = value
         return len(self.records)
 
@@ -1017,7 +1021,7 @@ class FakePlugin(plugin.TricirclePlugin):
         for allocation in TOP_IPALLOCATIONS:
             if allocation['port_id'] == port['id']:
                 ret = {}
-                for key, value in port.iteritems():
+                for key, value in six.iteritems(port):
                     if key == 'fixed_ips':
                         ret[key] = [{'subnet_id': allocation['subnet_id'],
                                      'ip_address': allocation['ip_address']}]
@@ -1215,10 +1219,10 @@ class PluginTest(unittest.TestCase,
                           {'id': 'top_id_3', 'name': 'top'}]
         for _ports in (ports1, ports2, ports3, ports4):
             ports.extend(_ports)
-        self.assertItemsEqual(expected_ports, ports)
+        six.assertCountEqual(self, expected_ports, ports)
 
         ports = fake_plugin.get_ports(neutron_context)
-        self.assertItemsEqual(expected_ports, ports)
+        six.assertCountEqual(self, expected_ports, ports)
 
     @patch.object(context, 'get_context_from_neutron_context',
                   new=fake_get_context_from_neutron_context)
@@ -1267,7 +1271,7 @@ class PluginTest(unittest.TestCase,
                      'device_id': 'router_id'},
                     {'id': 'top_id_2', 'name': 'bottom',
                      'device_id': 'router_id'}]
-        self.assertItemsEqual(expected, ports)
+        six.assertCountEqual(self, expected, ports)
 
     @patch.object(context, 'get_context_from_neutron_context')
     @patch.object(db_base_plugin_v2.NeutronDbPluginV2, 'delete_port')
@@ -1669,7 +1673,7 @@ class PluginTest(unittest.TestCase,
         # only one VLAN allocated since we just create one bridge network
         allocations = [
             allocation['allocated'] for allocation in TOP_VLANALLOCATIONS]
-        self.assertItemsEqual([True, False], allocations)
+        six.assertCountEqual(self, [True, False], allocations)
         for segment in TOP_SEGMENTS:
             self.assertIn(segment['segmentation_id'], (2000, 2001))
 

tricircle/tests/unit/network/test_helper.py

@@ -13,6 +13,7 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import six
 import unittest
 
 from oslo_utils import uuidutils
@@ -42,13 +43,15 @@ class HelperTest(unittest.TestCase):
         }
         body = self.helper.get_create_subnet_body(project_id, t_subnet,
                                                   b_net_id, '10.0.1.2')
-        self.assertItemsEqual([{'start': '10.0.1.3', 'end': '10.0.1.254'}],
+        six.assertCountEqual(self,
+                             [{'start': '10.0.1.3', 'end': '10.0.1.254'}],
                              body['subnet']['allocation_pools'])
         self.assertEqual('10.0.1.2', body['subnet']['gateway_ip'])
 
         body = self.helper.get_create_subnet_body(project_id, t_subnet,
                                                   b_net_id, '10.0.1.254')
-        self.assertItemsEqual([{'start': '10.0.1.2', 'end': '10.0.1.253'}],
+        six.assertCountEqual(self,
+                             [{'start': '10.0.1.2', 'end': '10.0.1.253'}],
                              body['subnet']['allocation_pools'])
         self.assertEqual('10.0.1.254', body['subnet']['gateway_ip'])
 
@@ -57,7 +60,7 @@ class HelperTest(unittest.TestCase):
             {'start': '10.0.1.20', 'end': '10.0.1.254'}]
         body = self.helper.get_create_subnet_body(project_id, t_subnet,
                                                   b_net_id, '10.0.1.5')
-        self.assertItemsEqual([{'start': '10.0.1.2', 'end': '10.0.1.4'},
+        six.assertCountEqual(self, [{'start': '10.0.1.2', 'end': '10.0.1.4'},
                              {'start': '10.0.1.6', 'end': '10.0.1.10'},
                              {'start': '10.0.1.20', 'end': '10.0.1.254'}],
                              body['subnet']['allocation_pools'])

tricircle/tests/unit/network/test_local_plugin.py

@@ -287,19 +287,19 @@ class PluginTest(unittest.TestCase):
         def ip_to_digit(ip):
             return int(ip[ip.rindex('.') + 1:])
 
-        pool_range = range(ip_to_digit(pool['start']),
+        pool_range = list(range(ip_to_digit(pool['start']),
-                           ip_to_digit(pool['end']) + 1)
+                                ip_to_digit(pool['end']) + 1))
-        b_pool_range1 = range(ip_to_digit(b_pools[0]['start']),
+        b_pool_range1 = list(range(ip_to_digit(b_pools[0]['start']),
-                              ip_to_digit(b_pools[0]['end']) + 1)
+                                   ip_to_digit(b_pools[0]['end']) + 1))
-        b_pool_range2 = range(ip_to_digit(b_pools[1]['start']),
+        b_pool_range2 = list(range(ip_to_digit(b_pools[1]['start']),
-                              ip_to_digit(b_pools[1]['end']) + 1)
+                                   ip_to_digit(b_pools[1]['end']) + 1))
         b_pool_range = b_pool_range1 + [
             ip_to_digit(b_gateway_ip)] + b_pool_range2
         port.pop('name')
         b_port.pop('name')
         self.assertDictEqual(net, b_net)
         self.assertDictEqual(subnet, b_subnet)
-        self.assertEqual(pool_range, b_pool_range)
+        self.assertListEqual(pool_range, b_pool_range)
         self.assertEqual('vlan', b_net_type)
         self.assertDictEqual(port, b_port)
 

tricircle/tests/unit/xjob/test_xmanager.py

@@ -16,6 +16,8 @@
 import datetime
 import mock
 from mock import patch
+import six
+from six.moves import xrange
 import unittest
 
 from oslo_config import cfg
@@ -306,7 +308,7 @@ class XManagerTest(unittest.TestCase):
             ctx, router_id, routes_body = actual_call[0]
             expect_ctx, expect_routes = except_map[router_id]
             self.assertEqual(expect_ctx, ctx)
-            self.assertItemsEqual(expect_routes,
+            six.assertCountEqual(self, expect_routes,
                                  routes_body['router']['routes'])
 
     @patch.object(FakeClient, 'update_routers')
@@ -460,7 +462,7 @@ class XManagerTest(unittest.TestCase):
         jobs = core.query_resource(self.context, models.AsyncJob, [], [])
         expected_status = [constants.JS_New, constants.JS_Success]
         job_status = [job['status'] for job in jobs]
-        self.assertItemsEqual(expected_status, job_status)
+        six.assertCountEqual(self, expected_status, job_status)
 
         self.assertEqual(fake_id, jobs[0]['resource_id'])
         self.assertEqual(fake_id, jobs[1]['resource_id'])
@@ -479,7 +481,7 @@ class XManagerTest(unittest.TestCase):
         jobs = core.query_resource(self.context, models.AsyncJob, [], [])
         expected_status = [constants.JS_New, constants.JS_Fail]
         job_status = [job['status'] for job in jobs]
-        self.assertItemsEqual(expected_status, job_status)
+        six.assertCountEqual(self, expected_status, job_status)
 
         self.assertEqual(fake_id, jobs[0]['resource_id'])
         self.assertEqual(fake_id, jobs[1]['resource_id'])
@@ -507,7 +509,7 @@ class XManagerTest(unittest.TestCase):
         jobs = core.query_resource(self.context, models.AsyncJob, [], [])
         expected_status = ['New', 'Fail', 'Success']
         job_status = [job['status'] for job in jobs]
-        self.assertItemsEqual(expected_status, job_status)
+        six.assertCountEqual(self, expected_status, job_status)
 
         for i in xrange(3):
             self.assertEqual(fake_id, jobs[i]['resource_id'])
@@ -564,7 +566,7 @@ class XManagerTest(unittest.TestCase):
         expected_ids = ['job_uuid3', 'job_uuid5']
         returned_jobs = db_api.get_latest_failed_jobs(self.context)
         actual_ids = [job['id'] for job in returned_jobs]
-        self.assertItemsEqual(expected_ids, actual_ids)
+        six.assertCountEqual(self, expected_ids, actual_ids)
 
     def tearDown(self):
         core.ModelBase.metadata.drop_all(core.get_engine())

tricircle/xjob/xmanager.py

@@ -634,10 +634,10 @@ class XManager(PeriodicTasks):
                 bottom_client.update_routers(
                     ctx, b_router_id, {'router': {'routes': extra_routes}})
                 continue
-            for router_id, cidr_ips_map in router_ips_map.iteritems():
+            for router_id, cidr_ips_map in six.iteritems(router_ips_map):
                 if router_id == b_router_id:
                     continue
-                for cidr, ips in cidr_ips_map.iteritems():
+                for cidr, ips in six.iteritems(cidr_ips_map):
                     if cidr in router_ips_map[b_router_id]:
                         continue
                     for ip in ips:
@@ -666,10 +666,10 @@ class XManager(PeriodicTasks):
 
         # handle extra routes for north-south router
         ip_bridge_ip_map = {}
-        for router_id, cidr_ips_map in router_ips_map.iteritems():
+        for router_id, cidr_ips_map in six.iteritems(router_ips_map):
             if router_id not in router_ew_bridge_ip_map:
                 continue
-            for cidr, ips in cidr_ips_map.iteritems():
+            for cidr, ips in six.iteritems(cidr_ips_map):
                 for ip in ips:
                     nexthop = router_ew_bridge_ip_map[router_id]
                     destination = ip + '/32'