From bd4b205f141dbfe26cc68f2d8d815998b185ceeb Mon Sep 17 00:00:00 2001 From: joehuang Date: Fri, 6 Jan 2017 19:12:23 -0500 Subject: [PATCH] Support python3 in tricircle 1.What is the problem? Tricircle doesn't support python3 yet, but python2 support will be stopped in 2020. OpenStack community has put the support of python3 as the community wide goal in Pike release, Tricircle needs to be ready for this. 2.What is the solution to the problem? Port the code to be compatible with both python2 and python3, for python3, only the python3.5 version will be supported. After this patch is merged, a new gate/check job for python3.5 should be enabled too. 3.What the features need to be implemented to the Tricircle to realize the solution? No new features. Change-Id: I18cb59cadb7a1c06f6cd729c4bda2c8e95d41e1e Signed-off-by: joehuang --- tox.ini | 2 +- tricircle/api/controllers/routing.py | 3 ++- tricircle/common/client.py | 1 + tricircle/common/httpclient.py | 5 ++--- tricircle/common/lock_handle.py | 1 + tricircle/common/restapp.py | 4 ++-- tricircle/common/xrpcapi.py | 9 ++++---- tricircle/network/central_plugin.py | 7 +++--- tricircle/network/helper.py | 3 ++- .../tests/unit/api/controllers/test_pod.py | 3 ++- .../unit/api/controllers/test_routing.py | 9 ++++---- tricircle/tests/unit/db/test_api.py | 1 + .../tests/unit/network/test_central_plugin.py | 22 +++++++++++-------- tricircle/tests/unit/network/test_helper.py | 19 +++++++++------- .../tests/unit/network/test_local_plugin.py | 14 ++++++------ tricircle/tests/unit/xjob/test_xmanager.py | 14 +++++++----- tricircle/xjob/xmanager.py | 8 +++---- 17 files changed, 70 insertions(+), 55 deletions(-) diff --git a/tox.ini b/tox.ini index 66d363c3..a6ec9750 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = py34,py27,pypy,pep8 +envlist = py35,py27,pypy,pep8 skipsdist = True [testenv] diff --git a/tricircle/api/controllers/routing.py b/tricircle/api/controllers/routing.py index d8efa2d9..4e72adca 100644 --- a/tricircle/api/controllers/routing.py +++ b/tricircle/api/controllers/routing.py @@ -13,6 +13,7 @@ import pecan from pecan import expose from pecan import rest +import six from oslo_log import log as logging @@ -113,7 +114,7 @@ class RoutingController(rest.RestController): filters = self._get_filters(kwargs) filters = [{'key': key, 'comparator': 'eq', - 'value': value} for key, value in filters.iteritems()] + 'value': value} for key, value in six.iteritems(filters)] try: return {'routings': db_api.list_resource_routings(context, diff --git a/tricircle/common/client.py b/tricircle/common/client.py index 5c260877..649507fa 100644 --- a/tricircle/common/client.py +++ b/tricircle/common/client.py @@ -17,6 +17,7 @@ import collections import functools import inspect import six +from six.moves import xrange import uuid from keystoneclient.auth.identity import v3 as auth_identity diff --git a/tricircle/common/httpclient.py b/tricircle/common/httpclient.py index 6cf082af..5ed75ab2 100644 --- a/tricircle/common/httpclient.py +++ b/tricircle/common/httpclient.py @@ -13,8 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. -import urllib -import urlparse +from six.moves.urllib import parse as urlparse from requests import Request from requests import Session @@ -104,7 +103,7 @@ def get_bottom_url(t_ver, t_url, b_ver, b_endpoint): if k == 'availability_zone': continue query_filters.append((k, v)) - query = urllib.urlencode(query_filters) + query = urlparse.urlencode(query_filters) fragment = t_parse.fragment diff --git a/tricircle/common/lock_handle.py b/tricircle/common/lock_handle.py index 3fe78755..bfdc6b50 100644 --- a/tricircle/common/lock_handle.py +++ b/tricircle/common/lock_handle.py @@ -15,6 +15,7 @@ import datetime import eventlet +from six.moves import xrange import oslo_db.exception as db_exc diff --git a/tricircle/common/restapp.py b/tricircle/common/restapp.py index 2844ffbc..99fe2715 100644 --- a/tricircle/common/restapp.py +++ b/tricircle/common/restapp.py @@ -17,8 +17,8 @@ from oslo_config import cfg from oslo_middleware import request_id from oslo_service import service -import exceptions as t_exc -from i18n import _ +from tricircle.common import exceptions as t_exc +from tricircle.common.i18n import _ def auth_app(app): diff --git a/tricircle/common/xrpcapi.py b/tricircle/common/xrpcapi.py index cfd613b2..53615a74 100644 --- a/tricircle/common/xrpcapi.py +++ b/tricircle/common/xrpcapi.py @@ -19,11 +19,10 @@ Client side of the job daemon RPC API. from oslo_config import cfg import oslo_messaging as messaging -import rpc -from serializer import TricircleSerializer as Serializer -import topics - from tricircle.common import constants +from tricircle.common import rpc +from tricircle.common import serializer as t_serializer +from tricircle.common import topics CONF = cfg.CONF @@ -59,7 +58,7 @@ class XJobAPI(object): else: version_cap = self.VERSION_ALIASES.get(upgrade_level, upgrade_level) - serializer = Serializer() + serializer = t_serializer.TricircleSerializer() self.client = rpc.get_client(target, version_cap=version_cap, serializer=serializer) diff --git a/tricircle/network/central_plugin.py b/tricircle/network/central_plugin.py index a3969bfe..c9fd3dd4 100644 --- a/tricircle/network/central_plugin.py +++ b/tricircle/network/central_plugin.py @@ -15,6 +15,7 @@ import collections import copy +import six from oslo_config import cfg from oslo_db.sqlalchemy import utils as sa_utils @@ -549,7 +550,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, port['id'] = port_id if fields: port = dict( - [(k, v) for k, v in port.iteritems() if k in fields]) + [(k, v) for k, v in six.iteritems(port) if k in fields]) if 'network_id' not in port and 'fixed_ips' not in port: return port @@ -589,7 +590,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, query = query.filter( models_v2.IPAllocation.subnet_id.in_(subnet_ids)) - for key, value in filters.iteritems(): + for key, value in six.iteritems(filters): column = getattr(model, key, None) if column is not None: if not value: @@ -870,7 +871,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, continue _filters = [] if filters: - for key, value in filters.iteritems(): + for key, value in six.iteritems(filters): if key == 'fixed_ips': if 'ip_address' in value: _filters.append( diff --git a/tricircle/network/helper.py b/tricircle/network/helper.py index 7a9fcedb..2b1f6117 100644 --- a/tricircle/network/helper.py +++ b/tricircle/network/helper.py @@ -14,6 +14,7 @@ # under the License. import netaddr +import six from neutron_lib import constants import neutronclient.common.exceptions as q_cli_exceptions @@ -376,7 +377,7 @@ class NetworkHelper(object): subnet_dhcp_map[subnet['id']] = subnet['enable_dhcp'] # dhcp port - for t_subnet_id, b_subnet_id in subnet_map.iteritems(): + for t_subnet_id, b_subnet_id in six.iteritems(subnet_map): if not subnet_dhcp_map[t_subnet_id]: continue self.prepare_dhcp_port(t_ctx, project_id, pod, t_net['id'], diff --git a/tricircle/tests/unit/api/controllers/test_pod.py b/tricircle/tests/unit/api/controllers/test_pod.py index 60ea36e8..e84bebb5 100644 --- a/tricircle/tests/unit/api/controllers/test_pod.py +++ b/tricircle/tests/unit/api/controllers/test_pod.py @@ -15,6 +15,7 @@ import mock from mock import patch +import six import unittest import pecan @@ -88,7 +89,7 @@ class PodsControllerTest(unittest.TestCase): actual = [(pod['region_name'], pod['az_name']) for pod in pods['pods']] expect = [('TopPod', ''), ('BottomPod', 'TopAZ')] - self.assertItemsEqual(expect, actual) + six.assertCountEqual(self, expect, actual) @patch.object(pecan, 'response', new=mock.Mock) @patch.object(context, 'extract_context_from_environ') diff --git a/tricircle/tests/unit/api/controllers/test_routing.py b/tricircle/tests/unit/api/controllers/test_routing.py index 898ee849..a559ca22 100644 --- a/tricircle/tests/unit/api/controllers/test_routing.py +++ b/tricircle/tests/unit/api/controllers/test_routing.py @@ -13,6 +13,7 @@ import mock from mock import patch from oslo_utils import uuidutils +import six import unittest import pecan @@ -41,7 +42,7 @@ class RoutingControllerTest(unittest.TestCase): policy.populate_default_rules() def _validate_error_code(self, res, code): - self.assertEqual(res[res.keys()[0]]['code'], code) + self.assertEqual(res[list(res.keys())[0]]['code'], code) @patch.object(pecan, 'response', new=FakeResponse) @patch.object(context, 'extract_context_from_environ') @@ -234,7 +235,7 @@ class RoutingControllerTest(unittest.TestCase): for routing in routings['routings']] expect = [('c7f641c9-8462-4007-84b2-3035d8cfb7a3', pod_id1), ('b669a2da-ca95-47db-a2a9-ba9e546d82ee', pod_id2)] - self.assertItemsEqual(expect, actual) + six.assertCountEqual(self, expect, actual) # apply a resource type filter to the retrieved routings. kw_filter1 = {'resource_type': 'port'} @@ -243,7 +244,7 @@ class RoutingControllerTest(unittest.TestCase): routing['resource_type']) for routing in routings['routings']] expect = [('b669a2da-ca95-47db-a2a9-ba9e546d82ee', pod_id2, 'port')] - self.assertItemsEqual(expect, actual) + six.assertCountEqual(self, expect, actual) # apply a filter and if it doesn't match with any of the retrieved # routings, then all of them will be discarded and the method returns @@ -259,7 +260,7 @@ class RoutingControllerTest(unittest.TestCase): for routing in routings['routings']] expect = [('c7f641c9-8462-4007-84b2-3035d8cfb7a3', pod_id1), ('b669a2da-ca95-47db-a2a9-ba9e546d82ee', pod_id2)] - self.assertItemsEqual(expect, actual) + six.assertCountEqual(self, expect, actual) # failure case, only admin can show all resource routings self.context.is_admin = False diff --git a/tricircle/tests/unit/db/test_api.py b/tricircle/tests/unit/db/test_api.py index d077ed3f..7b55eeb3 100644 --- a/tricircle/tests/unit/db/test_api.py +++ b/tricircle/tests/unit/db/test_api.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +from six.moves import xrange import unittest from tricircle.common import context diff --git a/tricircle/tests/unit/network/test_central_plugin.py b/tricircle/tests/unit/network/test_central_plugin.py index b99b3d35..674ee2f4 100644 --- a/tricircle/tests/unit/network/test_central_plugin.py +++ b/tricircle/tests/unit/network/test_central_plugin.py @@ -18,6 +18,8 @@ import copy import mock from mock import patch import netaddr +import six +from six.moves import xrange import unittest from sqlalchemy.orm import attributes @@ -217,7 +219,7 @@ class FakePool(driver.Pool): class DotDict(dict): def __init__(self, normal_dict=None): if normal_dict: - for key, value in normal_dict.iteritems(): + for key, value in six.iteritems(normal_dict): self[key] = value def __getattr__(self, item): @@ -255,7 +257,7 @@ class FakeNeutronClient(object): return_list = [] for port in port_list: is_selected = True - for key, value in params['filters'].iteritems(): + for key, value in six.iteritems(params['filters']): if key not in port or not port[key] or ( port[key] not in value): is_selected = False @@ -725,7 +727,7 @@ class FakeQuery(object): filtered_list = [] for record in self.records: selected = True - for key, value in kwargs.iteritems(): + for key, value in six.iteritems(kwargs): if key not in record or record[key] != value: selected = False break @@ -761,6 +763,8 @@ class FakeQuery(object): self.index += 1 return self.records[self.index - 1] + __next__ = next + def one(self): if len(self.records) == 0: raise exc.NoResultFound() @@ -774,7 +778,7 @@ class FakeQuery(object): def update(self, values): for record in self.records: - for key, value in values.iteritems(): + for key, value in six.iteritems(values): record[key] = value return len(self.records) @@ -1017,7 +1021,7 @@ class FakePlugin(plugin.TricirclePlugin): for allocation in TOP_IPALLOCATIONS: if allocation['port_id'] == port['id']: ret = {} - for key, value in port.iteritems(): + for key, value in six.iteritems(port): if key == 'fixed_ips': ret[key] = [{'subnet_id': allocation['subnet_id'], 'ip_address': allocation['ip_address']}] @@ -1215,10 +1219,10 @@ class PluginTest(unittest.TestCase, {'id': 'top_id_3', 'name': 'top'}] for _ports in (ports1, ports2, ports3, ports4): ports.extend(_ports) - self.assertItemsEqual(expected_ports, ports) + six.assertCountEqual(self, expected_ports, ports) ports = fake_plugin.get_ports(neutron_context) - self.assertItemsEqual(expected_ports, ports) + six.assertCountEqual(self, expected_ports, ports) @patch.object(context, 'get_context_from_neutron_context', new=fake_get_context_from_neutron_context) @@ -1267,7 +1271,7 @@ class PluginTest(unittest.TestCase, 'device_id': 'router_id'}, {'id': 'top_id_2', 'name': 'bottom', 'device_id': 'router_id'}] - self.assertItemsEqual(expected, ports) + six.assertCountEqual(self, expected, ports) @patch.object(context, 'get_context_from_neutron_context') @patch.object(db_base_plugin_v2.NeutronDbPluginV2, 'delete_port') @@ -1669,7 +1673,7 @@ class PluginTest(unittest.TestCase, # only one VLAN allocated since we just create one bridge network allocations = [ allocation['allocated'] for allocation in TOP_VLANALLOCATIONS] - self.assertItemsEqual([True, False], allocations) + six.assertCountEqual(self, [True, False], allocations) for segment in TOP_SEGMENTS: self.assertIn(segment['segmentation_id'], (2000, 2001)) diff --git a/tricircle/tests/unit/network/test_helper.py b/tricircle/tests/unit/network/test_helper.py index 0074cb21..f094cfe0 100644 --- a/tricircle/tests/unit/network/test_helper.py +++ b/tricircle/tests/unit/network/test_helper.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import six import unittest from oslo_utils import uuidutils @@ -42,14 +43,16 @@ class HelperTest(unittest.TestCase): } body = self.helper.get_create_subnet_body(project_id, t_subnet, b_net_id, '10.0.1.2') - self.assertItemsEqual([{'start': '10.0.1.3', 'end': '10.0.1.254'}], - body['subnet']['allocation_pools']) + six.assertCountEqual(self, + [{'start': '10.0.1.3', 'end': '10.0.1.254'}], + body['subnet']['allocation_pools']) self.assertEqual('10.0.1.2', body['subnet']['gateway_ip']) body = self.helper.get_create_subnet_body(project_id, t_subnet, b_net_id, '10.0.1.254') - self.assertItemsEqual([{'start': '10.0.1.2', 'end': '10.0.1.253'}], - body['subnet']['allocation_pools']) + six.assertCountEqual(self, + [{'start': '10.0.1.2', 'end': '10.0.1.253'}], + body['subnet']['allocation_pools']) self.assertEqual('10.0.1.254', body['subnet']['gateway_ip']) t_subnet['allocation_pools'] = [ @@ -57,8 +60,8 @@ class HelperTest(unittest.TestCase): {'start': '10.0.1.20', 'end': '10.0.1.254'}] body = self.helper.get_create_subnet_body(project_id, t_subnet, b_net_id, '10.0.1.5') - self.assertItemsEqual([{'start': '10.0.1.2', 'end': '10.0.1.4'}, - {'start': '10.0.1.6', 'end': '10.0.1.10'}, - {'start': '10.0.1.20', 'end': '10.0.1.254'}], - body['subnet']['allocation_pools']) + six.assertCountEqual(self, [{'start': '10.0.1.2', 'end': '10.0.1.4'}, + {'start': '10.0.1.6', 'end': '10.0.1.10'}, + {'start': '10.0.1.20', 'end': '10.0.1.254'}], + body['subnet']['allocation_pools']) self.assertEqual('10.0.1.5', body['subnet']['gateway_ip']) diff --git a/tricircle/tests/unit/network/test_local_plugin.py b/tricircle/tests/unit/network/test_local_plugin.py index 66c07bc5..7892cafc 100644 --- a/tricircle/tests/unit/network/test_local_plugin.py +++ b/tricircle/tests/unit/network/test_local_plugin.py @@ -287,19 +287,19 @@ class PluginTest(unittest.TestCase): def ip_to_digit(ip): return int(ip[ip.rindex('.') + 1:]) - pool_range = range(ip_to_digit(pool['start']), - ip_to_digit(pool['end']) + 1) - b_pool_range1 = range(ip_to_digit(b_pools[0]['start']), - ip_to_digit(b_pools[0]['end']) + 1) - b_pool_range2 = range(ip_to_digit(b_pools[1]['start']), - ip_to_digit(b_pools[1]['end']) + 1) + pool_range = list(range(ip_to_digit(pool['start']), + ip_to_digit(pool['end']) + 1)) + b_pool_range1 = list(range(ip_to_digit(b_pools[0]['start']), + ip_to_digit(b_pools[0]['end']) + 1)) + b_pool_range2 = list(range(ip_to_digit(b_pools[1]['start']), + ip_to_digit(b_pools[1]['end']) + 1)) b_pool_range = b_pool_range1 + [ ip_to_digit(b_gateway_ip)] + b_pool_range2 port.pop('name') b_port.pop('name') self.assertDictEqual(net, b_net) self.assertDictEqual(subnet, b_subnet) - self.assertEqual(pool_range, b_pool_range) + self.assertListEqual(pool_range, b_pool_range) self.assertEqual('vlan', b_net_type) self.assertDictEqual(port, b_port) diff --git a/tricircle/tests/unit/xjob/test_xmanager.py b/tricircle/tests/unit/xjob/test_xmanager.py index 67d80d0a..6b1b91c4 100644 --- a/tricircle/tests/unit/xjob/test_xmanager.py +++ b/tricircle/tests/unit/xjob/test_xmanager.py @@ -16,6 +16,8 @@ import datetime import mock from mock import patch +import six +from six.moves import xrange import unittest from oslo_config import cfg @@ -306,8 +308,8 @@ class XManagerTest(unittest.TestCase): ctx, router_id, routes_body = actual_call[0] expect_ctx, expect_routes = except_map[router_id] self.assertEqual(expect_ctx, ctx) - self.assertItemsEqual(expect_routes, - routes_body['router']['routes']) + six.assertCountEqual(self, expect_routes, + routes_body['router']['routes']) @patch.object(FakeClient, 'update_routers') def test_configure_extra_routes_with_floating_ips(self, mock_update): @@ -460,7 +462,7 @@ class XManagerTest(unittest.TestCase): jobs = core.query_resource(self.context, models.AsyncJob, [], []) expected_status = [constants.JS_New, constants.JS_Success] job_status = [job['status'] for job in jobs] - self.assertItemsEqual(expected_status, job_status) + six.assertCountEqual(self, expected_status, job_status) self.assertEqual(fake_id, jobs[0]['resource_id']) self.assertEqual(fake_id, jobs[1]['resource_id']) @@ -479,7 +481,7 @@ class XManagerTest(unittest.TestCase): jobs = core.query_resource(self.context, models.AsyncJob, [], []) expected_status = [constants.JS_New, constants.JS_Fail] job_status = [job['status'] for job in jobs] - self.assertItemsEqual(expected_status, job_status) + six.assertCountEqual(self, expected_status, job_status) self.assertEqual(fake_id, jobs[0]['resource_id']) self.assertEqual(fake_id, jobs[1]['resource_id']) @@ -507,7 +509,7 @@ class XManagerTest(unittest.TestCase): jobs = core.query_resource(self.context, models.AsyncJob, [], []) expected_status = ['New', 'Fail', 'Success'] job_status = [job['status'] for job in jobs] - self.assertItemsEqual(expected_status, job_status) + six.assertCountEqual(self, expected_status, job_status) for i in xrange(3): self.assertEqual(fake_id, jobs[i]['resource_id']) @@ -564,7 +566,7 @@ class XManagerTest(unittest.TestCase): expected_ids = ['job_uuid3', 'job_uuid5'] returned_jobs = db_api.get_latest_failed_jobs(self.context) actual_ids = [job['id'] for job in returned_jobs] - self.assertItemsEqual(expected_ids, actual_ids) + six.assertCountEqual(self, expected_ids, actual_ids) def tearDown(self): core.ModelBase.metadata.drop_all(core.get_engine()) diff --git a/tricircle/xjob/xmanager.py b/tricircle/xjob/xmanager.py index f3eaf803..9f5533b2 100644 --- a/tricircle/xjob/xmanager.py +++ b/tricircle/xjob/xmanager.py @@ -634,10 +634,10 @@ class XManager(PeriodicTasks): bottom_client.update_routers( ctx, b_router_id, {'router': {'routes': extra_routes}}) continue - for router_id, cidr_ips_map in router_ips_map.iteritems(): + for router_id, cidr_ips_map in six.iteritems(router_ips_map): if router_id == b_router_id: continue - for cidr, ips in cidr_ips_map.iteritems(): + for cidr, ips in six.iteritems(cidr_ips_map): if cidr in router_ips_map[b_router_id]: continue for ip in ips: @@ -666,10 +666,10 @@ class XManager(PeriodicTasks): # handle extra routes for north-south router ip_bridge_ip_map = {} - for router_id, cidr_ips_map in router_ips_map.iteritems(): + for router_id, cidr_ips_map in six.iteritems(router_ips_map): if router_id not in router_ew_bridge_ip_map: continue - for cidr, ips in cidr_ips_map.iteritems(): + for cidr, ips in six.iteritems(cidr_ips_map): for ip in ips: nexthop = router_ew_bridge_ip_map[router_id] destination = ip + '/32'