From b4f9798abfac3fed7482362c9e79625574b6c922 Mon Sep 17 00:00:00 2001 From: zhiyuan_cai Date: Mon, 16 Jan 2017 12:56:13 +0800 Subject: [PATCH] Shared VxLAN (Part3: tenant network l3) 1. What is the problem? VLAN network has some restrictions that VxLAN network doesn't have. For more flexible networking deployment, we consider supporting cross-pod VxLAN network. We are going to use shadow agent/port mechanism to synchronize VTEP information and make cross-pod VxLAN networking available, as discussed in the specification document[1]. With the previous parts[2, 3], we can create instances in the same VxLAN network but in different pods. With the help of shadow ports, tunnels are correctly created so instances can communicate with each other. But we have a problem during the association of a floating ip with an instance port because "setup_bottom_router" job will also create shadow port for floating ip association. 2. What is the solution to the problem? Let "setup_bottom_router" and "setup_shadow_ports" jobs call the same method to create shadow ports and handle conflict in that method. 3. What the features need to be implemented to the Tricircle to realize the solution? This is the third patch for cross-pod VxLAN networking support, which introduces the following changes: (1) Both creating floating ip and booting instance in vxlan network will create shadow port, so we leave shadow port deletion work to central plugin, it will delete shadow port when deleting instance port With this patch, floating ip binding to the port which is from cross-pod VxLAN network is supported. [1] https://review.openstack.org/#/c/429155/ [2] https://review.openstack.org/#/c/425128/ [3] https://review.openstack.org/#/c/425129/ Change-Id: I7ca3e124232baf265ec5a8ed3df0aca1303a2ff7 --- tricircle/common/constants.py | 6 +++ tricircle/network/central_plugin.py | 49 ++++++------------- tricircle/network/helper.py | 41 ++++++++++++++++ .../tests/unit/network/test_central_plugin.py | 4 +- tricircle/xjob/xmanager.py | 48 ++++++------------ 5 files changed, 79 insertions(+), 69 deletions(-) diff --git a/tricircle/common/constants.py b/tricircle/common/constants.py index 4aceb38a..5304901b 100644 --- a/tricircle/common/constants.py +++ b/tricircle/common/constants.py @@ -31,6 +31,12 @@ RT_ROUTER = 'router' RT_NS_ROUTER = 'ns_router' RT_SG = 'security_group' +REAL_SHADOW_TYPE_MAP = { + RT_NETWORK: RT_SD_NETWORK, + RT_SUBNET: RT_SD_SUBNET, + RT_PORT: RT_SD_PORT +} + # check whether the resource type is properly provisioned. def is_valid_resource_type(resource_type): diff --git a/tricircle/network/central_plugin.py b/tricircle/network/central_plugin.py index b3bf7e12..eb7baa01 100644 --- a/tricircle/network/central_plugin.py +++ b/tricircle/network/central_plugin.py @@ -310,19 +310,10 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, def delete_network(self, context, network_id): t_ctx = t_context.get_context_from_neutron_context(context) try: - mappings = db_api.get_bottom_mappings_by_top_id( - t_ctx, network_id, t_constants.RT_NETWORK) - mappings.extend(db_api.get_bottom_mappings_by_top_id( - t_ctx, network_id, t_constants.RT_SD_NETWORK)) - - processed_pod_set = set() - for mapping in mappings: - region_name = mapping[0]['region_name'] - if region_name in processed_pod_set: - continue - processed_pod_set.add(region_name) - bottom_network_id = mapping[1] - self._get_client(region_name).delete_networks( + for pod, bottom_network_id in ( + self.helper.get_real_shadow_resource_iterator( + t_ctx, t_constants.RT_NETWORK, network_id)): + self._get_client(pod['region_name']).delete_networks( t_ctx, bottom_network_id) # we do not specify resource_type when deleting routing entries # so if both "network" and "shadow_network" type entries exist @@ -334,7 +325,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, filters=[{'key': 'top_id', 'comparator': 'eq', 'value': network_id}, {'key': 'pod_id', 'comparator': 'eq', - 'value': mapping[0]['pod_id']}]) + 'value': pod['pod_id']}]) except Exception: raise with t_ctx.session.begin(): @@ -458,22 +449,14 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, def delete_subnet(self, context, subnet_id): t_ctx = t_context.get_context_from_neutron_context(context) try: - mappings = db_api.get_bottom_mappings_by_top_id( - t_ctx, subnet_id, t_constants.RT_SUBNET) - mappings.extend(db_api.get_bottom_mappings_by_top_id( - t_ctx, subnet_id, t_constants.RT_SD_SUBNET)) - - processed_pod_set = set() - for mapping in mappings: - region_name = mapping[0]['region_name'] - if region_name in processed_pod_set: - continue - processed_pod_set.add(region_name) - bottom_subnet_id = mapping[1] + for pod, bottom_subnet_id in ( + self.helper.get_real_shadow_resource_iterator( + t_ctx, t_constants.RT_SUBNET, subnet_id)): + region_name = pod['region_name'] self._get_client(region_name).delete_subnets( t_ctx, bottom_subnet_id) interface_name = t_constants.interface_port_name % ( - mapping[0]['region_name'], subnet_id) + region_name, subnet_id) self._delete_pre_created_port(t_ctx, context, interface_name) # we do not specify resource_type when deleting routing entries # so if both "subnet" and "shadow_subnet" type entries exist in @@ -485,7 +468,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, filters=[{'key': 'top_id', 'comparator': 'eq', 'value': subnet_id}, {'key': 'pod_id', 'comparator': 'eq', - 'value': mapping[0]['pod_id']}]) + 'value': pod['pod_id']}]) except Exception: raise dhcp_port_name = t_constants.dhcp_port_name % subnet_id @@ -730,13 +713,11 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, # we use a "delete_server_port" job to delete the local ports. if port.get('device_owner') not in NON_VM_PORT_TYPES: try: - mappings = db_api.get_bottom_mappings_by_top_id( - t_ctx, port_id, t_constants.RT_PORT) - if mappings: - pod_id = mappings[0][0]['pod_id'] - bottom_port_id = mappings[0][1] + for pod, bottom_port_id in ( + self.helper.get_real_shadow_resource_iterator( + t_ctx, t_constants.RT_PORT, port_id)): self.xjob_handler.delete_server_port(t_ctx, bottom_port_id, - pod_id) + pod['pod_id']) except Exception: raise with t_ctx.session.begin(): diff --git a/tricircle/network/helper.py b/tricircle/network/helper.py index fb8af2ff..273d3aa5 100644 --- a/tricircle/network/helper.py +++ b/tricircle/network/helper.py @@ -748,3 +748,44 @@ class NetworkHelper(object): agent_tunnel = profile[t_constants.PROFILE_TUNNEL_IP] db_api.ensure_agent_exists(t_ctx, pod['pod_id'], agent_host, agent_type, agent_tunnel) + + def prepare_shadow_port(self, ctx, project_id, target_pod, net_id, + port_body, agent=None): + host = port_body['binding:host_id'] + create_body = { + 'port': { + 'tenant_id': project_id, + 'admin_state_up': True, + 'name': t_constants.shadow_port_name % port_body['id'], + 'network_id': net_id, + 'fixed_ips': [{ + 'ip_address': port_body['fixed_ips'][0]['ip_address']}], + 'device_owner': t_constants.DEVICE_OWNER_SHADOW, + 'binding:host_id': host + } + } + if agent: + create_body['port'].update( + {'binding:profile': { + t_constants.PROFILE_AGENT_TYPE: agent['type'], + t_constants.PROFILE_TUNNEL_IP: agent['tunnel_ip']}}) + _, sw_port_id = self.prepare_bottom_element( + ctx, project_id, target_pod, {'id': port_body['id']}, + t_constants.RT_SD_PORT, create_body) + return sw_port_id + + @staticmethod + def get_real_shadow_resource_iterator(t_ctx, res_type, res_id): + shadow_res_type = t_constants.REAL_SHADOW_TYPE_MAP[res_type] + mappings = db_api.get_bottom_mappings_by_top_id( + t_ctx, res_id, res_type) + mappings.extend(db_api.get_bottom_mappings_by_top_id( + t_ctx, res_id, shadow_res_type)) + + processed_pod_set = set() + for pod, bottom_res_id in mappings: + region_name = pod['region_name'] + if region_name in processed_pod_set: + continue + processed_pod_set.add(region_name) + yield pod, bottom_res_id diff --git a/tricircle/tests/unit/network/test_central_plugin.py b/tricircle/tests/unit/network/test_central_plugin.py index 9b957ce8..480c2c9e 100644 --- a/tricircle/tests/unit/network/test_central_plugin.py +++ b/tricircle/tests/unit/network/test_central_plugin.py @@ -3082,7 +3082,9 @@ class PluginTest(unittest.TestCase, 'fixed_ips': [ {'subnet_id': db_api.get_bottom_id_by_top_id_region_name( t_ctx, t_subnet_id, 'pod_1', constants.RT_SUBNET), - 'ip_address': '10.0.0.4'}] + 'ip_address': '10.0.0.4'}], + 'binding:host_id': 'host_1', + 'binding:vif_type': 'ovs' } TOP_PORTS.append(t_port) BOTTOM1_PORTS.append(b_port) diff --git a/tricircle/xjob/xmanager.py b/tricircle/xjob/xmanager.py index 5f5e3c50..43206c28 100644 --- a/tricircle/xjob/xmanager.py +++ b/tricircle/xjob/xmanager.py @@ -423,20 +423,16 @@ class XManager(PeriodicTasks): # ip association purpose t_int_net_id = t_int_port['network_id'] t_int_subnet_id = t_int_port['fixed_ips'][0]['subnet_id'] - # TODO(zhiyuan) adapt shadow agent way to create shadow port - port_body = { - 'port': { - 'tenant_id': project_id, - 'admin_state_up': True, - 'name': constants.shadow_port_name % t_int_port['id'], - 'network_id': t_int_net_id, - 'fixed_ips': [{'ip_address': t_int_port[ - 'fixed_ips'][0]['ip_address']}] - } - } - self.helper.prepare_bottom_element( - ctx, project_id, b_ext_pod, t_int_port, - constants.RT_SD_PORT, port_body) + + b_int_port = b_client.get_ports(ctx, b_int_port_id) + host = b_int_port['binding:host_id'] + agent_type = self.helper.get_agent_type_by_vif( + b_int_port['binding:vif_type']) + agent = db_api.get_agent_by_host_type(ctx, host, agent_type) + self.helper.prepare_shadow_port( + ctx, project_id, b_ext_pod, t_int_net_id, + b_int_port, agent) + # create routing entries for shadow network and subnet so we # can easily find them during central network and subnet # deletion, create_resource_mapping will catch DBDuplicateEntry @@ -539,7 +535,8 @@ class XManager(PeriodicTasks): q_constants.DEVICE_OWNER_DVR_INTERFACE, q_constants.DEVICE_OWNER_ROUTER_SNAT, q_constants.DEVICE_OWNER_ROUTER_GW, - q_constants.DEVICE_OWNER_DHCP] + q_constants.DEVICE_OWNER_DHCP, + constants.DEVICE_OWNER_SHADOW] ew_attached_port_types = [q_constants.DEVICE_OWNER_ROUTER_INTF, q_constants.DEVICE_OWNER_DVR_INTERFACE, q_constants.DEVICE_OWNER_ROUTER_GW] @@ -992,31 +989,14 @@ class XManager(PeriodicTasks): continue agent_info_map[key] = agent - create_body = { - 'port': { - 'tenant_id': project_id, - 'admin_state_up': True, - 'name': constants.shadow_port_name % port_id, - 'network_id': t_net_id, - 'fixed_ips': [{ - 'ip_address': port_body[ - 'fixed_ips'][0]['ip_address']}], - 'device_owner': constants.DEVICE_OWNER_SHADOW, - 'binding:host_id': host, - 'binding:profile': { - constants.PROFILE_AGENT_TYPE: agent_type, - constants.PROFILE_TUNNEL_IP: agent['tunnel_ip']} - } - } + sw_port_id = self.helper.prepare_shadow_port( + ctx, project_id, target_pod, t_net_id, port_body, agent) # value for key constants.PROFILE_FORCE_UP does not matter update_body = { 'port': { 'binding:profile': {constants.PROFILE_FORCE_UP: 'True'} } } - _, sw_port_id = self.helper.prepare_bottom_element( - ctx, project_id, target_pod, {'id': port_id}, - constants.RT_SD_PORT, create_body) self._get_client(target_pod['region_name']).update_ports( ctx, sw_port_id, update_body)