diff --git a/devstack/local.conf.node_2.sample b/devstack/local.conf.node_2.sample index 1525b997..50826876 100644 --- a/devstack/local.conf.node_2.sample +++ b/devstack/local.conf.node_2.sample @@ -70,6 +70,7 @@ TRICIRCLE_NEUTRON_PORT=20001 [DEFAULT] core_plugin=tricircle.network.local_plugin.TricirclePlugin +service_plugins=tricircle.network.local_l3_plugin.TricircleL3Plugin [client] admin_username=admin diff --git a/releasenotes/notes/combine-bridge-network-c137a03f067c49a7.yaml b/releasenotes/notes/combine-bridge-network-c137a03f067c49a7.yaml new file mode 100644 index 00000000..864e4aee --- /dev/null +++ b/releasenotes/notes/combine-bridge-network-c137a03f067c49a7.yaml @@ -0,0 +1,4 @@ +--- +features: + - North-south bridge network and east-west bridge network are combined into + one to bring better DVR and shared VxLAN network support. diff --git a/tricircle/common/client.py b/tricircle/common/client.py index fb79e9cc..5c260877 100644 --- a/tricircle/common/client.py +++ b/tricircle/common/client.py @@ -64,12 +64,9 @@ client_opts = [ default='Default', help='tenant domain name of admin account, needed when' ' auto_refresh_endpoint set to True'), - cfg.StrOpt('ew_bridge_cidr', + cfg.StrOpt('bridge_cidr', default='100.0.0.0/9', - help='cidr pool of the east-west bridge network'), - cfg.StrOpt('ns_bridge_cidr', - default='100.128.0.0/9', - help='cidr pool of the north-south bridge network') + help='cidr pool of the bridge network') ] client_opt_group = cfg.OptGroup('client') cfg.CONF.register_group(client_opt_group) diff --git a/tricircle/common/constants.py b/tricircle/common/constants.py index 439dc239..4e803a90 100644 --- a/tricircle/common/constants.py +++ b/tricircle/common/constants.py @@ -30,9 +30,13 @@ RT_VOl_METADATA = 'volume_metadata' RT_BACKUP = 'backup' RT_SNAPSHOT = 'snapshot' RT_NETWORK = 'network' +RT_SD_NETWORK = 'shadow_network' RT_SUBNET = 'subnet' +RT_SD_SUBNET = 'shadow_subnet' RT_PORT = 'port' +RT_SD_PORT = 'shadow_port' RT_ROUTER = 'router' +RT_NS_ROUTER = 'ns_router' RT_SG = 'security_group' @@ -52,23 +56,23 @@ R_LIBERTY = 'liberty' R_MITAKA = 'mitaka' # l3 bridge networking elements -ew_bridge_subnet_pool_name = 'ew_bridge_subnet_pool' -ew_bridge_net_name = 'ew_bridge_net_%s' # project_id -ew_bridge_subnet_name = 'ew_bridge_subnet_%s' # project_id -ew_bridge_port_name = 'ew_bridge_port_%s_%s' # project_id b_router_id +bridge_subnet_pool_name = 'bridge_subnet_pool' +bridge_net_name = 'bridge_net_%s' # project_id +bridge_subnet_name = 'bridge_subnet_%s' # project_id +bridge_port_name = 'bridge_port_%s_%s' # project_id b_router_id -ns_bridge_subnet_pool_name = 'ns_bridge_subnet_pool' -ns_bridge_net_name = 'ns_bridge_net_%s' # project_id -ns_bridge_subnet_name = 'ns_bridge_subnet_%s' # project_id # for external gateway port: project_id b_router_id None # for floating ip port: project_id None b_internal_port_id ns_bridge_port_name = 'ns_bridge_port_%s_%s_%s' +ns_router_name = 'ns_router_%s' +shadow_port_name = 'shadow_port_%s' dhcp_port_name = 'dhcp_port_%s' # subnet_id interface_port_name = 'interface_%s_%s' # b_pod_id t_subnet_id interface_port_device_id = 'reserved_gateway_port' MAX_INT = 0x7FFFFFFF +DEFAULT_DESTINATION = '0.0.0.0/0' expire_time = datetime.datetime(2000, 1, 1) # job status diff --git a/tricircle/common/lock_handle.py b/tricircle/common/lock_handle.py index d580b8d2..3fe78755 100644 --- a/tricircle/common/lock_handle.py +++ b/tricircle/common/lock_handle.py @@ -40,7 +40,9 @@ def get_or_create_route(t_ctx, q_ctx, t_ctx, models.ResourceRouting, [{'key': 'top_id', 'comparator': 'eq', 'value': _id}, {'key': 'pod_id', 'comparator': 'eq', - 'value': pod['pod_id']}], []) + 'value': pod['pod_id']}, + {'key': 'resource_type', 'comparator': 'eq', + 'value': _type}], []) if routes: route = routes[0] if route['bottom_id']: @@ -109,7 +111,7 @@ def get_or_create_element(t_ctx, q_ctx, core.delete_resource(t_ctx, models.ResourceRouting, route['id']) - except db_exc.ResourceNotFound: + except db_exc.ColumnError: # NOTE(zhiyuan) this is a rare case that other worker # considers the route expires and delete it though it # was just created, maybe caused by out-of-sync time diff --git a/tricircle/db/models.py b/tricircle/db/models.py index ccec8224..98556247 100644 --- a/tricircle/db/models.py +++ b/tricircle/db/models.py @@ -60,8 +60,8 @@ class ResourceRouting(core.ModelBase, core.DictBase, models.TimestampMixin): __tablename__ = 'resource_routings' __table_args__ = ( schema.UniqueConstraint( - 'top_id', 'pod_id', - name='resource_routings0top_id0pod_id'), + 'top_id', 'pod_id', 'resource_type', + name='resource_routings0top_id0pod_id0resource_type'), ) attributes = ['id', 'top_id', 'bottom_id', 'pod_id', 'project_id', 'resource_type', 'created_at', 'updated_at'] diff --git a/tricircle/network/central_plugin.py b/tricircle/network/central_plugin.py index 3e322f97..a3969bfe 100644 --- a/tricircle/network/central_plugin.py +++ b/tricircle/network/central_plugin.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import collections import copy from oslo_config import cfg @@ -274,11 +275,22 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, try: mappings = db_api.get_bottom_mappings_by_top_id( t_ctx, network_id, t_constants.RT_NETWORK) + mappings.extend(db_api.get_bottom_mappings_by_top_id( + t_ctx, network_id, t_constants.RT_SD_NETWORK)) + + processed_pod_set = set() for mapping in mappings: region_name = mapping[0]['region_name'] + if region_name in processed_pod_set: + continue + processed_pod_set.add(region_name) bottom_network_id = mapping[1] self._get_client(region_name).delete_networks( t_ctx, bottom_network_id) + # we do not specify resource_type when deleting routing entries + # so if both "network" and "shadow_network" type entries exist + # in one pod(this is possible for cross-pod network), we delete + # them at the same time with t_ctx.session.begin(): core.delete_resources( t_ctx, models.ResourceRouting, @@ -386,14 +398,25 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, try: mappings = db_api.get_bottom_mappings_by_top_id( t_ctx, subnet_id, t_constants.RT_SUBNET) + mappings.extend(db_api.get_bottom_mappings_by_top_id( + t_ctx, subnet_id, t_constants.RT_SD_SUBNET)) + + processed_pod_set = set() for mapping in mappings: region_name = mapping[0]['region_name'] + if region_name in processed_pod_set: + continue + processed_pod_set.add(region_name) bottom_subnet_id = mapping[1] self._get_client(region_name).delete_subnets( t_ctx, bottom_subnet_id) interface_name = t_constants.interface_port_name % ( mapping[0]['region_name'], subnet_id) self._delete_pre_created_port(t_ctx, context, interface_name) + # we do not specify resource_type when deleting routing entries + # so if both "subnet" and "shadow_subnet" type entries exist in + # one pod(this is possible for cross-pod network), we delete + # them at the same time with t_ctx.session.begin(): core.delete_resources( t_ctx, models.ResourceRouting, @@ -473,6 +496,9 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, self.xjob_handler.setup_bottom_router( admin_context, res['network_id'], interfaces[0]['device_id'], pod['pod_id']) + else: + LOG.debug('Update port: no interfaces found, xjob not' + 'triggered') self.xjob_handler.configure_security_group_rules(t_ctx, res['tenant_id']) @@ -661,10 +687,18 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, id_list.append(_id) return id_list + @staticmethod + def _filter_shadow_port(ports, pod_id, port_pod_map): + port_list = [] + for port in ports: + if pod_id not in port_pod_map[port['id']]: + port_list.append(port) + return port_list + def _get_ports_from_pod_with_number(self, context, current_pod, number, last_port_id, bottom_top_map, top_bottom_map, - filters=None): + port_pod_map, filters=None): # NOTE(zhiyuan) last_port_id is top id, also id in returned port dict # also uses top id. when interacting with bottom pod, need to map # top to bottom in request and map bottom to top in response @@ -678,7 +712,8 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, for key, value in _filters: if key == 'fixed_ips': if 'ip_address' in value: - _filters[key] = 'ip_address=%s' % value['ip_address'] + _filters[key] = 'ip_address=%s' % value[ + 'ip_address'][0] continue id_list = self._get_map_filter_ids( key, value, current_pod['pod_id'], top_bottom_map) @@ -689,8 +724,10 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, # map top id to bottom id in request params['marker'] = top_bottom_map[last_port_id] res = q_client.get(q_client.ports_path, params=params) + ports = self._filter_shadow_port(res['ports'], current_pod['pod_id'], + port_pod_map) # map bottom id to top id in client response - mapped_port_list = self._map_ports_from_bottom_to_top(res['ports'], + mapped_port_list = self._map_ports_from_bottom_to_top(ports, bottom_top_map) del res['ports'] res['ports'] = mapped_port_list @@ -712,7 +749,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, # need to map next_res = self._get_ports_from_pod_with_number( context, next_pod, number - len(res['ports']), '', - bottom_top_map, top_bottom_map, filters) + bottom_top_map, top_bottom_map, port_pod_map, filters) next_res['ports'].extend(res['ports']) return next_res @@ -780,6 +817,16 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, key = '%s_%s' % (route['pod_id'], route['top_id']) top_bottom_map[key] = route['bottom_id'] + port_pod_map = collections.defaultdict(set) + route_filters = [{'key': 'resource_type', + 'comparator': 'eq', + 'value': t_constants.RT_SD_PORT}] + routes = core.query_resource(t_ctx, models.ResourceRouting, + route_filters, []) + for route in routes: + if route['bottom_id']: + port_pod_map[route['bottom_id']].add(route['pod_id']) + if limit: if marker: mappings = db_api.get_bottom_mappings_by_top_id( @@ -791,7 +838,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, current_pod = db_api.get_pod(t_ctx, pod_id) res = self._get_ports_from_pod_with_number( context, current_pod, limit, marker, - bottom_top_map, top_bottom_map, filters) + bottom_top_map, top_bottom_map, port_pod_map, filters) else: res = self._get_ports_from_top_with_number( context, limit, marker, top_bottom_map, filters) @@ -802,7 +849,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, if current_pod: res = self._get_ports_from_pod_with_number( context, current_pod, limit, '', - bottom_top_map, top_bottom_map, filters) + bottom_top_map, top_bottom_map, port_pod_map, filters) else: res = self._get_ports_from_top_with_number( context, limit, marker, top_bottom_map, filters) @@ -829,7 +876,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, _filters.append( {'key': key, 'comparator': 'eq', 'value': 'ip_address=%s' % value[ - 'ip_address']}) + 'ip_address'][0]}) continue id_list = self._get_map_filter_ids( key, value, pod['pod_id'], top_bottom_map) @@ -842,7 +889,9 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, 'comparator': 'eq', 'value': value}) client = self._get_client(pod['region_name']) - ret.extend(client.list_ports(t_ctx, filters=_filters)) + ports = client.list_ports(t_ctx, filters=_filters) + ret.extend(self._filter_shadow_port(ports, pod['pod_id'], + port_pod_map)) ret = self._map_ports_from_bottom_to_top(ret, bottom_top_map) ret.extend(self._get_ports_from_top(context, top_bottom_map, filters)) @@ -854,6 +903,11 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, def _delete_top_bridge_resource(self, t_ctx, q_ctx, resource_type, resource_id, resource_name): + # first we update the routing entry to clear bottom_id and expire the + # entry, if we succeed to delete the bridge resource next, we continue + # to delete this expired entry; otherwise, we fail to delete the bridge + # resource, then when the resource is accessed via lock_handle module, + # that module will find the resource and update the entry with t_ctx.session.begin(): core.update_resources( t_ctx, models.ResourceRouting, @@ -877,26 +931,20 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, def _delete_top_bridge_network_subnet(self, t_ctx, q_ctx): project_id = t_ctx.project_id - ew_bridge_subnet_name = t_constants.ew_bridge_subnet_name % project_id - ns_bridge_subnet_name = t_constants.ns_bridge_subnet_name % project_id - subnet_names = [ew_bridge_subnet_name, ns_bridge_subnet_name] - for subnet_name in subnet_names: - bridge_subnets = super(TricirclePlugin, self).get_subnets( - q_ctx, {'name': [subnet_name]}) - if bridge_subnets: - self._delete_top_bridge_resource( - t_ctx, q_ctx, t_constants.RT_SUBNET, - bridge_subnets[0]['id'], subnet_name) - ew_bridge_net_name = t_constants.ew_bridge_net_name % project_id - ns_bridge_net_name = t_constants.ns_bridge_net_name % project_id - net_names = [ew_bridge_net_name, ns_bridge_net_name] - for net_name in net_names: - bridge_nets = super(TricirclePlugin, self).get_networks( - q_ctx, {'name': [net_name]}) - if bridge_nets: - self._delete_top_bridge_resource( - t_ctx, q_ctx, t_constants.RT_NETWORK, bridge_nets[0]['id'], - net_name) + bridge_subnet_name = t_constants.bridge_subnet_name % project_id + bridge_subnets = super(TricirclePlugin, self).get_subnets( + q_ctx, {'name': [bridge_subnet_name]}) + if bridge_subnets: + self._delete_top_bridge_resource( + t_ctx, q_ctx, t_constants.RT_SUBNET, + bridge_subnets[0]['id'], bridge_subnet_name) + bridge_net_name = t_constants.bridge_net_name % project_id + bridge_nets = super(TricirclePlugin, self).get_networks( + q_ctx, {'name': [bridge_net_name]}) + if bridge_nets: + self._delete_top_bridge_resource( + t_ctx, q_ctx, t_constants.RT_NETWORK, bridge_nets[0]['id'], + bridge_net_name) def _delete_top_bridge_port(self, t_ctx, q_ctx, bridge_port_id, bridge_port_name): @@ -912,65 +960,46 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, t_constants.RT_ROUTER) for pod, b_router_id in mappings: b_client = self._get_client(pod['region_name']) + bridge_port_name = t_constants.bridge_port_name % (project_id, + b_router_id) + bridge_ports = super(TricirclePlugin, self).get_ports( + context, {'name': [bridge_port_name]}) + if bridge_ports: + t_ns_port_id = bridge_ports[0]['id'] + b_client.action_routers(t_ctx, 'remove_gateway', b_router_id) + self._delete_top_bridge_port(t_ctx, context, t_ns_port_id, + bridge_port_name) + b_client.delete_routers(t_ctx, b_router_id) + db_api.delete_mappings_by_bottom_id(t_ctx, b_router_id) - ew_port_name = t_constants.ew_bridge_port_name % (project_id, - b_router_id) - ew_ports = super(TricirclePlugin, self).get_ports( - context, {'name': [ew_port_name]}) - if ew_ports: - t_ew_port_id = ew_ports[0]['id'] - b_ew_port_id = db_api.get_bottom_id_by_top_id_region_name( - t_ctx, t_ew_port_id, pod['region_name'], - t_constants.RT_PORT) - if b_ew_port_id: - request_body = {'port_id': b_ew_port_id} + mappings = db_api.get_bottom_mappings_by_top_id( + t_ctx, _id, t_constants.RT_NS_ROUTER) + for pod, b_ns_router_id in mappings: + b_client = self._get_client(pod['region_name']) + bridge_subnet_name = t_constants.bridge_subnet_name % project_id + bridge_subnets = super(TricirclePlugin, + self).get_subnets( + context, {'name': [bridge_subnet_name]}) + if bridge_subnets: + t_bridge_subnet_id = bridge_subnets[0]['id'] + b_bridge_subnet_id = \ + db_api.get_bottom_id_by_top_id_region_name( + t_ctx, t_bridge_subnet_id, pod['region_name'], + t_constants.RT_SUBNET) + if b_bridge_subnet_id: + request_body = {'subnet_id': b_bridge_subnet_id} try: b_client.action_routers(t_ctx, 'remove_interface', - b_router_id, request_body) + b_ns_router_id, request_body) except Exception as e: if e.status_code == 404: # 404 error means that the router interface has # been already detached, skip this exception pass raise - db_api.delete_mappings_by_top_id(t_ctx, t_ew_port_id) - self._delete_top_bridge_port(t_ctx, context, t_ew_port_id, - ew_port_name) - ns_port_name = t_constants.ns_bridge_port_name % ( - project_id, b_router_id, None) - ns_ports = super(TricirclePlugin, self).get_ports( - context, {'name': [ns_port_name]}) - if ns_ports: - t_ns_port_id = ns_ports[0]['id'] - b_client.action_routers(t_ctx, 'remove_gateway', b_router_id) - self._delete_top_bridge_port(t_ctx, context, t_ns_port_id, - ns_port_name) - else: - ns_subnet_name = t_constants.ns_bridge_subnet_name % project_id - ns_subnets = super(TricirclePlugin, - self).get_subnets( - context, {'name': [ns_subnet_name]}) - if ns_subnets: - t_ns_subnet_id = ns_subnets[0]['id'] - b_ns_subnet_id = \ - db_api.get_bottom_id_by_top_id_region_name( - t_ctx, t_ns_subnet_id, pod['region_name'], - t_constants.RT_SUBNET) - if b_ns_subnet_id: - request_body = {'subnet_id': b_ns_subnet_id} - try: - b_client.action_routers(t_ctx, 'remove_interface', - b_router_id, request_body) - except Exception as e: - if e.status_code == 404: - # 404 error means that the router interface has - # been already detached, skip this exception - pass - raise - - b_client.delete_routers(t_ctx, b_router_id) - db_api.delete_mappings_by_bottom_id(t_ctx, b_router_id) + b_client.delete_routers(t_ctx, b_ns_router_id) + db_api.delete_mappings_by_bottom_id(t_ctx, b_ns_router_id) routers = super(TricirclePlugin, self).get_routers( context, {'tenant_id': [project_id]}) @@ -989,13 +1018,9 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, return self.helper.prepare_bottom_element( t_ctx, project_id, pod, ele, _type, body) - def _get_bridge_subnet_pool_id(self, t_ctx, q_ctx, project_id, pod, is_ew): - if is_ew: - pool_name = t_constants.ew_bridge_subnet_pool_name - pool_cidr = cfg.CONF.client.ew_bridge_cidr - else: - pool_name = t_constants.ns_bridge_subnet_pool_name - pool_cidr = cfg.CONF.client.ns_bridge_cidr + def _get_bridge_subnet_pool_id(self, t_ctx, q_ctx, project_id, pod): + pool_name = t_constants.bridge_subnet_pool_name + pool_cidr = cfg.CONF.client.bridge_cidr pool_ele = {'id': pool_name} body = {'subnetpool': {'tenant_id': project_id, 'name': pool_name, @@ -1012,17 +1037,11 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, return pool_id def _get_bridge_network_subnet(self, t_ctx, q_ctx, project_id, pod, - pool_id, is_ew): - if is_ew: - net_name = t_constants.ew_bridge_net_name % project_id - net_ele = {'id': net_name} - subnet_name = t_constants.ew_bridge_subnet_name % project_id - subnet_ele = {'id': subnet_name} - else: - net_name = t_constants.ns_bridge_net_name % project_id - net_ele = {'id': net_name} - subnet_name = t_constants.ns_bridge_subnet_name % project_id - subnet_ele = {'id': subnet_name} + pool_id): + net_name = t_constants.bridge_net_name % project_id + net_ele = {'id': net_name} + subnet_name = t_constants.bridge_subnet_name % project_id + subnet_ele = {'id': subnet_name} is_admin = q_ctx.is_admin q_ctx.is_admin = True @@ -1063,10 +1082,9 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, return net, subnet def _get_bridge_interface(self, t_ctx, q_ctx, project_id, pod, - t_net_id, b_router_id, b_port_id, is_ew): + t_net_id, b_router_id): port_id = self.helper.get_bridge_interface(t_ctx, q_ctx, project_id, - pod, t_net_id, b_router_id, - b_port_id, is_ew) + pod, t_net_id, b_router_id) return super(TricirclePlugin, self).get_port(q_ctx, port_id) def _get_bottom_bridge_elements(self, q_ctx, project_id, @@ -1125,11 +1143,11 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, # TODO(zhiyuan) decide router is distributed or not from pod table # currently "distributed" is set to False, should add a metadata field # to pod table, and decide distributed or not from the metadata later - body = {'router': {'name': router_id, + body = {'router': {'name': t_constants.ns_router_name % router_id, 'distributed': False}} _, b_router_id = self._prepare_bottom_element( t_ctx, t_router['tenant_id'], pod, t_router, - t_constants.RT_ROUTER, body) + t_constants.RT_NS_ROUTER, body) # both router and external network in bottom pod are ready, attach # external network to router in bottom pod. @@ -1156,13 +1174,12 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, # bridge network. t_pod = db_api.get_top_pod(t_ctx) project_id = t_router['tenant_id'] - pool_id = self._get_bridge_subnet_pool_id( - t_ctx, context, None, t_pod, False) + pool_id = self._get_bridge_subnet_pool_id(t_ctx, context, None, t_pod) t_bridge_net, t_bridge_subnet = self._get_bridge_network_subnet( - t_ctx, context, project_id, t_pod, pool_id, False) + t_ctx, context, project_id, t_pod, pool_id) (_, _, b_bridge_subnet_id, b_bridge_net_id) = self._get_bottom_bridge_elements( - context, project_id, pod, t_bridge_net, False, t_bridge_subnet, + context, project_id, pod, t_bridge_net, True, t_bridge_subnet, None) # here we attach the bridge network to the router in bottom pod. to @@ -1181,6 +1198,8 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, is_attach = _is_bridge_network_attached() if not is_attach: + # no need to explicitly create the top bridge port, the ip reserved + # for router interface will be used. b_client.action_routers(t_ctx, 'add_interface', b_router_id, {'subnet_id': b_bridge_subnet_id}) @@ -1199,7 +1218,7 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, region_name = t_network[az_ext.AZ_HINTS][0] b_router_id = db_api.get_bottom_id_by_top_id_region_name( - t_ctx, router_id, region_name, t_constants.RT_ROUTER) + t_ctx, router_id, region_name, t_constants.RT_NS_ROUTER) b_client = self._get_client(region_name) b_client.action_routers(t_ctx, 'remove_gateway', b_router_id) @@ -1233,12 +1252,15 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, context, router_id, router) router_data[l3.EXTERNAL_GW_INFO].update(ret[l3.EXTERNAL_GW_INFO]) self._add_router_gateway(context, router_id, router_data) - return ret else: self._remove_router_gateway(context, router_id) - return super(TricirclePlugin, self).update_router( + ret = super(TricirclePlugin, self).update_router( context, router_id, router) + t_ctx = t_context.get_context_from_neutron_context(context) + self.xjob_handler.configure_extra_routes(t_ctx, router_id) + return ret + def add_router_interface(self, context, router_id, interface_info): t_ctx = t_context.get_context_from_neutron_context(context) @@ -1251,33 +1273,17 @@ class TricirclePlugin(db_base_plugin_v2.NeutronDbPluginV2, t_pod = db_api.get_top_pod(t_ctx) assert t_pod - # bridge network for E-W networking + # bridge network for E-W and N-S networking pool_id = self._get_bridge_subnet_pool_id( - t_ctx, context, None, t_pod, True) + t_ctx, context, None, t_pod) self._get_bridge_network_subnet( - t_ctx, context, project_id, t_pod, pool_id, True) - - # bridge network for N-S networking - ext_nets = self.get_networks(context, {external_net.EXTERNAL: [True]}) - if not ext_nets: - need_ns_bridge = False - else: - ext_net_region_names = set( - [ext_net[az_ext.AZ_HINTS][0] for ext_net in ext_nets]) - need_ns_bridge = False - for b_pod in b_pods: - if b_pod['region_name'] not in ext_net_region_names: - need_ns_bridge = True - break - if need_ns_bridge: - pool_id = self._get_bridge_subnet_pool_id( - t_ctx, context, None, t_pod, False) - self._get_bridge_network_subnet( - t_ctx, context, project_id, t_pod, pool_id, False) + t_ctx, context, project_id, t_pod, pool_id) return_info = super(TricirclePlugin, self).add_router_interface( context, router_id, interface_info) if not b_pods: + LOG.debug('Add router interface: no interfaces found, xjob not' + 'triggered') return return_info try: if len(b_pods) == 1: diff --git a/tricircle/network/helper.py b/tricircle/network/helper.py index 51fe3f34..7a9fcedb 100644 --- a/tricircle/network/helper.py +++ b/tricircle/network/helper.py @@ -117,7 +117,7 @@ class NetworkHelper(object): t_ctx, q_ctx, project_id, pod, ele, _type, body) def get_bridge_interface(self, t_ctx, q_ctx, project_id, pod, - t_net_id, b_router_id, b_port_id, is_ew): + t_net_id, b_router_id): """Get or create top bridge interface :param t_ctx: tricircle context @@ -126,19 +126,10 @@ class NetworkHelper(object): :param pod: dict of top pod :param t_net_id: top bridge network id :param b_router_id: bottom router id - :param b_port_id: needed when creating bridge interface for south- - north network, id of the internal port bound to floating ip - :param is_ew: create the bridge interface for east-west network or - south-north network :return: bridge interface id """ - if is_ew: - port_name = t_constants.ew_bridge_port_name % (project_id, - b_router_id) - else: - port_name = t_constants.ns_bridge_port_name % (project_id, - b_router_id, - b_port_id) + port_name = t_constants.bridge_port_name % (project_id, + b_router_id) port_ele = {'id': port_name} port_body = { 'port': { @@ -174,7 +165,13 @@ class NetworkHelper(object): """ def list_resources(t_ctx_, q_ctx, pod_, ele_, _type_): client = self._get_client(pod_['region_name']) - if _type_ == t_constants.RT_NETWORK: + if _type_ == t_constants.RT_NS_ROUTER: + _type_ = t_constants.RT_ROUTER + value = t_constants.ns_router_name % ele_['id'] + elif _type_ == t_constants.RT_SD_PORT: + _type_ = t_constants.RT_PORT + value = t_constants.shadow_port_name % ele_['id'] + elif _type_ == t_constants.RT_NETWORK: value = utils.get_bottom_network_name(ele_) else: value = ele_['id'] @@ -183,6 +180,10 @@ class NetworkHelper(object): 'value': value}]) def create_resources(t_ctx_, q_ctx, pod_, body_, _type_): + if _type_ == t_constants.RT_NS_ROUTER: + _type_ = t_constants.RT_ROUTER + elif _type_ == t_constants.RT_SD_PORT: + _type_ = t_constants.RT_PORT client = self._get_client(pod_['region_name']) return client.create_resources(_type_, t_ctx_, body_) @@ -421,14 +422,6 @@ class NetworkHelper(object): 'cidr': t_subnet['cidr'], 'enable_dhcp': False, 'tenant_id': project_id}} - # In the pod hosting external network, where ns bridge network is used - # as an internal network, need to allocate ip address from .3 because - # .2 is used by the router gateway port in the pod hosting servers, - # where ns bridge network is used as an external network. - # if t_subnet['name'].startswith('ns_bridge_') and not is_external: - # prefix = t_subnet['cidr'][:t_subnet['cidr'].rindex('.')] - # subnet_body['subnet']['allocation_pools'] = [ - # {'start': prefix + '.3', 'end': prefix + '.254'}] _, b_subnet_id = self.prepare_bottom_element( t_ctx, project_id, pod, t_subnet, 'subnet', subnet_body) diff --git a/tricircle/network/local_l3_plugin.py b/tricircle/network/local_l3_plugin.py new file mode 100644 index 00000000..6c0143d5 --- /dev/null +++ b/tricircle/network/local_l3_plugin.py @@ -0,0 +1,60 @@ +# Copyright 2015 Huawei Technologies Co., Ltd. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import orm + +from neutron_lib import constants + +from neutron.db.models import l3 as l3_models +from neutron.db import models_v2 +from neutron.extensions import l3 +from neutron.services.l3_router import l3_router_plugin + + +class TricircleL3Plugin(l3_router_plugin.L3RouterPlugin): + # Override the original implementation to allow associating a floating ip + # to a port whose network is not attached to the router. Tricircle will + # configures extra routes to guarantee packets can reach the port. + def get_router_for_floatingip(self, context, internal_port, + internal_subnet, external_network_id): + """Find a router to handle the floating-ip association. + + :param internal_port: The port for the fixed-ip. + :param internal_subnet: The subnet for the fixed-ip. + :param external_network_id: The external network for floating-ip. + + :raises: ExternalGatewayForFloatingIPNotFound if no suitable router + is found. + """ + router_port = l3_models.RouterPort + gw_port = orm.aliased(models_v2.Port, name="gw_port") + router_port_qry = context.session.query( + router_port.router_id + ).join(gw_port, gw_port.device_id == router_port.router_id).filter( + gw_port.network_id == external_network_id, + gw_port.device_owner == constants.DEVICE_OWNER_ROUTER_GW + ).distinct() + + first_router_id = None + for router in router_port_qry: + if not first_router_id: + first_router_id = router.router_id + if first_router_id: + return first_router_id + + raise l3.ExternalGatewayForFloatingIPNotFound( + subnet_id=internal_subnet['id'], + external_network_id=external_network_id, + port_id=internal_port['id']) diff --git a/tricircle/network/local_plugin.py b/tricircle/network/local_plugin.py index ae0f7496..e574e6d8 100644 --- a/tricircle/network/local_plugin.py +++ b/tricircle/network/local_plugin.py @@ -480,10 +480,15 @@ class TricirclePlugin(plugin.Ml2Plugin): t_ctx = t_context.get_context_from_neutron_context(context) try: b_port = self.core_plugin.get_port(context, _id) + # to support floating ip, we create a copy port if the target port + # is not in the pod where the real external network is located. to + # distinguish it from normal port, we name it with a prefix + do_top_delete = b_port['device_owner'].startswith( + q_constants.DEVICE_OWNER_COMPUTE_PREFIX) + skip_top_delete = t_constants.RT_SD_PORT in b_port['name'] except q_exceptions.NotFound: return - if b_port['device_owner'].startswith( - q_constants.DEVICE_OWNER_COMPUTE_PREFIX): + if do_top_delete and not skip_top_delete: self.neutron_handle.handle_delete(t_ctx, t_constants.RT_PORT, _id) self.core_plugin.delete_port(context, _id, l3_port_check) diff --git a/tricircle/tests/unit/network/test_central_plugin.py b/tricircle/tests/unit/network/test_central_plugin.py index 3ec203c8..b99b3d35 100644 --- a/tricircle/tests/unit/network/test_central_plugin.py +++ b/tricircle/tests/unit/network/test_central_plugin.py @@ -294,7 +294,7 @@ class FakeClient(object): 'security_group': BOTTOM2_SGS, 'floatingip': BOTTOM2_FIPS}} - def __init__(self, region_name): + def __init__(self, region_name=None): if not region_name: self.region_name = 'top' else: @@ -336,6 +336,12 @@ class FakeClient(object): if ip in ip_range: fixed_ip['subnet_id'] = subnet['id'] break + if 'subnet_id' not in fixed_ip: + # we still cannot find the proper subnet, that's because + # this is a copy port. local plugin will create the missing + # subnet for this port but FakeClient won't. we just skip + # the ip address check + continue if fixed_ip['ip_address'] in subnet_ips_map.get( fixed_ip['subnet_id'], set()): raise q_exceptions.IpAddressInUseClient() @@ -1118,6 +1124,11 @@ class PluginTest(unittest.TestCase, DotDict({'physical_network': phynet, 'vlan_id': vlan, 'allocated': False})) + def fake_get_plugin(alias=q_constants.CORE): + return FakePlugin() + from neutron_lib.plugins import directory + directory.get_plugin = fake_get_plugin + def _basic_pod_route_setup(self): pod1 = {'pod_id': 'pod_id_1', 'region_name': 'pod_1', @@ -1440,12 +1451,11 @@ class PluginTest(unittest.TestCase, # test _prepare_top_element pool_id = fake_plugin._get_bridge_subnet_pool_id( - t_ctx, q_ctx, 'project_id', t_pod, True) + t_ctx, q_ctx, 'project_id', t_pod) net, subnet = fake_plugin._get_bridge_network_subnet( - t_ctx, q_ctx, 'project_id', t_pod, pool_id, True) + t_ctx, q_ctx, 'project_id', t_pod, pool_id) port = fake_plugin._get_bridge_interface(t_ctx, q_ctx, 'project_id', - pod, net['id'], 'b_router_id', - None, True) + pod, net['id'], 'b_router_id') top_entry_map = {} with t_ctx.session.begin(): @@ -1485,10 +1495,8 @@ class PluginTest(unittest.TestCase, @staticmethod def _prepare_network_test(tenant_id, ctx, region_name, index): - t_net_id = uuidutils.generate_uuid() - t_subnet_id = uuidutils.generate_uuid() - b_net_id = uuidutils.generate_uuid() - b_subnet_id = uuidutils.generate_uuid() + t_net_id = b_net_id = uuidutils.generate_uuid() + t_subnet_id = b_subnet_id = uuidutils.generate_uuid() # no need to specify az, we will setup router in the pod where bottom # network is created @@ -1632,9 +1640,10 @@ class PluginTest(unittest.TestCase, '_allocate_ips_for_port', new=fake_allocate_ips_for_port) @patch.object(db_base_plugin_common.DbBasePluginCommon, '_make_subnet_dict', new=fake_make_subnet_dict) + @patch.object(FakeClient, 'add_gateway_routers') @patch.object(FakeBaseRPCAPI, 'configure_extra_routes') @patch.object(context, 'get_context_from_neutron_context') - def test_add_interface(self, mock_context, mock_rpc): + def test_add_interface(self, mock_context, mock_rpc, mock_action): self._basic_pod_route_setup() fake_plugin = FakePlugin() @@ -1651,185 +1660,63 @@ class PluginTest(unittest.TestCase, q_ctx, t_router_id, {'subnet_id': t_subnet_id})['port_id'] _, b_router_id = db_api.get_bottom_mappings_by_top_id( - t_ctx, t_router_id, 'router')[0] + t_ctx, t_router_id, constants.RT_ROUTER)[0] mock_rpc.assert_called_once_with(t_ctx, t_router_id) for b_net in BOTTOM1_NETS: if 'provider:segmentation_id' in b_net: self.assertIn(b_net['provider:segmentation_id'], (2000, 2001)) - # only one VLAN allocated, for E-W bridge network + # only one VLAN allocated since we just create one bridge network allocations = [ allocation['allocated'] for allocation in TOP_VLANALLOCATIONS] self.assertItemsEqual([True, False], allocations) for segment in TOP_SEGMENTS: self.assertIn(segment['segmentation_id'], (2000, 2001)) - bridge_port_name = constants.ew_bridge_port_name % (tenant_id, - b_router_id) + bridge_port_name = constants.bridge_port_name % (tenant_id, + b_router_id) _, t_bridge_port_id = db_api.get_bottom_mappings_by_top_id( t_ctx, bridge_port_name, 'port')[0] - _, b_bridge_port_id = db_api.get_bottom_mappings_by_top_id( - t_ctx, t_bridge_port_id, 'port')[0] - - (t_net_id, t_subnet_id, t_router_id, - b_another_net_id, b_another_subnet_id) = self._prepare_router_test( - tenant_id, t_ctx, 'pod_1', 2) - - fake_plugin.add_router_interface( - q_ctx, t_router_id, {'subnet_id': t_subnet_id})['port_id'] - - t_ns_bridge_net_id = None - for net in TOP_NETS: - if net['name'].startswith('ns_bridge'): - t_ns_bridge_net_id = net['id'] - # N-S bridge not created since no external network created - self.assertIsNone(t_ns_bridge_net_id) - - device_ids = ['', '', ''] - for port in BOTTOM1_PORTS: - if port['id'] == b_bridge_port_id: - device_ids[0] = port['device_id'] - elif port['network_id'] == b_net_id and ( - port['device_owner'] == 'network:router_interface'): - device_ids[1] = port['device_id'] - elif port['network_id'] == b_another_net_id and ( - port['device_owner'] == 'network:router_interface'): - device_ids[2] = port['device_id'] - - self.assertEqual(device_ids, [b_router_id, b_router_id, b_router_id]) - - @patch.object(directory, 'get_plugin', new=fake_get_plugin) - @patch.object(driver.Pool, 'get_instance', new=fake_get_instance) - @patch.object(ipam_pluggable_backend.IpamPluggableBackend, - '_allocate_ips_for_port', new=fake_allocate_ips_for_port) - @patch.object(db_base_plugin_common.DbBasePluginCommon, - '_make_subnet_dict', new=fake_make_subnet_dict) - @patch.object(FakeBaseRPCAPI, 'configure_extra_routes') - @patch.object(FakeClient, 'add_gateway_routers') - @patch.object(context, 'get_context_from_neutron_context') - def test_add_interface_with_external_network(self, mock_context, - mock_action, mock_rpc): - self._basic_pod_route_setup() - - fake_plugin = FakePlugin() - q_ctx = FakeNeutronContext() - t_ctx = context.get_db_context() - mock_context.return_value = t_ctx - - tenant_id = TEST_TENANT_ID - (t_net_id, t_subnet_id, - t_router_id, b_net_id, b_subnet_id) = self._prepare_router_test( - tenant_id, t_ctx, 'pod_1', 1) - - e_net_id = uuidutils.generate_uuid() - e_net = {'id': e_net_id, - 'name': 'ext-net', - 'admin_state_up': True, - 'shared': False, - 'tenant_id': tenant_id, - 'router:external': True, - 'availability_zone_hints': '["pod_2"]'} - TOP_NETS.append(e_net) - - fake_plugin.add_router_interface( - q_ctx, t_router_id, {'subnet_id': t_subnet_id})['port_id'] - - b_router_id = db_api.get_bottom_id_by_top_id_region_name( - t_ctx, t_router_id, 'pod_1', 'router') - - mock_rpc.assert_called_once_with(t_ctx, t_router_id) - for b_net in BOTTOM1_NETS: - if 'provider:segmentation_id' in b_net: - self.assertIn(b_net['provider:segmentation_id'], (2000, 2001)) - # two VLANs allocated, for E-W and N-S bridge network - allocations = [ - allocation['allocated'] for allocation in TOP_VLANALLOCATIONS] - self.assertItemsEqual([True, True], allocations) - for segment in TOP_SEGMENTS: - self.assertIn(segment['segmentation_id'], (2000, 2001)) - - bridge_port_name = constants.ew_bridge_port_name % (tenant_id, - b_router_id) - _, t_bridge_port_id = db_api.get_bottom_mappings_by_top_id( - t_ctx, bridge_port_name, 'port')[0] - _, b_bridge_port_id = db_api.get_bottom_mappings_by_top_id( - t_ctx, t_bridge_port_id, 'port')[0] - - (t_net_id, t_subnet_id, t_router_id, - b_another_net_id, b_another_subnet_id) = self._prepare_router_test( - tenant_id, t_ctx, 'pod_1', 2) - - fake_plugin.add_router_interface( - q_ctx, t_router_id, {'subnet_id': t_subnet_id})['port_id'] - - for net in TOP_NETS: - if net['name'].startswith('ns_bridge'): - t_ns_bridge_net_id = net['id'] - for subnet in TOP_SUBNETS: - if subnet['name'].startswith('ns_bridge'): - t_ns_bridge_subnet_id = subnet['id'] + for t_port in TOP_PORTS: + if t_port['id'] == t_bridge_port_id: + t_ns_bridge_net_id = t_port['network_id'] + t_ns_bridge_subnet_id = t_port['fixed_ips'][0]['subnet_id'] b_ns_bridge_net_id = db_api.get_bottom_id_by_top_id_region_name( t_ctx, t_ns_bridge_net_id, 'pod_1', constants.RT_NETWORK) b_ns_bridge_subnet_id = db_api.get_bottom_id_by_top_id_region_name( t_ctx, t_ns_bridge_subnet_id, 'pod_1', constants.RT_SUBNET) - # internal network and external network are in different pods, need - # to create N-S bridge network and set gateway, add_router_interface - # is called two times, so add_gateway is also called two times. - # add_interface is called three times because the second time - # add_router_interface is called, bottom router is already attached - # to E-W bridge network, only need to attach internal network to - # bottom router - calls = [mock.call(t_ctx, b_router_id, - {'network_id': b_ns_bridge_net_id, - 'external_fixed_ips': [ - {'subnet_id': b_ns_bridge_subnet_id, - 'ip_address': '100.128.0.2'}]}), - mock.call(t_ctx, b_router_id, - {'network_id': b_ns_bridge_net_id, - 'external_fixed_ips': [ - {'subnet_id': b_ns_bridge_subnet_id, - 'ip_address': '100.128.0.2'}]})] - mock_action.assert_has_calls(calls) - device_ids = ['', '', ''] - for port in BOTTOM1_PORTS: - if port['id'] == b_bridge_port_id: - device_ids[0] = port['device_id'] - elif port['network_id'] == b_net_id and ( - port['device_owner'] == 'network:router_interface'): - device_ids[1] = port['device_id'] - elif port['network_id'] == b_another_net_id and ( - port['device_owner'] == 'network:router_interface'): - device_ids[2] = port['device_id'] - self.assertEqual(device_ids, [b_router_id, b_router_id, b_router_id]) - - (t_net_id, t_subnet_id, - t_router_id, b_net_id, b_subnet_id) = self._prepare_router_test( - tenant_id, t_ctx, 'pod_2', 2) + (t_net_id, t_subnet_id, t_router_id, + b_another_net_id, b_another_subnet_id) = self._prepare_router_test( + tenant_id, t_ctx, 'pod_1', 2) fake_plugin.add_router_interface( q_ctx, t_router_id, {'subnet_id': t_subnet_id})['port_id'] - b_router_id = db_api.get_bottom_id_by_top_id_region_name( - t_ctx, t_router_id, 'pod_2', 'router') - bridge_port_name = constants.ew_bridge_port_name % (tenant_id, - b_router_id) - _, t_bridge_port_id = db_api.get_bottom_mappings_by_top_id( - t_ctx, bridge_port_name, 'port')[0] - _, b_bridge_port_id = db_api.get_bottom_mappings_by_top_id( - t_ctx, t_bridge_port_id, 'port')[0] - # internal network and external network are in the same pod, no need - # to create N-S bridge network when attaching router interface(N-S - # bridge network is created when setting router external gateway), so - # add_gateway is not called. + mappings = db_api.get_bottom_mappings_by_top_id( + t_ctx, t_router_id, constants.RT_NS_ROUTER) + # router for north-south networking is not created since no external + # network created + self.assertEqual(len(mappings), 0) + device_ids = ['', ''] - for port in BOTTOM2_PORTS: - if port['id'] == b_bridge_port_id: + for port in BOTTOM1_PORTS: + if port['network_id'] == b_net_id and ( + port['device_owner'] == 'network:router_interface'): device_ids[0] = port['device_id'] - elif port['network_id'] == b_net_id and ( + elif port['network_id'] == b_another_net_id and ( port['device_owner'] == 'network:router_interface'): device_ids[1] = port['device_id'] + self.assertEqual(device_ids, [b_router_id, b_router_id]) + call = mock.call(t_ctx, b_router_id, + {'network_id': b_ns_bridge_net_id, + 'enable_snat': False, + 'external_fixed_ips': [ + {'subnet_id': b_ns_bridge_subnet_id, + 'ip_address': '100.0.0.2'}]}) + # each router interface adding will call add_gateway once + mock_action.assert_has_calls([call, call]) @patch.object(directory, 'get_plugin', new=fake_get_plugin) @patch.object(driver.Pool, 'get_instance', new=fake_get_instance) @@ -1870,24 +1757,25 @@ class PluginTest(unittest.TestCase, [{'key': 'resource_type', 'comparator': 'eq', 'value': 'port'}], []) - # two new entries, for top and bottom bridge ports - self.assertEqual(entry_num + 2, len(entries)) - # top and bottom interface is deleted, only bridge port left + # one new entry, for top bridge port + self.assertEqual(entry_num + 1, len(entries)) + # top and bottom interface is deleted, only top bridge port left self.assertEqual(1, len(TOP_PORTS)) - self.assertEqual(1, len(BOTTOM1_PORTS)) + self.assertEqual(0, len(BOTTOM1_PORTS)) mock_action.side_effect = None fake_plugin.add_router_interface(q_ctx, t_router_id, {'subnet_id': t_subnet_id}) - # bottom dhcp port and bridge port - self.assertEqual(2, len(BOTTOM1_PORTS)) + # just bottom dhcp port, bottom interface is not created because + # action_routers function is mocked + self.assertEqual(1, len(BOTTOM1_PORTS)) with t_ctx.session.begin(): entries = core.query_resource(t_ctx, models.ResourceRouting, [{'key': 'resource_type', 'comparator': 'eq', 'value': 'port'}], []) # three more entries, for top and bottom dhcp ports, top interface - self.assertEqual(entry_num + 2 + 3, len(entries)) + self.assertEqual(entry_num + 1 + 3, len(entries)) @patch.object(directory, 'get_plugin', new=fake_get_plugin) @patch.object(driver.Pool, 'get_instance', new=fake_get_instance) @@ -1921,8 +1809,8 @@ class PluginTest(unittest.TestCase, # test that we can success when bottom pod comes back fake_plugin.add_router_interface( q_ctx, t_router_id, {'subnet_id': t_subnet_id}) - # bottom dhcp port, bottom interface and bridge port - self.assertEqual(3, len(BOTTOM1_PORTS)) + # bottom dhcp port and bottom interface + self.assertEqual(2, len(BOTTOM1_PORTS)) @patch.object(directory, 'get_plugin', new=fake_get_plugin) @patch.object(driver.Pool, 'get_instance', new=fake_get_instance) @@ -2082,24 +1970,24 @@ class PluginTest(unittest.TestCase, 'external_fixed_ips': [{'subnet_id': TOP_SUBNETS[0]['id'], 'ip_address': '100.64.0.5'}]}}}) - b_router_id = BOTTOM1_ROUTERS[0]['id'] + b_ns_router_id = BOTTOM1_ROUTERS[0]['id'] b_net_id = db_api.get_bottom_id_by_top_id_region_name( t_ctx, t_net_id, 'pod_1', constants.RT_NETWORK) b_subnet_id = db_api.get_bottom_id_by_top_id_region_name( t_ctx, t_subnet_id, 'pod_1', constants.RT_SUBNET) for subnet in TOP_SUBNETS: - if subnet['name'].startswith('ns_bridge_subnet'): - t_ns_bridge_subnet_id = subnet['id'] - b_ns_bridge_subnet_id = db_api.get_bottom_id_by_top_id_region_name( - t_ctx, t_ns_bridge_subnet_id, 'pod_1', constants.RT_SUBNET) + if subnet['name'].startswith('bridge_subnet'): + t_bridge_subnet_id = subnet['id'] + b_bridge_subnet_id = db_api.get_bottom_id_by_top_id_region_name( + t_ctx, t_bridge_subnet_id, 'pod_1', constants.RT_SUBNET) body = {'network_id': b_net_id, 'enable_snat': False, 'external_fixed_ips': [{'subnet_id': b_subnet_id, 'ip_address': '100.64.0.5'}]} - calls = [mock.call(t_ctx, 'add_gateway', b_router_id, body), - mock.call(t_ctx, 'add_interface', b_router_id, - {'subnet_id': b_ns_bridge_subnet_id})] + calls = [mock.call(t_ctx, 'add_gateway', b_ns_router_id, body), + mock.call(t_ctx, 'add_interface', b_ns_router_id, + {'subnet_id': b_bridge_subnet_id})] mock_action.assert_has_calls(calls) @patch.object(directory, 'get_plugin', new=fake_get_plugin) @@ -2165,14 +2053,14 @@ class PluginTest(unittest.TestCase, 'enable_snat': False, 'external_fixed_ips': [{'subnet_id': t_subnet_id, 'ip_address': '100.64.0.5'}]}}}) - _, b_router_id = db_api.get_bottom_mappings_by_top_id( - t_ctx, t_router_id, constants.RT_ROUTER)[0] + _, b_ns_router_id = db_api.get_bottom_mappings_by_top_id( + t_ctx, t_router_id, constants.RT_NS_ROUTER)[0] # then remove router gateway fake_plugin.update_router( q_ctx, t_router_id, {'router': {'external_gateway_info': {}}}) - mock_action.assert_called_with(t_ctx, 'remove_gateway', b_router_id) + mock_action.assert_called_with(t_ctx, 'remove_gateway', b_ns_router_id) def _prepare_associate_floatingip_test(self, t_ctx, q_ctx, fake_plugin): tenant_id = TEST_TENANT_ID @@ -2221,7 +2109,8 @@ class PluginTest(unittest.TestCase, {'subnet_id': t_subnet_id}) # create internal port t_port_id = uuidutils.generate_uuid() - b_port_id = uuidutils.generate_uuid() + # now top id and bottom id are the same + b_port_id = t_port_id t_port = { 'id': t_port_id, 'network_id': t_net_id, @@ -2272,6 +2161,12 @@ class PluginTest(unittest.TestCase, (t_port_id, b_port_id, fip, e_net) = self._prepare_associate_floatingip_test(t_ctx, q_ctx, fake_plugin) + # here we attach a subnet in pod2 to the router to test if the two + # bottom routers can be successfully created + _, t_subnet_id2, t_router_id, _, _ = self._prepare_router_test( + TEST_TENANT_ID, t_ctx, 'pod_2', 2) + fake_plugin.add_router_interface(q_ctx, t_router_id, + {'subnet_id': t_subnet_id2}) # associate floating ip fip_body = {'port_id': t_port_id} @@ -2280,25 +2175,28 @@ class PluginTest(unittest.TestCase, b_ext_net_id = db_api.get_bottom_id_by_top_id_region_name( t_ctx, e_net['id'], 'pod_2', constants.RT_NETWORK) - for port in BOTTOM2_PORTS: - if port['name'] == 'ns_bridge_port': - ns_bridge_port = port - for net in TOP_NETS: - if net['name'].startswith('ns_bridge'): - b_bridge_net_id = db_api.get_bottom_id_by_top_id_region_name( - t_ctx, net['id'], 'pod_1', constants.RT_NETWORK) calls = [mock.call(t_ctx, - {'floatingip': { - 'floating_network_id': b_bridge_net_id, - 'floating_ip_address': '100.128.0.3', - 'port_id': b_port_id}}), - mock.call(t_ctx, {'floatingip': { 'floating_network_id': b_ext_net_id, 'floating_ip_address': fip[ 'floating_ip_address'], - 'port_id': ns_bridge_port['id']}})] + 'port_id': b_port_id}})] mock_create.assert_has_calls(calls) + # routers for east-west networking and north-south networking + self.assertEqual(2, len(BOTTOM2_ROUTERS)) + + # check routing entries for copied resources have been created + fake_client = FakeClient() + t_port = fake_client.get_ports(t_ctx, t_port_id) + cp_port_mappings = db_api.get_bottom_mappings_by_top_id( + t_ctx, t_port_id, constants.RT_SD_PORT) + cp_subnet_mappings = db_api.get_bottom_mappings_by_top_id( + t_ctx, t_port['fixed_ips'][0]['subnet_id'], constants.RT_SD_SUBNET) + cp_network_mappings = db_api.get_bottom_mappings_by_top_id( + t_ctx, t_port['network_id'], constants.RT_SD_NETWORK) + self.assertEqual(1, len(cp_port_mappings)) + self.assertEqual(1, len(cp_subnet_mappings)) + self.assertEqual(1, len(cp_network_mappings)) @patch.object(directory, 'get_plugin', new=fake_get_plugin) @patch.object(driver.Pool, 'get_instance', new=fake_get_instance) @@ -2364,34 +2262,22 @@ class PluginTest(unittest.TestCase, fake_plugin.update_floatingip(q_ctx, fip['id'], {'floatingip': fip_body}) - bridge_port_name = constants.ns_bridge_port_name % ( - e_net['tenant_id'], None, b_port_id) - t_pod = db_api.get_top_pod(t_ctx) - mapping = db_api.get_bottom_id_by_top_id_region_name( - t_ctx, bridge_port_name, t_pod['region_name'], constants.RT_PORT) - # check routing for bridge port in top pod exists - self.assertIsNotNone(mapping) - # disassociate floating ip fip_body = {'port_id': None} fake_plugin.update_floatingip(q_ctx, fip['id'], {'floatingip': fip_body}) - fip_id1 = BOTTOM1_FIPS[0]['id'] - fip_id2 = BOTTOM2_FIPS[0]['id'] - - calls = [mock.call(t_ctx, fip_id1), - mock.call(t_ctx, fip_id2)] - mock_delete.assert_has_calls(calls) - mapping = db_api.get_bottom_id_by_top_id_region_name( - t_ctx, bridge_port_name, t_pod['region_name'], constants.RT_PORT) - # check routing for bridge port in top pod is deleted - self.assertIsNone(mapping) + fip_id = BOTTOM2_FIPS[0]['id'] + mock_delete.assert_called_once_with(t_ctx, fip_id) # check the association information is cleared self.assertIsNone(TOP_FLOATINGIPS[0]['fixed_port_id']) self.assertIsNone(TOP_FLOATINGIPS[0]['fixed_ip_address']) self.assertIsNone(TOP_FLOATINGIPS[0]['router_id']) + # check routing entry for copied port has been created + cp_port_mappings = db_api.get_bottom_mappings_by_top_id( + t_ctx, t_port_id, constants.RT_SD_PORT) + self.assertEqual(0, len(cp_port_mappings)) @patch.object(directory, 'get_plugin', new=fake_get_plugin) @patch.object(driver.Pool, 'get_instance', new=fake_get_instance) @@ -2419,26 +2305,11 @@ class PluginTest(unittest.TestCase, fake_plugin.update_floatingip(q_ctx, fip['id'], {'floatingip': fip_body}) - bridge_port_name = constants.ns_bridge_port_name % ( - e_net['tenant_id'], None, b_port_id) - t_pod = db_api.get_top_pod(t_ctx) - mapping = db_api.get_bottom_id_by_top_id_region_name( - t_ctx, bridge_port_name, t_pod['region_name'], constants.RT_PORT) - # check routing for bridge port in top pod exists - self.assertIsNotNone(mapping) - + # disassociate floating ip fake_plugin.delete_floatingip(q_ctx, fip['id']) - fip_id1 = BOTTOM1_FIPS[0]['id'] - fip_id2 = BOTTOM2_FIPS[0]['id'] - - calls = [mock.call(t_ctx, fip_id1), - mock.call(t_ctx, fip_id2)] - mock_delete.assert_has_calls(calls) - mapping = db_api.get_bottom_id_by_top_id_region_name( - t_ctx, bridge_port_name, t_pod['region_name'], constants.RT_PORT) - # check routing for bridge port in top pod is deleted - self.assertIsNone(mapping) + fip_id = BOTTOM2_FIPS[0]['id'] + mock_delete.assert_called_once_with(t_ctx, fip_id) @patch.object(directory, 'get_plugin', new=fake_get_plugin) @patch.object(driver.Pool, 'get_instance', new=fake_get_instance) diff --git a/tricircle/tests/unit/xjob/test_xmanager.py b/tricircle/tests/unit/xjob/test_xmanager.py index 7cd8bddd..67d80d0a 100644 --- a/tricircle/tests/unit/xjob/test_xmanager.py +++ b/tricircle/tests/unit/xjob/test_xmanager.py @@ -38,27 +38,36 @@ BOTTOM1_SUBNET = [] BOTTOM2_SUBNET = [] BOTTOM1_PORT = [] BOTTOM2_PORT = [] +TOP_ROUTER = [] BOTTOM1_ROUTER = [] BOTTOM2_ROUTER = [] TOP_SG = [] BOTTOM1_SG = [] BOTTOM2_SG = [] +TOP_FIP = [] +BOTTOM1_FIP = [] +BOTTOM2_FIP = [] RES_LIST = [TOP_NETWORK, BOTTOM1_NETWORK, BOTTOM2_NETWORK, TOP_SUBNET, BOTTOM1_SUBNET, BOTTOM2_SUBNET, BOTTOM1_PORT, BOTTOM2_PORT, - BOTTOM1_ROUTER, BOTTOM2_ROUTER, TOP_SG, BOTTOM1_SG, BOTTOM2_SG] + TOP_ROUTER, BOTTOM1_ROUTER, BOTTOM2_ROUTER, TOP_SG, BOTTOM1_SG, + BOTTOM2_SG, TOP_FIP, BOTTOM1_FIP, BOTTOM2_FIP] RES_MAP = {'top': {'network': TOP_NETWORK, 'subnet': TOP_SUBNET, - 'security_group': TOP_SG}, + 'router': TOP_ROUTER, + 'security_group': TOP_SG, + 'floatingips': TOP_FIP}, 'pod_1': {'network': BOTTOM1_NETWORK, 'subnet': BOTTOM1_SUBNET, 'port': BOTTOM1_PORT, 'router': BOTTOM1_ROUTER, - 'security_group': BOTTOM1_SG}, + 'security_group': BOTTOM1_SG, + 'floatingips': BOTTOM1_FIP}, 'pod_2': {'network': BOTTOM2_NETWORK, 'subnet': BOTTOM2_SUBNET, 'port': BOTTOM2_PORT, 'router': BOTTOM2_ROUTER, - 'security_group': BOTTOM2_SG}} + 'security_group': BOTTOM2_SG, + 'floatingips': BOTTOM2_FIP}} class FakeXManager(xmanager.XManager): @@ -84,7 +93,7 @@ class FakeClient(object): if _filter['key'] not in res: is_selected = False break - if res[_filter['key']] != _filter['value']: + if res[_filter['key']] not in _filter['value']: is_selected = False break if is_selected: @@ -102,6 +111,11 @@ class FakeClient(object): 'subnet', cxt, [{'key': 'id', 'comparator': 'eq', 'value': subnet_id}])[0] + def get_routers(self, cxt, router_id): + return self.list_resources( + 'router', cxt, + [{'key': 'id', 'comparator': 'eq', 'value': router_id}])[0] + def update_routers(self, cxt, *args, **kwargs): pass @@ -119,6 +133,9 @@ class FakeClient(object): def create_security_group_rules(self, cxt, *args, **kwargs): pass + def list_floatingips(self, cxt, filters=None): + return self.list_resources('floatingips', cxt, filters) + class XManagerTest(unittest.TestCase): def setUp(self): @@ -132,11 +149,68 @@ class XManagerTest(unittest.TestCase): cfg.CONF.register_opt(opt) self.context = context.Context() self.xmanager = FakeXManager() - self.xmanager = FakeXManager() - @patch.object(FakeClient, 'update_routers') - def test_configure_extra_routes(self, mock_update): - top_router_id = 'router_id' + def _prepare_dnat_test(self): + for subnet in BOTTOM2_SUBNET: + if 'ext' in subnet['id']: + ext_subnet = subnet + ext_cidr = ext_subnet['cidr'] + ext_cidr_prefix = ext_cidr[:ext_cidr.rindex('.')] + vm_ports = [] + # get one vm port from each bottom pod + for ports in [BOTTOM1_PORT, BOTTOM2_PORT]: + for port in ports: + if port['device_owner'] == 'compute:None': + vm_ports.append(port) + break + for i, vm_port in enumerate(vm_ports): + vm_ip = vm_port['fixed_ips'][0]['ip_address'] + fip = {'floating_network_id': ext_subnet['network_id'], + 'floating_ip_address': '%s.%d' % (ext_cidr_prefix, i + 1), + 'port_id': vm_port['id'], + 'fixed_ip_address': vm_ip} + TOP_FIP.append(fip) + BOTTOM2_FIP.append(fip) + + def _prepare_snat_test(self, top_router_id): + ext_network = {'id': 'ext_network_id', + 'router:external': True} + ext_subnet = { + 'id': 'ext_subnet_id', + 'network_id': ext_network['id'], + 'cidr': '162.3.124.0/24', + 'gateway_ip': '162.3.124.1' + } + for router in TOP_ROUTER: + if router['id'] == top_router_id: + router['external_gateway_info'] = { + 'network_id': ext_network['id']} + router = {'id': 'ns_router_id'} + for subnet in BOTTOM2_SUBNET: + if 'bridge' in subnet['id']: + bridge_subnet = subnet + bridge_port = { + 'network_id': bridge_subnet['network_id'], + 'device_id': router['id'], + 'device_owner': 'network:router_interface', + 'fixed_ips': [{'subnet_id': bridge_subnet['id'], + 'ip_address': bridge_subnet['gateway_ip']}] + } + BOTTOM2_NETWORK.append(ext_network) + BOTTOM2_SUBNET.append(ext_subnet) + BOTTOM2_PORT.append(bridge_port) + BOTTOM2_ROUTER.append(router) + route = {'top_id': top_router_id, 'bottom_id': router['id'], + 'pod_id': 'pod_id_2', 'resource_type': constants.RT_NS_ROUTER} + with self.context.session.begin(): + core.create_resource(self.context, models.ResourceRouting, route) + return bridge_subnet['gateway_ip'], router['id'] + + def _prepare_east_west_network_test(self, top_router_id): + bridge_infos = [] + + router = {'id': top_router_id} + TOP_ROUTER.append(router) for i in xrange(1, 3): pod_dict = {'pod_id': 'pod_id_%d' % i, 'region_name': 'pod_%d' % i, @@ -156,7 +230,7 @@ class XManagerTest(unittest.TestCase): 'id': 'bridge_subnet_%d_id' % i, 'network_id': bridge_network['id'], 'cidr': '100.0.1.0/24', - 'gateway_ip': '100.0.1.%d' % i, + 'gateway_ip': '100.0.1.1', } port = { 'network_id': network['id'], @@ -166,18 +240,22 @@ class XManagerTest(unittest.TestCase): 'ip_address': subnet['gateway_ip']}] } vm_port = { + 'id': 'vm_port_%d_id' % i, 'network_id': network['id'], 'device_id': 'vm%d_id' % i, 'device_owner': 'compute:None', 'fixed_ips': [{'subnet_id': subnet['id'], 'ip_address': '10.0.%d.3' % i}] } + bridge_cidr = bridge_subnet['cidr'] + bridge_port_ip = '%s.%d' % (bridge_cidr[:bridge_cidr.rindex('.')], + 2 + i) bridge_port = { 'network_id': bridge_network['id'], 'device_id': router['id'], - 'device_owner': 'network:router_interface', + 'device_owner': 'network:router_gateway', 'fixed_ips': [{'subnet_id': bridge_subnet['id'], - 'ip_address': bridge_subnet['gateway_ip']}] + 'ip_address': bridge_port_ip}] } region_name = 'pod_%d' % i RES_MAP[region_name]['network'].append(network) @@ -194,6 +272,13 @@ class XManagerTest(unittest.TestCase): with self.context.session.begin(): core.create_resource(self.context, models.ResourceRouting, route) + + bridge_info = { + 'router_id': router['id'], + 'bridge_ip': bridge_port['fixed_ips'][0]['ip_address'], + 'vm_ips': ['10.0.%d.3' % i]} + bridge_infos.append(bridge_info) + BOTTOM1_NETWORK.append({'id': 'network_3_id'}) BOTTOM1_SUBNET.append({'id': 'subnet_3_id', 'network_id': 'network_3_id', @@ -209,30 +294,88 @@ class XManagerTest(unittest.TestCase): 'device_owner': 'compute:None', 'fixed_ips': [{'subnet_id': 'subnet_3_id', 'ip_address': '10.0.3.3'}]}) + bridge_infos[0]['vm_ips'].append('10.0.3.3') + return bridge_infos + def _check_extra_routes_calls(self, except_list, actual_list): + except_map = {} + for except_call in except_list: + ctx, router_id, routes_body = except_call[1] + except_map[router_id] = (ctx, routes_body['router']['routes']) + for actual_call in actual_list: + ctx, router_id, routes_body = actual_call[0] + expect_ctx, expect_routes = except_map[router_id] + self.assertEqual(expect_ctx, ctx) + self.assertItemsEqual(expect_routes, + routes_body['router']['routes']) + + @patch.object(FakeClient, 'update_routers') + def test_configure_extra_routes_with_floating_ips(self, mock_update): + top_router_id = 'router_id' + bridge_infos = self._prepare_east_west_network_test(top_router_id) + ns_bridge_ip, ns_router_id = self._prepare_snat_test(top_router_id) + self._prepare_dnat_test() self.xmanager.configure_extra_routes(self.context, payload={'router': top_router_id}) - calls = [mock.call(self.context, 'router_1_id', - {'router': { - 'routes': [{'nexthop': '100.0.1.2', - 'destination': '10.0.2.3/32'}]}}), - mock.call(self.context, 'router_2_id', - {'router': { - 'routes': [{'nexthop': '100.0.1.1', - 'destination': '10.0.1.3/32'}, - {'nexthop': '100.0.1.1', - 'destination': '10.0.3.3/32'}]}}), - mock.call(self.context, 'router_2_id', - {'router': { - 'routes': [{'nexthop': '100.0.1.1', - 'destination': '10.0.3.3/32'}, - {'nexthop': '100.0.1.1', - 'destination': '10.0.1.3/32'}]}})] + calls = [] + ns_routes = [] + for i in range(2): + routes = [] + for ip in bridge_infos[i]['vm_ips']: + route = {'nexthop': bridge_infos[i]['bridge_ip'], + 'destination': ip + '/32'} + routes.append(route) + ns_routes.append(route) + routes.append({'nexthop': ns_bridge_ip, + 'destination': '0.0.0.0/0'}) + call = mock.call(self.context, bridge_infos[1 - i]['router_id'], + {'router': {'routes': routes}}) + calls.append(call) + calls.append(mock.call(self.context, ns_router_id, + {'router': {'routes': ns_routes}})) + self._check_extra_routes_calls(calls, mock_update.call_args_list) - called = mock_update.call_args_list[1] == calls[1] - called = called or (mock_update.call_args_list[1] == calls[2]) - called = called and (mock_update.call_args_list[0] == calls[0]) - self.assertTrue(called) + @patch.object(FakeClient, 'update_routers') + def test_configure_extra_routes_with_external_network(self, mock_update): + top_router_id = 'router_id' + bridge_infos = self._prepare_east_west_network_test(top_router_id) + ns_bridge_ip, ns_router_id = self._prepare_snat_test(top_router_id) + self.xmanager.configure_extra_routes(self.context, + payload={'router': top_router_id}) + calls = [] + ns_routes = [] + for i in range(2): + routes = [] + for ip in bridge_infos[i]['vm_ips']: + route = {'nexthop': bridge_infos[i]['bridge_ip'], + 'destination': ip + '/32'} + routes.append(route) + ns_routes.append(route) + routes.append({'nexthop': ns_bridge_ip, + 'destination': '0.0.0.0/0'}) + call = mock.call(self.context, bridge_infos[1 - i]['router_id'], + {'router': {'routes': routes}}) + calls.append(call) + calls.append(mock.call(self.context, ns_router_id, + {'router': {'routes': ns_routes}})) + self._check_extra_routes_calls(calls, mock_update.call_args_list) + + @patch.object(FakeClient, 'update_routers') + def test_configure_extra_routes(self, mock_update): + top_router_id = 'router_id' + bridge_infos = self._prepare_east_west_network_test(top_router_id) + self.xmanager.configure_extra_routes(self.context, + payload={'router': top_router_id}) + calls = [] + for i in range(2): + routes = [] + for ip in bridge_infos[i]['vm_ips']: + routes.append({'nexthop': bridge_infos[i]['bridge_ip'], + 'destination': ip + '/32'}) + call = mock.call(self.context, bridge_infos[1 - i]['router_id'], + {'router': {'routes': routes}}) + calls.append(call) + self._check_extra_routes_calls(calls, mock_update.call_args_list) @patch.object(FakeClient, 'delete_security_group_rules') @patch.object(FakeClient, 'create_security_group_rules') diff --git a/tricircle/xjob/xmanager.py b/tricircle/xjob/xmanager.py index 441e96e7..f3eaf803 100644 --- a/tricircle/xjob/xmanager.py +++ b/tricircle/xjob/xmanager.py @@ -24,6 +24,7 @@ from oslo_log import log as logging import oslo_messaging as messaging from oslo_service import periodic_task +import neutron_lib.constants as q_constants import neutron_lib.exceptions as q_exceptions import neutronclient.common.exceptions as q_cli_exceptions @@ -37,7 +38,6 @@ from tricircle.db import models import tricircle.network.exceptions as t_network_exc from tricircle.network import helper - CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -294,8 +294,13 @@ class XManager(PeriodicTasks): raise def _setup_router_one_pod(self, ctx, t_pod, b_pod, t_client, t_net, - t_router, t_ew_bridge_net, t_ew_bridge_subnet, - need_ns_bridge): + t_router, t_bridge_net, t_bridge_subnet, + is_ext_net_pod): + # NOTE(zhiyuan) after the bridge network combination, external network + # is attached to a separate router, which is created in central plugin, + # so is_ext_net_pod is not used in the current implementation, but we + # choose to keep this parameter since it's an important attribute of a + # pod and we may need to use it later. b_client = self._get_client(b_pod['region_name']) router_body = {'router': {'name': t_router['id'], @@ -304,62 +309,32 @@ class XManager(PeriodicTasks): # create bottom router in target bottom pod _, b_router_id = self.helper.prepare_bottom_element( - ctx, project_id, b_pod, t_router, 'router', router_body) + ctx, project_id, b_pod, t_router, constants.RT_ROUTER, router_body) - # handle E-W networking - # create top E-W bridge port + # create top bridge port q_ctx = None # no need to pass neutron context when using client - t_ew_bridge_port_id = self.helper.get_bridge_interface( - ctx, q_ctx, project_id, t_pod, t_ew_bridge_net['id'], - b_router_id, None, True) + t_bridge_port_id = self.helper.get_bridge_interface( + ctx, q_ctx, project_id, t_pod, t_bridge_net['id'], b_router_id) - # create bottom E-W bridge port - t_ew_bridge_port = t_client.get_ports(ctx, t_ew_bridge_port_id) - (is_new, b_ew_bridge_port_id, - _, _) = self.helper.get_bottom_bridge_elements( - ctx, project_id, b_pod, t_ew_bridge_net, False, t_ew_bridge_subnet, - t_ew_bridge_port) + # create bottom bridge port + # if target bottom pod is hosting real external network, we create + # another bottom router and attach the bridge network as internal + # network, but this work is done by central plugin when user sets + # router gateway. + t_bridge_port = t_client.get_ports(ctx, t_bridge_port_id) + (is_new, b_bridge_port_id, b_bridge_subnet_id, + b_bridge_net_id) = self.helper.get_bottom_bridge_elements( + ctx, project_id, b_pod, t_bridge_net, True, t_bridge_subnet, None) - # attach bottom E-W bridge port to bottom router - if is_new: - # only attach bridge port the first time - b_client.action_routers(ctx, 'add_interface', b_router_id, - {'port_id': b_ew_bridge_port_id}) - else: - # still need to check if the bridge port is bound - port = b_client.get_ports(ctx, b_ew_bridge_port_id) - if not port.get('device_id'): - b_client.action_routers(ctx, 'add_interface', b_router_id, - {'port_id': b_ew_bridge_port_id}) - - # handle N-S networking - if need_ns_bridge: - t_ns_bridge_net_name = constants.ns_bridge_net_name % project_id - t_ns_bridge_subnet_name = constants.ns_bridge_subnet_name % ( - project_id) - t_ns_bridge_net = self._get_resource_by_name( - t_client, ctx, 'network', t_ns_bridge_net_name) - t_ns_bridge_subnet = self._get_resource_by_name( - t_client, ctx, 'subnet', t_ns_bridge_subnet_name) - # create bottom N-S bridge network and subnet - (_, _, b_ns_bridge_subnet_id, - b_ns_bridge_net_id) = self.helper.get_bottom_bridge_elements( - ctx, project_id, b_pod, t_ns_bridge_net, True, - t_ns_bridge_subnet, None) - # create top N-S bridge gateway port - t_ns_bridge_gateway_id = self.helper.get_bridge_interface( - ctx, q_ctx, project_id, t_pod, t_ns_bridge_net['id'], - b_router_id, None, False) - t_ns_bridge_gateway = t_client.get_ports(ctx, - t_ns_bridge_gateway_id) - # add external gateway for bottom router - # add gateway is update operation, can run multiple times - gateway_ip = t_ns_bridge_gateway['fixed_ips'][0]['ip_address'] - b_client.action_routers( - ctx, 'add_gateway', b_router_id, - {'network_id': b_ns_bridge_net_id, - 'external_fixed_ips': [{'subnet_id': b_ns_bridge_subnet_id, - 'ip_address': gateway_ip}]}) + # we attach the bridge port as router gateway + # add_gateway is update operation, which can run multiple times + gateway_ip = t_bridge_port['fixed_ips'][0]['ip_address'] + b_client.action_routers( + ctx, 'add_gateway', b_router_id, + {'network_id': b_bridge_net_id, + 'enable_snat': False, + 'external_fixed_ips': [{'subnet_id': b_bridge_subnet_id, + 'ip_address': gateway_ip}]}) # attach internal port to bottom router t_ports = self._get_router_interfaces(t_client, ctx, t_router['id'], @@ -438,107 +413,84 @@ class XManager(PeriodicTasks): if t_int_port['network_id'] != t_net['id']: # only handle floating ip association for the given top network continue - if need_ns_bridge: - # create top N-S bridge interface port - t_ns_bridge_port_id = self.helper.get_bridge_interface( - ctx, q_ctx, project_id, t_pod, t_ns_bridge_net['id'], None, - b_int_port_id, False) - t_ns_bridge_port = t_client.get_ports(ctx, t_ns_bridge_port_id) - b_ext_bridge_net_id = \ - db_api.get_bottom_id_by_top_id_region_name( - ctx, t_ns_bridge_net['id'], b_ext_pod['region_name'], - constants.RT_NETWORK) + + if b_ext_pod['pod_id'] != b_pod['pod_id']: + # if the internal port is not located in the external network + # pod, we need to create a copied port in that pod for floating + # ip association purpose + t_int_net_id = t_int_port['network_id'] + t_int_subnet_id = t_int_port['fixed_ips'][0]['subnet_id'] port_body = { 'port': { 'tenant_id': project_id, 'admin_state_up': True, - 'name': 'ns_bridge_port', - 'network_id': b_ext_bridge_net_id, - 'fixed_ips': [{'ip_address': t_ns_bridge_port[ + 'name': constants.shadow_port_name % t_int_port['id'], + 'network_id': t_int_net_id, + 'fixed_ips': [{'ip_address': t_int_port[ 'fixed_ips'][0]['ip_address']}] } } - _, b_ns_bridge_port_id = self.helper.prepare_bottom_element( - ctx, project_id, b_ext_pod, t_ns_bridge_port, - constants.RT_PORT, port_body) - # swap these two lines - self._safe_create_bottom_floatingip( - ctx, b_pod, b_client, b_ns_bridge_net_id, - t_ns_bridge_port['fixed_ips'][0]['ip_address'], - b_int_port_id) - self._safe_create_bottom_floatingip( - ctx, b_ext_pod, b_ext_client, b_ext_net_id, add_fip, - b_ns_bridge_port_id) - else: - self._safe_create_bottom_floatingip( - ctx, b_pod, b_client, b_ext_net_id, add_fip, - b_int_port_id) + self.helper.prepare_bottom_element( + ctx, project_id, b_ext_pod, t_int_port, + constants.RT_SD_PORT, port_body) + # create routing entries for copied network and subnet so we + # can easily find them during central network and subnet + # deletion, create_resource_mapping will catch DBDuplicateEntry + # exception and ignore it so it's safe to call this function + # multiple times + db_api.create_resource_mapping(ctx, t_int_net_id, t_int_net_id, + b_ext_pod['pod_id'], project_id, + constants.RT_SD_NETWORK) + db_api.create_resource_mapping(ctx, t_int_subnet_id, + t_int_subnet_id, + b_ext_pod['pod_id'], project_id, + constants.RT_SD_SUBNET) + + self._safe_create_bottom_floatingip( + ctx, b_pod, b_ext_client, b_ext_net_id, add_fip, + b_int_port_id) for del_fip in del_fips: fip = b_ip_fip_map[del_fip] - if not fip['port_id']: - b_ext_client.delete_floatingips(ctx, fip['id']) - continue - if need_ns_bridge: - b_ns_bridge_port = b_ext_client.get_ports(ctx, fip['port_id']) - entries = core.query_resource( - ctx, models.ResourceRouting, - [{'key': 'bottom_id', 'comparator': 'eq', - 'value': b_ns_bridge_port['id']}, - {'key': 'pod_id', 'comparator': 'eq', - 'value': b_ext_pod['pod_id']}], []) - t_ns_bridge_port_id = entries[0]['top_id'] - top_entries = core.query_resource( - ctx, models.ResourceRouting, - [{'key': 'bottom_id', 'comparator': 'eq', - 'value': t_ns_bridge_port_id}, - {'key': 'pod_id', 'comparator': 'eq', - 'value': t_pod['pod_id']}], []) - t_ns_bridge_port_name = top_entries[0]['top_id'] - b_int_fips = b_client.list_floatingips( - ctx, - [{'key': 'floating_ip_address', - 'comparator': 'eq', - 'value': b_ns_bridge_port['fixed_ips'][0]['ip_address']}, - {'key': 'floating_network_id', - 'comparator': 'eq', - 'value': b_ns_bridge_net_id}]) - if b_int_fips: - b_client.delete_floatingips(ctx, b_int_fips[0]['id']) - - # for bridge port, we have two resource routing entries, one - # for bridge port in top pod, another for bridge port in bottom - # pod. calling t_client.delete_ports will delete bridge port in - # bottom pod as well as routing entry for it, but we also need - # to remove routing entry for bridge port in top pod, bridge - # network will be deleted when deleting router - - # first we update the routing entry to set bottom_id to None - # and expire the entry, so if we succeed to delete the bridge - # port next, this expired entry will be deleted; otherwise, we - # fail to delete the bridge port, when the port is accessed via - # lock_handle module, that module will find the port and update - # the entry + if b_ext_pod['pod_id'] != b_pod['pod_id'] and fip['port_id']: + # expire the routing entry for copy port with ctx.session.begin(): core.update_resources( ctx, models.ResourceRouting, [{'key': 'bottom_id', 'comparator': 'eq', - 'value': t_ns_bridge_port_id}, - {'key': 'pod_id', 'comparator': 'eq', - 'value': t_pod['pod_id']}], + 'value': fip['port_id']}, + {'key': 'resource_type', 'comparator': 'eq', + 'value': constants.RT_SD_PORT}], {'bottom_id': None, 'created_at': constants.expire_time, 'updated_at': constants.expire_time}) - # delete bridge port - t_client.delete_ports(ctx, t_ns_bridge_port_id) + # delete copy port + b_ext_client.delete_ports(ctx, fip['port_id']) # delete the expired entry, even if this deletion fails, we # still have a chance that lock_handle module will delete it with ctx.session.begin(): core.delete_resources(ctx, models.ResourceRouting, [{'key': 'top_id', 'comparator': 'eq', - 'value': t_ns_bridge_port_name}]) + 'value': fip['port_id']}, + {'key': 'resource_type', + 'comparator': 'eq', + 'value': constants.RT_SD_PORT}]) + # delete port before floating ip disassociation, copy + # network and copy subnet are deleted during central + # network and subnet deletion b_ext_client.delete_floatingips(ctx, fip['id']) + # we first delete the internal port then delete the floating + # ip. during the deletion of the internal port, the floating + # ip will be disassociated automatically. + + # the reason we delete the internal port first is that if we + # succeed to delete the internal port but fail to delete the + # floating ip, in the next run, we can still find the floating + # ip and try to delete it. but if we delete the floating ip + # first, after we fail to delete the internal port, it's not + # easy for us to find the internal port again because we cannot + # find the internal port id the floating ip body @_job_handle(constants.JT_ROUTER_SETUP) def setup_bottom_router(self, ctx, payload): @@ -570,12 +522,12 @@ class XManager(PeriodicTasks): b_pod = db_api.get_pod(ctx, b_pod_id) - t_ew_bridge_net_name = constants.ew_bridge_net_name % project_id - t_ew_bridge_subnet_name = constants.ew_bridge_subnet_name % project_id - t_ew_bridge_net = self._get_resource_by_name(t_client, ctx, 'network', - t_ew_bridge_net_name) - t_ew_bridge_subnet = self._get_resource_by_name( - t_client, ctx, 'subnet', t_ew_bridge_subnet_name) + t_bridge_net_name = constants.bridge_net_name % project_id + t_bridge_subnet_name = constants.bridge_subnet_name % project_id + t_bridge_net = self._get_resource_by_name(t_client, ctx, 'network', + t_bridge_net_name) + t_bridge_subnet = self._get_resource_by_name( + t_client, ctx, 'subnet', t_bridge_subnet_name) ext_nets = t_client.list_networks(ctx, filters=[{'key': 'router:external', @@ -585,48 +537,79 @@ class XManager(PeriodicTasks): [ext_net[AZ_HINTS][0] for ext_net in ext_nets]) if not ext_net_region_names: - need_ns_bridge = False + is_ext_net_pod = False elif b_pod['region_name'] in ext_net_region_names: - need_ns_bridge = False + is_ext_net_pod = True else: - need_ns_bridge = True + is_ext_net_pod = False self._setup_router_one_pod(ctx, t_pod, b_pod, t_client, t_net, - t_router, t_ew_bridge_net, - t_ew_bridge_subnet, need_ns_bridge) + t_router, t_bridge_net, + t_bridge_subnet, is_ext_net_pod) self.xjob_handler.configure_extra_routes(ctx, t_router_id) @_job_handle(constants.JT_ROUTER) def configure_extra_routes(self, ctx, payload): t_router_id = payload[constants.JT_ROUTER] + t_client = self._get_client() + t_router = t_client.get_routers(ctx, t_router_id) + if not t_router: + return + if t_router.get('external_gateway_info'): + t_ext_net_id = t_router['external_gateway_info']['network_id'] + else: + t_ext_net_id = None - non_vm_port_types = ['network:router_interface', - 'network:router_gateway', - 'network:dhcp'] + non_vm_port_types = [q_constants.DEVICE_OWNER_ROUTER_INTF, + q_constants.DEVICE_OWNER_ROUTER_GW, + q_constants.DEVICE_OWNER_DHCP] + ew_attached_port_types = [q_constants.DEVICE_OWNER_ROUTER_INTF, + q_constants.DEVICE_OWNER_ROUTER_GW] + ns_attached_port_types = q_constants.DEVICE_OWNER_ROUTER_INTF - b_pods, b_router_ids = zip(*db_api.get_bottom_mappings_by_top_id( - ctx, t_router_id, constants.RT_ROUTER)) + mappings = db_api.get_bottom_mappings_by_top_id(ctx, t_router_id, + constants.RT_ROUTER) + if not mappings: + b_pods, b_router_ids = [], [] + else: + b_pods, b_router_ids = map(list, zip(*mappings)) + ns_mappings = db_api.get_bottom_mappings_by_top_id( + ctx, t_router_id, constants.RT_NS_ROUTER) + b_ns_pdd, b_ns_router_id = None, None + if ns_mappings: + b_ns_pdd, b_ns_router_id = ns_mappings[0] + b_pods.append(b_ns_pdd) + b_router_ids.append(b_ns_router_id) - router_bridge_ip_map = {} + router_ew_bridge_ip_map = {} + router_ns_bridge_ip_map = {} router_ips_map = {} for i, b_pod in enumerate(b_pods): - bottom_client = self._get_client(region_name=b_pod['region_name']) + is_ns_router = b_router_ids[i] == b_ns_router_id + bottom_client = self._get_client(b_pod['region_name']) + if is_ns_router: + device_owner_filter = ns_attached_port_types + else: + device_owner_filter = ew_attached_port_types b_interfaces = bottom_client.list_ports( ctx, filters=[{'key': 'device_id', 'comparator': 'eq', 'value': b_router_ids[i]}, {'key': 'device_owner', 'comparator': 'eq', - 'value': 'network:router_interface'}]) + 'value': device_owner_filter}]) router_ips_map[b_router_ids[i]] = {} for b_interface in b_interfaces: ip = b_interface['fixed_ips'][0]['ip_address'] - ew_bridge_cidr = CONF.client.ew_bridge_cidr - ns_bridge_cidr = CONF.client.ns_bridge_cidr - if netaddr.IPAddress(ip) in netaddr.IPNetwork(ew_bridge_cidr): - router_bridge_ip_map[b_router_ids[i]] = ip - continue - if netaddr.IPAddress(ip) in netaddr.IPNetwork(ns_bridge_cidr): + bridge_cidr = CONF.client.bridge_cidr + if netaddr.IPAddress(ip) in netaddr.IPNetwork(bridge_cidr): + if is_ns_router: + # this ip is the default gateway ip for north-south + # networking + router_ns_bridge_ip_map[b_router_ids[i]] = ip + else: + # this ip is the next hop for east-west networking + router_ew_bridge_ip_map[b_router_ids[i]] = ip continue b_net_id = b_interface['network_id'] b_subnet = bottom_client.get_subnets( @@ -641,9 +624,11 @@ class XManager(PeriodicTasks): 'ip_address'] for vm_port in b_vm_ports] router_ips_map[b_router_ids[i]][b_subnet['cidr']] = ips + # handle extra routes for east-west traffic for i, b_router_id in enumerate(b_router_ids): - bottom_client = self._get_client( - region_name=b_pods[i]['region_name']) + if b_router_id == b_ns_router_id: + continue + bottom_client = self._get_client(b_pods[i]['region_name']) extra_routes = [] if not router_ips_map[b_router_id]: bottom_client.update_routers( @@ -656,12 +641,49 @@ class XManager(PeriodicTasks): if cidr in router_ips_map[b_router_id]: continue for ip in ips: - extra_routes.append( - {'nexthop': router_bridge_ip_map[router_id], - 'destination': ip + '/32'}) + route = {'nexthop': router_ew_bridge_ip_map[router_id], + 'destination': ip + '/32'} + extra_routes.append(route) + + if router_ns_bridge_ip_map and t_ext_net_id: + extra_routes.append( + {'nexthop': router_ns_bridge_ip_map.values()[0], + 'destination': constants.DEFAULT_DESTINATION}) bottom_client.update_routers( ctx, b_router_id, {'router': {'routes': extra_routes}}) + if not b_ns_router_id: + # router for north-south networking not exist, skip extra routes + # configuration for north-south router + return + if not t_ext_net_id: + # router not attached to external gateway but router for north- + # south networking exists, clear the extra routes + bottom_client = self._get_client(pod_name=b_ns_pdd['pod_name']) + bottom_client.update_routers( + ctx, b_ns_router_id, {'router': {'routes': []}}) + return + + # handle extra routes for north-south router + ip_bridge_ip_map = {} + for router_id, cidr_ips_map in router_ips_map.iteritems(): + if router_id not in router_ew_bridge_ip_map: + continue + for cidr, ips in cidr_ips_map.iteritems(): + for ip in ips: + nexthop = router_ew_bridge_ip_map[router_id] + destination = ip + '/32' + ip_bridge_ip_map[destination] = nexthop + + bottom_client = self._get_client(pod_name=b_ns_pdd['pod_name']) + extra_routes = [] + for fixed_ip in ip_bridge_ip_map: + extra_routes.append( + {'nexthop': ip_bridge_ip_map[fixed_ip], + 'destination': fixed_ip}) + bottom_client.update_routers( + ctx, b_ns_router_id, {'router': {'routes': extra_routes}}) + @_job_handle(constants.JT_PORT_DELETE) def delete_server_port(self, ctx, payload): b_pod_id, b_port_id = payload[constants.JT_PORT_DELETE].split('#') @@ -727,14 +749,10 @@ class XManager(PeriodicTasks): subnets = top_client.list_subnets( ctx, [{'key': 'tenant_id', 'comparator': 'eq', 'value': project_id}]) - ew_bridge_ip_net = netaddr.IPNetwork( - CONF.client.ew_bridge_cidr) - ns_bridge_ip_net = netaddr.IPNetwork( - CONF.client.ns_bridge_cidr) + bridge_ip_net = netaddr.IPNetwork(CONF.client.bridge_cidr) for subnet in subnets: ip_net = netaddr.IPNetwork(subnet['cidr']) - if ip_net in ew_bridge_ip_net or ( - ip_net in ns_bridge_ip_net): + if ip_net in bridge_ip_net: continue # leave sg_id empty here new_b_rules.append(